add log_level integration test and reduce order dependence between load and log_level

ldms/scripts/examples/loglvl

@@ -0,0 +1,49 @@
+export plugname=dstat
+portbase=61028
+VGARGS="--trace-children=yes --track-origins=yes --leak-check=full --show-leak-kinds=all"
+vgoff
+/bin/rm -f ${LOGDIR}/log_config.[1-3]
+LDMSD_EXTRA="-L 54:${LOGDIR}/log_config.1"
+LDMSD 1
+vgoff
+LDMSD_EXTRA="-L 2:${LOGDIR}/log_config.2"
+LDMSD 2
+LDMSD_EXTRA="-L 31:${LOGDIR}/log_config.3"
+LDMSD -p prolog.sampler 3
+MESSAGE ldms_ls on host 1:
+LDMS_LS 1 -v
+MESSAGE ldms_ls on host 2:
+LDMS_LS 2 -v
+MESSAGE ldms_ls on host 3:
+LDMS_LS 3
+MESSAGE "trying log_status on host 1"
+echo "log_status " | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "trying log_level=INFO on host 1"
+echo "log_level level=INFO" | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "trying log_status on host 1"
+echo "log_status " | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "trying log_status on host 3"
+echo "log_status " | ldmsctl -p 61031 -a none -x sock -h localhost
+MESSAGE "trying log_level name=sampler.bogon level=INFO on host 1"
+echo "log_level name=sampler.bogon level=info" | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "trying log_level name=sampler.dstat level=DEFAULT on host 1"
+echo "log_level name=sampler.dstat level=default" | ldmsctl -p 61029 -a none -x sock -h localhost
+SLEEP 5
+MESSAGE "trying log_status"
+echo "log_status " | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "trying log_level name=sampler.dstat level=default on host 1"
+echo "log_level name=sampler.dstat level=default" | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "trying log_level name=store.store_csv level=ERROR on host 3"
+echo "log_level name=sampler.dstat level=ERROR" | ldmsctl -p 61031 -a none -x sock -h localhost
+SLEEP 3
+MESSAGE "trying log_status all 3"
+MESSAGE "========== on 1"
+echo "log_status " | ldmsctl -p 61029 -a none -x sock -h localhost
+MESSAGE "========== on 2"
+echo "log_status " | ldmsctl -p 61030 -a none -x sock -h localhost
+MESSAGE "========== on 3"
+echo "log_status " | ldmsctl -p 61031 -a none -x sock -h localhost
+KILL_LDMSD `seq 3`
+file_created $STOREDIR/node/$testname
+if grep 'DEBUG: sampler.dstat'  $LOGDIR/1.txt >/dev/null ; then echo "FAIL: unexpected DEBUG: sampler.dstat in $LOGDIR/1.txt"; fi
+if ! grep 'DEBUG: sampler.dstat'  $LOGDIR/2.txt >/dev/null ; then echo "FAIL: expected DEBUG: sampler.dstat in $LOGDIR/2.txt"; fi

ldms/scripts/examples/loglvl.1

@@ -0,0 +1,9 @@
+# set default level warning, overriding startup
+log_level level=WARNING
+load name=${plugname}
+log_level name=sampler.dstat level=INFO
+# show config debug in log
+config name=${plugname} producer=localhost${i} schema=${plugname} instance=localhost${i}/${plugname} component_id=${i}
+start name=${plugname} interval=1000000 offset=0
+# stop logging debug
+log_level name=sampler.dstat level=DEBUG

ldms/scripts/examples/loglvl.2

@@ -0,0 +1,4 @@
+load name=${plugname}
+log_level name=sampler.dstat level=DEBUG
+config name=${plugname} producer=localhost${i} schema=${testname} instance=localhost${i}/${testname} component_id=${i}
+start name=${plugname} interval=1000000 offset=0

ldms/scripts/examples/loglvl.3

@@ -0,0 +1,19 @@
+log_level level=WARNING
+log_level name=store.store_csv level=INFO
+load name=store_csv
+config name=store_csv path=${STOREDIR} altheader=0
+
+prdcr_add name=localhost1 host=${HOST} type=active xprt=${XPRT} port=${port1} reconnect=10000000
+prdcr_start name=localhost1
+
+prdcr_add name=localhost2 host=${HOST} type=active xprt=${XPRT} port=${port2} reconnect=10000000
+prdcr_start name=localhost2
+
+updtr_add name=allhosts interval=1000000 offset=100000
+updtr_prdcr_add name=allhosts regex=.*
+updtr_start name=allhosts
+
+strgp_add name=store_${testname} plugin=store_csv schema=${testname} container=node
+strgp_prdcr_add name=store_${testname} regex=.*
+strgp_start name=store_${testname}
+log_status

lib/src/ovis_log/ovis_log.c

@@ -61,8 +61,18 @@
 #include "ovis_ev/ev.h"
 #include "ovis_util/util.h"
 
+const char *ovis_log_desc_pending = "(pending registration)";
+
 #include "ovis_log.h"
 
+struct ovis_log_s {
+        const char *name;
+        const char *desc; /* allocated pointer or ovis_log_desc_pending (if plugin not yet loaded) */
+        int level;
+        struct rbn rbn;
+        int ref_count;
+};
+
 static ev_worker_t logger_w;
 static ev_type_t log_type;
 static int is_init;
@@ -90,6 +100,7 @@ static int subsys_cmp(void *a, const void *b)
 static struct rbt subsys_tree = RBT_INITIALIZER(subsys_cmp);
 static pthread_mutex_t subsys_tree_lock = PTHREAD_MUTEX_INITIALIZER;
 
+
 #define OVIS_DEFAULT_FILE_PERM 0600
 
 struct log_data {
@@ -178,8 +189,11 @@ int ovis_log_set_level_by_name(const char *name, int level)
 	ovis_log_t log = NULL;
 	if (name) {
 		log = __find_log(name);
-		if (!log)
-			return ENOENT;
+		if (!log) {
+			log = ovis_log_register(name, ovis_log_desc_pending);
+			if (!log)
+				return ENOMEM;
+		}
 	}
 	return ovis_log_set_level(log, level);
 }
@@ -378,8 +392,11 @@ char *ovis_log_level_to_str(int level)
 
 static void __free_log(ovis_log_t log)
 {
+	if (!log)
+		return;
 	free((char *)log->name);
-	free((char *)log->desc);
+	if (log->desc != ovis_log_desc_pending)
+		free((char *)log->desc);
 	free(log);
 }
 
@@ -415,6 +432,15 @@ ovis_log_t ovis_log_register(const char *subsys_name, const char *desc)
 
 	log = __find_log(subsys_name);
 	if (log) {
+		if (log->desc == ovis_log_desc_pending) {
+			/* update desc if registered previously via set_level. */
+			log->desc = strdup(desc);
+			if (!log->desc) {
+				errno = ENOMEM;
+				return NULL;
+			}
+			return log;
+		}
 		errno = EEXIST;
 		return NULL;
 	}
@@ -428,7 +454,11 @@ ovis_log_t ovis_log_register(const char *subsys_name, const char *desc)
 		errno = ENOMEM;
 		goto free_log;
 	}
-	log->desc = strdup(desc);
+	if (desc == ovis_log_desc_pending) {
+		log->desc = ovis_log_desc_pending;
+	} else {
+		log->desc = strdup(desc);
+	}
 	if (!log->desc) {
 		errno = ENOMEM;
 		goto free_log;

lib/src/ovis_log/ovis_log.h

@@ -85,13 +85,7 @@
 #define OVIS_LALWAYS	(OVIS_LDEBUG|OVIS_LINFO|OVIS_LWARN|OVIS_LERROR|OVIS_LCRITICAL)
 #define OVIS_ALL_LEVELS	OVIS_LALWAYS
 
-typedef struct ovis_log_s {
-	const char *name;
-	const char *desc;
-	int level;
-	struct rbn rbn;
-	int ref_count;
-} *ovis_log_t;
+typedef struct ovis_log_s *ovis_log_t;
 
 /**
  * \brief Initialize the logging system
@@ -293,6 +287,11 @@ int ovis_vlog(ovis_log_t log, int level, const char *fmt, va_list ap);
  *  If \c subsys_name or \c mylog is NULL, and \c level is OVIS_LDEFAULT,
  *  \c level is ignored and 0 is returned.
  *
+ * For set_level_by_name if the subsys_name is not yet registered,
+ * it is registered with the description still pending.
+ * A subsequent call to ovis_log_register will succeed
+ * and retain the level set.
+ *
  * \param regex_s         A regular expression string to match subsystem names
  * \param subsys_name     The name of the subsystem to set the log level.
  *                        If NULL is given, the default log level will be set.