Merge branch 'pct_brace_store_template_34' into 'OVIS-3.4'

%{var} for csv rollover template v3.4

This adds %{envvar} support to renaming templates used in csv rollover, per a request from LLNL.
Test and docs included. Runs valgrind clean. Ancillary log messages repaired for missing labels.

See merge request !1031

ldms/man/Plugin_store_csv.man

@@ -1,6 +1,6 @@
 .\" Manpage for Plugin_store_csv
 .\" Contact [email protected] to correct errors or typos.
-.TH man 7 "22 Oct 2017" "v3.5" "LDMS Plugin store_csv man page"
+.TH man 7 "18 Sep 2018" "v3.10" "LDMS Plugin store_csv man page"
 
 .SH NAME
 Plugin_store_csv - man page for the LDMS store_csv plugin
@@ -28,7 +28,7 @@ The configuration parameters altheader, rolltype, rollover, buffer, buffertype,
 A subset of these can be overriden by the action=custom values.
 .TP
 .BR config
-name=<plugin_name> action=init path=<path> [ altheader=<0/!0> buffer=<0/1/N> buffertype=<3/4> rolltype=<rolltype> rollover=<rollover> userdata=<0/!0>] [notify=<path> [notify_isfifo=<bool>]] [rename_template=<metapath> [rename_uid=<int-uid> [rename_gid=<int-gid] rename_perm=<octal-mode>]] [create_uid=<int-uid> [create_gid=<int-gid] create_perm=<octal-mode>]
+name=<plugin_name> action=init path=<path> [ altheader=<0/!0> buffer=<0/1/N> buffertype=<3/4> rolltype=<rolltype> rollover=<rollover> userdata=<0/!0>] [notify=<path> [notify_isfifo=<bool>]] [rename_template=<metapath> [rename_uid=<int-uid> [rename_gid=<int-gid] rename_perm=<octal-mode>]] [create_uid=<int-uid>] [create_gid=<int-gid] [create_perm=<octal-mode>]
 .br
 ldmsd_controller configuration line
 .RS
@@ -111,20 +111,22 @@ Specify a new group id for data files. If unspecified, no change in group owners
 .TP
 rename_template=<metapath>
 .br
-This option provides relocating closed CSV files, typically to a subdirectory, for processing by other tools that watch directories. The metapath template is applied to define a rename after file closure. The rename is limited to location on the same mount point, per the C rename(2) call. The following substitutions in the template are performed: %P by plugin name, %C by container name, %S by schema name, %T by file event notification type, %B by basename(closed-file-name), %D by dirname(closed-file-name), %s by timestamp suffix, if it exists. Errors in template specification will cause the rename to be skipped. As part of the rename process, the mode and ownership of the file may also be adjusted by specifying rename_perm, rename_uid, and rename_gid. Missing intermediate directories will be created if possible. To enable greater flexibility than renaming, the notify option must be used instead of the rename_template option.
+This option relocates closed CSV files, typically to a subdirectory, for processing by other tools that watch directories. The metapath template is applied to define a new name after file closure. The rename is limited to locations on the same mount point, per the C rename(2) call. Substitutions (%) in the provided template are performed as described in METAPATH SUBSTITUTIONS below.
+Errors in template specification will cause the rename to be skipped. As part of the renaming process, the mode and ownership of the file may also be adjusted by specifying rename_perm, rename_uid, and rename_gid. Missing intermediate directories will be created if possible. To enable greater flexibility than the renaming just described (e.g. crossing file systems), the notify option must be used to call another program.
+
 .TP
 rename_perm=<modebits>
 .br
-Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The permissions are changed before the rename and even if the rename fails.
+Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The permissions are changed before the rename and even if the rename fails. This option is applied only if rename_template is applied.
 .TP
 rename_uid=<numeric-uid>
 .br
 Specify a new user id for the file. If unspecified, no change in user ownership is made.
-Changes in ownership of the files do not affect intermediate directories that might be created following the template.
+Changes in ownership of the files do not affect intermediate directories that might be created following the template. This option is applied only if rename_template is applied.
 .TP
 rename_gid=<numeric-gid>
 .br
-Specify a new group id for the file. If unspecified, no change in group ownership is made.
+Specify a new group id for the file. If unspecified, no change in group ownership is made. This option is applied only if rename_template is applied.
 
 .RE
 
@@ -135,7 +137,7 @@ unless overriden by the action=custom values. Only a subset can be overridden.
 
 .TP
 .BR config
-name=<plugin_name> action=custom container=<container> schema=<schema> [ altheader=<0/!0> userdata=<0/!0>] [notify=<path> [notify_isfifo=<bool>]] [rename_template=<metapath> [rename_uid=<int-uid> [rename_gid=<int-gid] rename_perm=<octal-mode>]]
+name=<plugin_name> action=custom container=<container> schema=<schema> [ altheader=<0/!0> userdata=<0/!0>] [notify=<path> [notify_isfifo=<bool>]] [rename_template=<metapath> [rename_uid=<int-uid> [rename_gid=<int-gid] rename_perm=<octal-mode>]] [create_uid=<int-uid>] [create_gid=<int-gid] [create_perm=<octal-mode>]
 
 .br
 ldmsd_controller configuration line
@@ -301,7 +303,44 @@ The name of the file.
 
 .RE
 
+.SH METAPATH SUBSTITUTION
 
+The following % escape sequence replacements are performed on the rename_template value for file renamings:
+.PP
+.TP
+%P 
+.br
+plugin name
+.TP
+%C
+.br
+container name
+.TP
+%S
+.br
+schema name
+.TP
+%T 
+.br
+file event notification type
+.TP
+%B
+.br
+basename(closed-file-name)
+.TP
+%D
+.br
+dirname(closed-file-name)
+.TP
+%{ENV_VAR_NAME}
+.br
+getenv(ENV_VAR_NAME). The use of undefined or empty environment vars yields an empty substitution, not an error.
+Characters in the environment variable are restricted to: 'A-Za-z0-9%@()+-_./:='; other characters present will prevent the rename.
+.TP
+%s
+.br
+timestamp suffix, if it exists. 
+.PP
 
 .SH NOTES
 .PP
@@ -315,7 +354,7 @@ The 'sequence' option has been removed.
 There is a maximum of 20 concurrent CSV stores.
 
 .SH IMPERFECT FEATURES
-The rename and create options do not accept symbolic permissions, uid, or gid.
+The rename and create options do not accept symbolic permissions, uid, or gid. There is no metapath substitution for file creation.
 
 .SH EXAMPLES
 .PP
@@ -326,6 +365,11 @@ config name=store_csv action=init altheader=1 path=/XXX/storedir
 config name=store_csv action=custom container=loadavg_store schema=loadavg
 strgp_add name=csv_mem_policy plugin=store_csv container=loadavg_store schema=loadavg
 .fi
+Or with modifications for file properties
+.nf
+load name=store_csv
+config name=store_csv action=init altheader=1 rolltype=2 rollover=0 path=/mprojects/ovis/ClusterData/${LDMSCLUSTER} create_gid=1000000039 create_perm=640 rename_template=%D/archive-spool/%{HOSTNAME}/%B rename_perm=444
+.fi
 
 .SH SEE ALSO
 ldmsd(8), ldms_quickstart(7), ldmsd_controller(8)

ldms/man/ldms-static-test.man

@@ -6,7 +6,11 @@
 ldms-static-test.sh \- Run a canned test scenario
 
 .SH SYNOPSIS
-ldms-static-test.sh [-l]
+.PP
+ldms-static-test.sh -l
+.PP
+ldms-static-test.sh -h
+.PP
 ldms-static-test.sh <input_file> [test_dir]
 
 .SH DESCRIPTION
@@ -22,6 +26,10 @@ happen as a system service) and shut down with a signal.
 -l
 .br
 List the canned tests available.
+.TP
+-h
+.br
+List help message.
 
 .SH LANGUAGE
 
@@ -79,6 +87,10 @@ file_created <filename>
 .br
 Verifies the existence and readability of filename.
 .TP
+rollover_created <filename>
+.br
+Verifies the existence and readability of rollover files matching pattern filename.[0-9]*.
+.TP
 bypass=<0,1>
 .br
 This variable assignment disables (1) or enables (0) all the macros described

ldms/scripts/examples/.canned

@@ -20,6 +20,7 @@ procnfs
 rabbitkw
 rabbitl2remote
 rabbitv3
+renamecsv
 
 # working with job metrics
 clock.job

ldms/scripts/examples/prolog.store2_rename

@@ -0,0 +1,13 @@
+load name=store_csv
+config name=store_csv action=init path=${STOREDIR} altheader=0 container=node1 rollover=2 rolltype=1 rename_template=${STOREDIR}/%{HOSTNAME}/%B
+
+prdcr_add name=localhost1 host=localhost type=active xprt=${XPRT} port=${port1} interval=10000000
+prdcr_start name=localhost1
+
+updtr_add name=allhosts interval=1000000 offset=100000
+updtr_prdcr_add name=allhosts regex=.*
+updtr_start name=allhosts
+
+strgp_add name=store_${testname} plugin=store_csv schema=${testname} container=node
+strgp_prdcr_add name=store_${testname} regex=.*
+strgp_start name=store_${testname}

ldms/scripts/examples/renamecsv

@@ -0,0 +1,15 @@
+export plugname=meminfo
+portbase=61096
+#export VGARGS="--track-origins=yes --leak-check=full"
+LDMSD -p prolog.sampler 1
+#vgon
+LDMSD -p prolog.store2_rename 2
+#vgoff
+MESSAGE ldms_ls on host 1:
+LDMS_LS 1 -l
+MESSAGE ldms_ls on host 2:
+LDMS_LS 2 -l
+SLEEP 15
+KILL_LDMSD `seq 2`
+rollover_created $STOREDIR/$HOSTNAME/$testname
+

ldms/scripts/ldms-static-test.sh.in

@@ -16,12 +16,27 @@ export PACKAGE_TARNAME=@PACKAGE_TARNAME@
 export docdir=@docdir@
 export exdir=@docdir@-@PACKAGE_VERSION@
 
+function clusage {
+cat << EOF
+$0: usage:
+$0 -l 
+$0 -h
+$0 <input_file> [test_dir]
+	-h produces help message
+	-l produces list of canned test names
+See man ldms-static-test(8) for details.
+EOF
+}
 input=$1
 export input
 if test -z "$1"; then
 	echo "FAIL: no input file to $0"
 	exit 1
 fi
+if test "$1" = "-h"; then
+	clusage
+	exit 0
+fi
 if test "$1" = "-l"; then
 	echo "Canned tests are:"
 	echo "$exdir/examples/static-test/"
@@ -298,6 +313,25 @@ function KILL_LDMSD {
 	done
 }
 
+function rollover_created {
+	if test "$bypass" = "1"; then
+		return 0
+	fi
+	x=`ls $1.[0-9]*`
+	if test -z "$x"; then
+		echo "FAIL: roll-over files $1.* not created."
+		bypass=1
+		return 1
+	fi
+	for i in $x; do
+		if ! test -r $i; then
+			echo FAIL: file $i not readable.
+			bypass=1
+			return 1
+		fi
+	done
+	return 0
+}
 function file_created {
 	if test "$bypass" = "1"; then
 		return 0