diff --git a/ldms/man/Plugin_store_csv.man b/ldms/man/Plugin_store_csv.man index 44357c99a..728bcb893 100644 --- a/ldms/man/Plugin_store_csv.man +++ b/ldms/man/Plugin_store_csv.man @@ -1,6 +1,6 @@ .\" Manpage for Plugin_store_csv .\" Contact ovis-help@ca.sandia.gov to correct errors or typos. -.TH man 7 "22 Oct 2017" "v3.5" "LDMS Plugin store_csv man page" +.TH man 7 "18 Sep 2018" "v3.10" "LDMS Plugin store_csv man page" .SH NAME Plugin_store_csv - man page for the LDMS store_csv plugin @@ -28,7 +28,7 @@ The configuration parameters altheader, rolltype, rollover, buffer, buffertype, A subset of these can be overriden by the action=custom values. .TP .BR config -name= action=init path= [ altheader=<0/!0> buffer=<0/1/N> buffertype=<3/4> rolltype= rollover= userdata=<0/!0>] [notify= [notify_isfifo=]] [rename_template= [rename_uid= [rename_gid=]] [create_uid= [create_gid=] +name= action=init path= [ altheader=<0/!0> buffer=<0/1/N> buffertype=<3/4> rolltype= rollover= userdata=<0/!0>] [notify= [notify_isfifo=]] [rename_template= [rename_uid= [rename_gid=]] [create_uid=] [create_gid=] .br ldmsd_controller configuration line .RS @@ -111,20 +111,22 @@ Specify a new group id for data files. If unspecified, no change in group owners .TP rename_template= .br -This option provides relocating closed CSV files, typically to a subdirectory, for processing by other tools that watch directories. The metapath template is applied to define a rename after file closure. The rename is limited to location on the same mount point, per the C rename(2) call. The following substitutions in the template are performed: %P by plugin name, %C by container name, %S by schema name, %T by file event notification type, %B by basename(closed-file-name), %D by dirname(closed-file-name), %s by timestamp suffix, if it exists. Errors in template specification will cause the rename to be skipped. As part of the rename process, the mode and ownership of the file may also be adjusted by specifying rename_perm, rename_uid, and rename_gid. Missing intermediate directories will be created if possible. To enable greater flexibility than renaming, the notify option must be used instead of the rename_template option. +This option relocates closed CSV files, typically to a subdirectory, for processing by other tools that watch directories. The metapath template is applied to define a new name after file closure. The rename is limited to locations on the same mount point, per the C rename(2) call. Substitutions (%) in the provided template are performed as described in METAPATH SUBSTITUTIONS below. +Errors in template specification will cause the rename to be skipped. As part of the renaming process, the mode and ownership of the file may also be adjusted by specifying rename_perm, rename_uid, and rename_gid. Missing intermediate directories will be created if possible. To enable greater flexibility than the renaming just described (e.g. crossing file systems), the notify option must be used to call another program. + .TP rename_perm= .br -Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The permissions are changed before the rename and even if the rename fails. +Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The permissions are changed before the rename and even if the rename fails. This option is applied only if rename_template is applied. .TP rename_uid= .br Specify a new user id for the file. If unspecified, no change in user ownership is made. -Changes in ownership of the files do not affect intermediate directories that might be created following the template. +Changes in ownership of the files do not affect intermediate directories that might be created following the template. This option is applied only if rename_template is applied. .TP rename_gid= .br -Specify a new group id for the file. If unspecified, no change in group ownership is made. +Specify a new group id for the file. If unspecified, no change in group ownership is made. This option is applied only if rename_template is applied. .RE @@ -135,7 +137,7 @@ unless overriden by the action=custom values. Only a subset can be overridden. .TP .BR config -name= action=custom container= schema= [ altheader=<0/!0> userdata=<0/!0>] [notify= [notify_isfifo=]] [rename_template= [rename_uid= [rename_gid=]] +name= action=custom container= schema= [ altheader=<0/!0> userdata=<0/!0>] [notify= [notify_isfifo=]] [rename_template= [rename_uid= [rename_gid=]] [create_uid=] [create_gid=] .br ldmsd_controller configuration line @@ -301,7 +303,44 @@ The name of the file. .RE +.SH METAPATH SUBSTITUTION +The following % escape sequence replacements are performed on the rename_template value for file renamings: +.PP +.TP +%P +.br +plugin name +.TP +%C +.br +container name +.TP +%S +.br +schema name +.TP +%T +.br +file event notification type +.TP +%B +.br +basename(closed-file-name) +.TP +%D +.br +dirname(closed-file-name) +.TP +%{ENV_VAR_NAME} +.br +getenv(ENV_VAR_NAME). The use of undefined or empty environment vars yields an empty substitution, not an error. +Characters in the environment variable are restricted to: 'A-Za-z0-9%@()+-_./:='; other characters present will prevent the rename. +.TP +%s +.br +timestamp suffix, if it exists. +.PP .SH NOTES .PP @@ -315,7 +354,7 @@ The 'sequence' option has been removed. There is a maximum of 20 concurrent CSV stores. .SH IMPERFECT FEATURES -The rename and create options do not accept symbolic permissions, uid, or gid. +The rename and create options do not accept symbolic permissions, uid, or gid. There is no metapath substitution for file creation. .SH EXAMPLES .PP @@ -326,6 +365,11 @@ config name=store_csv action=init altheader=1 path=/XXX/storedir config name=store_csv action=custom container=loadavg_store schema=loadavg strgp_add name=csv_mem_policy plugin=store_csv container=loadavg_store schema=loadavg .fi +Or with modifications for file properties +.nf +load name=store_csv +config name=store_csv action=init altheader=1 rolltype=2 rollover=0 path=/mprojects/ovis/ClusterData/${LDMSCLUSTER} create_gid=1000000039 create_perm=640 rename_template=%D/archive-spool/%{HOSTNAME}/%B rename_perm=444 +.fi .SH SEE ALSO ldmsd(8), ldms_quickstart(7), ldmsd_controller(8) diff --git a/ldms/man/ldms-static-test.man b/ldms/man/ldms-static-test.man index c52a00bdf..cebfcff3d 100644 --- a/ldms/man/ldms-static-test.man +++ b/ldms/man/ldms-static-test.man @@ -6,7 +6,11 @@ ldms-static-test.sh \- Run a canned test scenario .SH SYNOPSIS -ldms-static-test.sh [-l] +.PP +ldms-static-test.sh -l +.PP +ldms-static-test.sh -h +.PP ldms-static-test.sh [test_dir] .SH DESCRIPTION @@ -22,6 +26,10 @@ happen as a system service) and shut down with a signal. -l .br List the canned tests available. +.TP +-h +.br +List help message. .SH LANGUAGE @@ -79,6 +87,10 @@ file_created .br Verifies the existence and readability of filename. .TP +rollover_created +.br +Verifies the existence and readability of rollover files matching pattern filename.[0-9]*. +.TP bypass=<0,1> .br This variable assignment disables (1) or enables (0) all the macros described diff --git a/ldms/scripts/examples/.canned b/ldms/scripts/examples/.canned index df9e3db3c..6ddef5fe0 100644 --- a/ldms/scripts/examples/.canned +++ b/ldms/scripts/examples/.canned @@ -20,6 +20,7 @@ procnfs rabbitkw rabbitl2remote rabbitv3 +renamecsv # working with job metrics clock.job diff --git a/ldms/scripts/examples/prolog.store2_rename b/ldms/scripts/examples/prolog.store2_rename new file mode 100644 index 000000000..091e84776 --- /dev/null +++ b/ldms/scripts/examples/prolog.store2_rename @@ -0,0 +1,13 @@ +load name=store_csv +config name=store_csv action=init path=${STOREDIR} altheader=0 container=node1 rollover=2 rolltype=1 rename_template=${STOREDIR}/%{HOSTNAME}/%B + +prdcr_add name=localhost1 host=localhost type=active xprt=${XPRT} port=${port1} interval=10000000 +prdcr_start name=localhost1 + +updtr_add name=allhosts interval=1000000 offset=100000 +updtr_prdcr_add name=allhosts regex=.* +updtr_start name=allhosts + +strgp_add name=store_${testname} plugin=store_csv schema=${testname} container=node +strgp_prdcr_add name=store_${testname} regex=.* +strgp_start name=store_${testname} diff --git a/ldms/scripts/examples/renamecsv b/ldms/scripts/examples/renamecsv new file mode 100644 index 000000000..fd694076a --- /dev/null +++ b/ldms/scripts/examples/renamecsv @@ -0,0 +1,15 @@ +export plugname=meminfo +portbase=61096 +#export VGARGS="--track-origins=yes --leak-check=full" +LDMSD -p prolog.sampler 1 +#vgon +LDMSD -p prolog.store2_rename 2 +#vgoff +MESSAGE ldms_ls on host 1: +LDMS_LS 1 -l +MESSAGE ldms_ls on host 2: +LDMS_LS 2 -l +SLEEP 15 +KILL_LDMSD `seq 2` +rollover_created $STOREDIR/$HOSTNAME/$testname + diff --git a/ldms/scripts/examples/renamecsv.1 b/ldms/scripts/examples/renamecsv.1 new file mode 100644 index 000000000..e69de29bb diff --git a/ldms/scripts/examples/renamecsv.2 b/ldms/scripts/examples/renamecsv.2 new file mode 100644 index 000000000..e69de29bb diff --git a/ldms/scripts/ldms-static-test.sh.in b/ldms/scripts/ldms-static-test.sh.in index b2f93885d..c95e17cda 100755 --- a/ldms/scripts/ldms-static-test.sh.in +++ b/ldms/scripts/ldms-static-test.sh.in @@ -16,12 +16,27 @@ export PACKAGE_TARNAME=@PACKAGE_TARNAME@ export docdir=@docdir@ export exdir=@docdir@-@PACKAGE_VERSION@ +function clusage { +cat << EOF +$0: usage: +$0 -l +$0 -h +$0 [test_dir] + -h produces help message + -l produces list of canned test names +See man ldms-static-test(8) for details. +EOF +} input=$1 export input if test -z "$1"; then echo "FAIL: no input file to $0" exit 1 fi +if test "$1" = "-h"; then + clusage + exit 0 +fi if test "$1" = "-l"; then echo "Canned tests are:" echo "$exdir/examples/static-test/" @@ -298,6 +313,25 @@ function KILL_LDMSD { done } +function rollover_created { + if test "$bypass" = "1"; then + return 0 + fi + x=`ls $1.[0-9]*` + if test -z "$x"; then + echo "FAIL: roll-over files $1.* not created." + bypass=1 + return 1 + fi + for i in $x; do + if ! test -r $i; then + echo FAIL: file $i not readable. + bypass=1 + return 1 + fi + done + return 0 +} function file_created { if test "$bypass" = "1"; then return 0 diff --git a/ldms/src/store/store_csv_common.c b/ldms/src/store/store_csv_common.c index c2427c5c3..d453429d7 100644 --- a/ldms/src/store/store_csv_common.c +++ b/ldms/src/store/store_csv_common.c @@ -240,6 +240,41 @@ void notify_output(const char *event, const char *name, const char *ftype, free(msg); } +/* Disallow odd characters and space in environment variables + * for template assembly. + * Allow A-z0-9%@()+-_./:= + */ +static int validate_env(const char *var, const char *val, struct csv_plugin_static *cps) { + int rc = 0; + const char *c = val; + const char *b = NULL; + for ( ; *c != '\0'; c++) { + switch (*c) { + case '%': + case '(': + case ')': + case '+': + case '-': + case '=': + case '_': + case '.': + case '/': + case ':': + case '@': + break; + default: + if (!rc && !isalnum(*c)) { + rc = ENOTSUP; + b = c; + } + } + } + if (rc) + cps->msglog(LDMSD_LERROR, "%s: rename_output: unsupported character %c in template use of env(%s): %s\n", + cps->pname, *b, var, val); + return rc; +} + int create_outdir(const char *path, struct csv_store_handle_common *s_handle, struct csv_plugin_static *cps) { #define EBSIZE 512 @@ -325,8 +360,8 @@ void rename_output(const char *name, int rc = errno; if (merr) { strerror_r(rc, errbuf, EBSIZE); - cps->msglog(LDMSD_LERROR,"rename_output: unable to chmod(%s,%o): %s.\n", - name, s_handle->rename_perm, errbuf); + cps->msglog(LDMSD_LERROR,"%s: rename_output: unable to chmod(%s,%o): %s.\n", + cps->pname, name, s_handle->rename_perm, errbuf); } } @@ -339,8 +374,8 @@ void rename_output(const char *name, int rc = errno; if (merr) { strerror_r(rc, errbuf, EBSIZE); - cps->msglog(LDMSD_LERROR,"rename_output: unable to chown(%s, %u, %u): %s.\n", - name, newuid, newgid, errbuf); + cps->msglog(LDMSD_LERROR,"%s: rename_output: unable to chown(%s, %u, %u): %s.\n", + cps->pname, name, newuid, newgid, errbuf); } } @@ -375,7 +410,7 @@ void rename_output(const char *name, dscat(ds, bname); free(namedup); } else { - cps->msglog(LDMSD_LERROR,"rename_output: ENOMEM\n"); + cps->msglog(LDMSD_LERROR,"%s: rename_output: ENOMEM\n", cps->pname); dstr_free(&ds); return; } @@ -388,16 +423,51 @@ void rename_output(const char *name, dscat(ds, dname); free(namedup); } else { - cps->msglog(LDMSD_LERROR,"rename_output: ENOMEM\n"); + cps->msglog(LDMSD_LERROR,"%s: rename_output: ENOMEM\n", cps->pname); dstr_free(&ds); return; } break; + case '{': + head = end + 2; + char *vend = strchr(head,'}'); + if (!vend) { + cps->msglog(LDMSD_LERROR, + "%s: rename_output: unterminated %%{ in template at %s\n", + cps->pname, head); + dstr_free(&ds); + return; + } else { + size_t vlen = vend - head + 1; + char var[vlen]; + memset(var, 0, vlen); + strncpy(var, head, vlen-1); + var[vlen] = '\0'; + head = vend + 1; + char *val = getenv(var); + if (val) { + cps->msglog(LDMSD_LDEBUG, + "%s: rename_output: getenv(%s) = %s\n", cps->pname, var, val); + if (validate_env(var, val, cps)) { + dstr_free(&ds); + cps->msglog(LDMSD_LERROR, + "%s: rename_output: rename cancelled\n", + cps->pname); + return; + } + dscat(ds, val); + } else { + cps->msglog(LDMSD_LDEBUG, + "%s: rename_output: empty %%{%s}\n", + cps->pname, var); + } + } + break; case 's': head = end + 2; char *dot = strrchr(name,'.'); if (!dot) { - cps->msglog(LDMSD_LERROR,"rename_output: no timestamp\n"); + cps->msglog(LDMSD_LERROR,"%s: rename_output: no timestamp\n", cps->pname); dstr_free(&ds); return; } @@ -407,7 +477,7 @@ void rename_output(const char *name, num++; } if (*num != '\0') { - cps->msglog(LDMSD_LERROR,"rename_output: no timestamp at end\n"); + cps->msglog(LDMSD_LERROR,"%s: rename_output: no timestamp at end\n", cps->pname); dstr_free(&ds); return; } @@ -424,8 +494,8 @@ void rename_output(const char *name, char *newname = dsdone(ds); dstr_free(&ds); if (!newname) { - cps->msglog(LDMSD_LERROR,"rename_output: failed to create new filename for %s\n", - name); + cps->msglog(LDMSD_LERROR,"%s: rename_output: failed to create new filename for %s\n", + cps->pname, name); return; } @@ -455,20 +525,24 @@ void rename_output(const char *name, break; default: strerror_r(err, errbuf, EBSIZE); - cps->msglog(LDMSD_LERROR,"rename_output: failed to create directory for %s: %s\n", - newname, errbuf); + cps->msglog(LDMSD_LERROR, "%s: rename_output: failed to create directory for %s: %s\n", + cps->pname, newname, errbuf); return; } } - cps->msglog(LDMSD_LDEBUG,"rename_output: rename(%s, %s)\n", name, newname); + cps->msglog(LDMSD_LDEBUG, "%s: rename_output: rename(%s, %s)\n", + cps->pname, name, newname); err = rename(name, newname); if (err) { int ec = errno; + if (ec != ENOENT) { strerror_r(ec, errbuf, EBSIZE); - cps->msglog(LDMSD_LERROR,"rename_output: failed rename(%s, %s): %s\n", - name, newname, errbuf); + cps->msglog(LDMSD_LERROR,"%s: rename_output: failed rename(%s, %s): %s\n", + cps->pname, name, newname, errbuf); + } + /* enoent happens if altheader = 0 */ } free(newname); #undef EBSIZE diff --git a/ldms/src/store/store_csv_common.h b/ldms/src/store/store_csv_common.h index 2b0d580eb..1d7667d9e 100644 --- a/ldms/src/store/store_csv_common.h +++ b/ldms/src/store/store_csv_common.h @@ -202,6 +202,7 @@ int create_outdir(const char *path, * %B expands to basename(name), * %D expands to dirname(name), * %s timestamp suffix, if it exists. + * %{var} expands to env(var) * Specifying both output event notification and output * renaming produces a race condition between this function * and the event-processor and should be avoided.