add rollover to blob_stream_writer in b4.4

this will help reduce incomplete file writes as well.

ldms/scripts/examples/blob_writer_rollover

@@ -0,0 +1,35 @@
+export plugname=dstat
+export dsname=$(ldms_dstat_schema_name mmalloc=1 io=1 fd=1 auto-schema=1)
+export dstat_schema=$dsname
+portbase=61060
+# memcheck
+VGARGS="--trace-children=yes --track-origins=yes --leak-check=full --show-leak-kinds=all"
+# drd
+#VGARGS="--trace-children=yes  --tool=drd --trace-mutex=yes"
+# track everything notifier config:
+${BUILDDIR}/sbin/ldms-netlink-notifier --port=61061 --auth=none --reconnect=1 -D 60 -r -j $LOGDIR/json.log --exclude-dir-path= --exclude-short-path= --exclude-programs -v 1 &
+LDMSD 1
+#vgon
+LDMSD 2
+#vgoff
+MESSAGE ldms_ls on host 1:
+LDMS_LS 1 -l
+MESSAGE ldms_ls on host 2:
+LDMS_LS 2 -l
+for kt2 in $(seq 5); do
+#MESSAGE "trying rollover via reconfig: $(date +%s)"
+#echo "config name=blob_stream_writer path=${STOREDIR} container=blobs stream=slurm timing=1 types=1 spool=1" | \
+#ldmsctl -p 61062 -a none -x sock -h localhost
+for kt in $(seq 4); do
+	SLEEP 2
+done
+done
+SLEEP 1
+SLEEP 2
+SLEEP 2
+SLEEP 2
+SLEEP 1
+KILL_LDMSD 1 2
+file_created $STOREDIR/blobs/spool/slurm.TIMING.1*
+file_created $STOREDIR/blobs/spool/slurm.DAT.1*
+file_created $STOREDIR/blobs/spool/slurm.OFFSET.1*

ldms/scripts/examples/blob_writer_rollover.1

@@ -0,0 +1,3 @@
+#load name=dstat
+#config name=dstat producer=localhost${i} instance=localhost${i}/${dstat_schema} component_id=${i} mmalloc=1 io=1 fd=1 auto-schema=1)
+#start name=dstat interval=1000000 offset=0

ldms/scripts/examples/blob_writer_rollover.2

@@ -0,0 +1,17 @@
+load name=blob_stream_writer plugin=blob_stream_writer
+config name=blob_stream_writer path=${STOREDIR} container=blobs stream=slurm timing=1 types=1 spool=1 rolltype=3 rollover=10
+
+prdcr_add name=localhost1 host=${HOST} type=active xprt=${XPRT} port=${port1} interval=2000000
+prdcr_subscribe regex=.* stream=slurm
+prdcr_start name=localhost1
+
+updtr_add name=allhosts interval=1000000 offset=100000
+updtr_prdcr_add name=allhosts regex=.*
+updtr_start name=allhosts
+
+# load name=store_csv
+# config name=store_csv path=${STOREDIR} altheader=0
+
+# strgp_add name=store_${testname} plugin=store_csv schema=${dstat_schema} container=node
+# strgp_prdcr_add name=store_${testname} regex=.*
+# strgp_start name=store_${testname}

ldms/src/sampler/blob_stream/Makefile.am

@@ -6,9 +6,10 @@ dist_man7_MANS=
 
 AM_LDFLAGS = @OVIS_LIB_ABS@
 AM_CPPFLAGS = @OVIS_INCLUDE_ABS@
-STORE_LIBADD = $(top_builddir)/ldms/src/core/libldms.la \
-	       $(top_builddir)/lib/src/coll/libcoll.la \
-	       $(top_builddir)/lib/src/ovis_util/libovis_util.la
+STORE_LIBADD = $(top_builddir)/ldms/src/ldmsd/libldmsd_plugattr.la \
+	$(top_builddir)/ldms/src/core/libldms.la \
+	$(top_builddir)/lib/src/coll/libcoll.la \
+	$(top_builddir)/lib/src/ovis_util/libovis_util.la
 
 
 

ldms/src/sampler/blob_stream/Plugin_blob_stream_writer.man

@@ -47,11 +47,46 @@ Enable logging of messages stored to the log file.
 timing=1
 .br
 Enable writing timestamps to a separate file.
-.RE
 .TP
 spool=1
 .br
 Move closed files to the directory <path>/<container>/spool/.
+.TP
+rolltype=<rolltype>
+.br
+By default, the store does not rollover and the data is written to a continously open filehandle. Rolltype and rollover are used in conjunction to enable the store to manage rollover, including flushing before rollover. The header will be rewritten when a roll occurs. Valid options are:
+.RS
+.TP
+1
+.br
+wake approximately every rollover seconds and roll.
+Rollover is suppressed if no data at all has been written and rollempty=0.
+.TP
+2
+.br
+wake daily at rollover seconds after midnight (>=0) and roll.
+Rollover is suppressed if no data at all has been written and rollempty=0.
+.TP
+3
+.br
+roll after approximately rollover records are written.
+.TP
+4
+roll after approximately rollover bytes are written.
+.TP
+5
+.br
+wake at rollover seconds after midnight (>=0) and roll, then repeat every rollagain (> rollover) seconds during the day. For example "rollagain=3600 rollover=0 rolltype=5" rolls files hourly.
+Rollover is suppressed if no data at all has been written and rollempty=0.
+.RE
+.TP
+rollover=<rollover>
+.br
+Rollover value controls the frequency of rollover (e.g., number of bytes, number of records, time interval, seconds after midnight). Note that these values are estimates.
+.TP
+rollempty=0
+.br
+Turn off rollover of empty files. Default value is 1 (create extra empty files).
 .RE
 
 .SH OUTPUT FORMAT
@@ -86,7 +121,8 @@ This writer is in development and may be changed at any time.
 Cannot support stream=.* as there is no corresponding regex subscription policy
 currently available in the C stream API.
 .PP
-The config operation may called at any time or repeated.
+The config operation may called at any time or repeated, though the use of rollover
+policies is recommended instead. Repeated configuration of rollover is silently ignored.
 The start and stop operations will start and stop storage of all streams.
 .PP
 The plugin appears in C code as a sampler plugin, since the storage policy and store