Skip to content

Commit

Permalink
Merge branch 'flatfile_update1' into 'OVIS-3.4'
Browse files Browse the repository at this point in the history 
Flatfile update

flat file has now been improved to follow v3 container/schema idea.
tested in production at sandia.

See merge request !641
  • Loading branch information
@narategithub
narategithub committed Aug 31, 2017
2 parents a53fb22 + 3bb2045 commit d606eb6
Show file tree
Hide file tree
Showing 3 changed files with 188 additions and 22 deletions.
1 change: 1 addition & 0 deletions ldms/man/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Plugin_procinterrupts.man \
Plugin_procnetdev.man \
Plugin_procnfs.man \
Plugin_store_csv.man \
Plugin_store_flatfile.man \
Plugin_store_sos.man \
Plugin_vmstat.man

Expand Down
90 changes: 90 additions & 0 deletions ldms/man/Plugin_store_flatfile.man
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
.\" Manpage for Plugin_store_flatfile
.\" Contact [email protected] to correct errors or typos.
.TH man 7 "29 Aug 2017" "v3.4" "LDMS Plugin store_flatfile man page"

.SH NAME
Plugin_store_flatfile - man page for the LDMS store_flatfile plugin

.SH SYNOPSIS
Within ldmsd_controller script or a configuration file:
.br
load name=store_flatfile
.br
config name=store_flatfile path=datadir
.br
strgp_add plugin=store_flatfile [ <attr> = <value> ]
.br


.SH DESCRIPTION
The flatfile store generates one file per metric with time, producer, component id, and value columns separated by spaces. The file name is $datadir/$container/$schema/$metric_name.

.PP

.SH STRGP_ADD ATTRIBUTE SYNTAX
The strgp_add sets the policies being added. This line determines the output files via
identification of the container and schema.
.TP
.BR strgp_add
plugin=store_flatfile name=<policy_name> schema=<schema> container=<container>
.br
ldmsd_controller strgp_add line
.br
.RS
.TP
plugin=<plugin_name>
.br
This MUST be store_flatfile.
.TP
name=<policy_name>
.br
The policy name for this strgp.
.TP
container=<container>
.br
The container and the schema determine where the output files will be written (see path above). They also are used to match any action=custom configuration.node/meminfo.
.TP
schema=<schema>
.br
The container and schema determines where the output files will be written (see path above).
.RE


.SH NOTES
.PP
.IP \[bu]
We expect to develop additional options controlling output files and
output file format.
.IP \[bu]
Please note the argument changes from v2.
.PP

.SH BUGS
Numeric array metrics are not presently supported.
There is a maximum of 20 concurrent flatfile stores.
The store is brittle to schema with multiple definitions, though it ought not to be.
Unchanged meta-metrics are always written anyway.
There is no option to quote string values, handle rollover, or handle buffering.

.SH EXAMPLES
.PP
Within ldmsd_controller or in a configuration file
.nf
load name=store_flatfile
config name=store_flatfile path=/XXX/datadir

# log only Active from the meminfo sampler
strgp_add name=store_flatfile_meminfo plugin=store_flatfile schema=meminfo container=flat
strgp_prdcr_add name=store_flatfile_meminfo regex=localhost1
strgp_metric_add name=store_flatfile_meminfo metric=Active
strgp_start name=store_flatfile_meminfo regex=localhost1

# log all from vmstat
strgp_add name=store_flatfile_vmstat plugin=store_flatfile schema=vmstat container=flat
strgp_prdcr_add name=store_flatfile_vmstat regex=localhost1
strgp_start name=store_flatfile_vmstat regex=localhost1

.fi

.SH SEE ALSO
ldmsd(8), ldms_quickstart(7), ldmsd_controller(8)
119 changes: 97 additions & 22 deletions ldms/src/store/store_flatfile.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,11 @@
#include <coll/idx.h>
#include "ldms.h"
#include "ldmsd.h"
#include <ovis_util/util.h>

/*
* NOTE:
* (flatfile::path) = (root_path)/(comp_type)/(metric)
* (flatfile::path) = (root_path)/(container)/(schema)/(metric)
*/

static idx_t store_idx;
Expand All @@ -90,13 +91,13 @@ struct flatfile_metric_store {

struct flatfile_store_instance {
struct ldmsd_store *store;
char *path; /**< (root_path)/(comp_type) */
char *container;
char *path; /**< (root_path)/(container)/schema */
char *schema;
void *ucontext;
idx_t ms_idx;
LIST_HEAD(ms_list, flatfile_metric_store) ms_list;
int metric_count;
struct flatfile_metric_store *ms[0];
struct flatfile_metric_store *ms[/*flex array*/];
};

static pthread_mutex_t cfg_lock;
Expand Down Expand Up @@ -146,7 +147,7 @@ static void *get_ucontext(ldmsd_store_handle_t _sh)
}

static ldmsd_store_handle_t
open_store(struct ldmsd_store *s, const char *comp_type, const char *container,
open_store(struct ldmsd_store *s, const char *container, const char *schema,
struct ldmsd_strgp_metric_list *metric_list, void *ucontext)
{
struct flatfile_store_instance *si;
Expand All @@ -158,7 +159,7 @@ open_store(struct ldmsd_store *s, const char *comp_type, const char *container,
* Add a component type directory if one does not
* already exist
*/
si = idx_find(store_idx, (void *)container, strlen(container));
si = idx_find(store_idx, (void *)schema, strlen(schema));
if (!si) {
/*
* First, count the metric.
Expand All @@ -168,8 +169,8 @@ open_store(struct ldmsd_store *s, const char *comp_type, const char *container,
TAILQ_FOREACH(x, metric_list, entry) {
metric_count++;
}
sprintf(tmp_path, "%s/%s", root_path, comp_type);
mkdir(tmp_path, 0777);
sprintf(tmp_path, "%s/%s/%s", root_path, container, schema);
f_mkdir_p(tmp_path, 0777);

/*
* Open a new store for this component-type and
Expand All @@ -189,8 +190,8 @@ open_store(struct ldmsd_store *s, const char *comp_type, const char *container,
si->path = strdup(tmp_path);
if (!si->path)
goto err2;
si->container = strdup(container);
if (!si->container)
si->schema = strdup(schema);
if (!si->schema)
goto err3;
i = 0;
char mname[128];
Expand All @@ -213,17 +214,25 @@ open_store(struct ldmsd_store *s, const char *comp_type, const char *container,
ms = calloc(1, sizeof(*ms));
sprintf(tmp_path, "%s/%s", si->path, name);
ms->path = strdup(tmp_path);
if (!ms->path)
if (!ms->path) {
msglog(LDMSD_LERROR, "Out of memory at %s:%d\n",
__FILE__, __LINE__);
goto err4;
}
ms->file = fopen_perm(ms->path, "a+", LDMSD_DEFAULT_FILE_PERM);
if (!ms->file)
if (!ms->file) {
int eno = errno;
msglog(LDMSD_LERROR, "Error opening %s: %d: %s at %s:%d\n",
ms->path, eno, strerror(eno),
__FILE__, __LINE__);
goto err4;
}
pthread_mutex_init(&ms->lock, NULL);
idx_add(si->ms_idx, name, strlen(name), ms);
LIST_INSERT_HEAD(&si->ms_list, ms, entry);
si->ms[i++] = ms;
}
idx_add(store_idx, (void *)container, strlen(container), si);
idx_add(store_idx, (void *)schema, strlen(schema), si);
}
goto out;
err4:
Expand All @@ -236,13 +245,14 @@ open_store(struct ldmsd_store *s, const char *comp_type, const char *container,
free(ms);
}

free(si->container);
free(si->schema);
err3:
free(si->path);
err2:
idx_destroy(si->ms_idx);
err1:
free(si);
si = NULL;
out:
pthread_mutex_unlock(&cfg_lock);
return si;
Expand All @@ -253,7 +263,7 @@ store(ldmsd_store_handle_t _sh, ldms_set_t set, int *metric_arry, size_t metric_
{
struct flatfile_store_instance *si;
int i;
int rc = 0;
int rc = 0, rc2 = 0;
int last_rc = 0;
int last_errno = 0;

Expand All @@ -265,16 +275,81 @@ store(ldmsd_store_handle_t _sh, ldms_set_t set, int *metric_arry, size_t metric_
const struct ldms_timestamp *ts = &_ts;
uint64_t comp_id;

const char *prod;
prod = ldms_set_producer_name_get(set);
for (i=0; i<metric_count; i++) {
pthread_mutex_lock(&si->ms[i]->lock);
comp_id = ldms_metric_user_data_get(set, metric_arry[i]);
rc = fprintf(si->ms[i]->file, "%"PRIu32".%"PRIu32" %"PRIu64
" %"PRIu64"\n", ts->sec,
ts->usec, comp_id,
/* time, host, compid, value */
#define STAMP \
rc2 = fprintf(si->ms[i]->file, "%"PRIu32".%06"PRIu32" %s %"PRIu64, \
ts->sec, ts->usec, prod, comp_id)
enum ldms_value_type metric_type =
ldms_metric_type_get(set, metric_arry[i]);
switch (metric_type) {
case LDMS_V_CHAR_ARRAY:
STAMP;
rc = fprintf(si->ms[i]->file, " %s\n",
ldms_metric_array_get_str(set, metric_arry[i]));
break;
case LDMS_V_U8:
STAMP;
rc = fprintf(si->ms[i]->file, " %u\n",
(unsigned)ldms_metric_get_u8(set, metric_arry[i]));
break;
case LDMS_V_S8:
STAMP;
rc = fprintf(si->ms[i]->file, " %d\n",
(int)ldms_metric_get_s8(set, metric_arry[i]));
break;
case LDMS_V_U16:
STAMP;
rc = fprintf(si->ms[i]->file, " %u\n",
(unsigned)ldms_metric_get_u16(set, metric_arry[i]));
break;
case LDMS_V_S16:
STAMP;
rc = fprintf(si->ms[i]->file, " %d\n",
(int)ldms_metric_get_s16(set, metric_arry[i]));
break;
case LDMS_V_U32:
STAMP;
rc = fprintf(si->ms[i]->file, " %u\n",
(unsigned)ldms_metric_get_u32(set, metric_arry[i]));
break;
case LDMS_V_S32:
STAMP;
rc = fprintf(si->ms[i]->file, " %d\n",
ldms_metric_get_s32(set, metric_arry[i]));
break;
case LDMS_V_U64:
STAMP;
rc = fprintf(si->ms[i]->file, " %"PRIu64"\n",
ldms_metric_get_u64(set, metric_arry[i]));
if (rc < 0) {
break;
case LDMS_V_S64:
STAMP;
rc = fprintf(si->ms[i]->file, " %"PRId64"\n",
ldms_metric_get_s64(set, metric_arry[i]));
break;
case LDMS_V_F32:
STAMP;
rc = fprintf(si->ms[i]->file, " %.9g\n",
ldms_metric_get_float(set, metric_arry[i]));
break;
case LDMS_V_D64:
STAMP;
rc = fprintf(si->ms[i]->file, " %.17g\n",
ldms_metric_get_double(set, metric_arry[i]));
break;
default:
/* array types not supported yet. want row and split files options */
break;
}
#undef STAMP
if (rc < 0 || rc2 < 0) {
last_errno = errno;
last_rc = rc;
last_rc = (rc != 0 ? rc : rc2);
msglog(LDMSD_LERROR, "Error %d: %s at %s:%d\n", last_errno,
strerror(last_errno), __FILE__,
__LINE__);
Expand Down Expand Up @@ -329,9 +404,9 @@ static void close_store(ldmsd_store_handle_t _sh)
free(ms->path);
free(ms);
}
idx_delete(store_idx, (void *)(si->container), strlen(si->container));
idx_delete(store_idx, (void *)(si->schema), strlen(si->schema));
free(si->path);
free(si->container);
free(si->schema);
idx_destroy(si->ms_idx);
free(si);
pthread_mutex_unlock(&cfg_lock);
Expand Down

0 comments on commit d606eb6

Please sign in to comment.