From 22225bd99ec89a86061aa19083de1350138d4e9c Mon Sep 17 00:00:00 2001 From: Sara Walton Date: Tue, 16 Jul 2024 14:46:41 -0600 Subject: [PATCH] removing white spaces --- rtd/README.rst | 6 +- rtd/docs/source/asf/asf-quickstart.rst | 28 +-- rtd/docs/source/asf/grafanapanel.rst | 10 +- rtd/docs/source/asf/index.rst | 44 ++-- rtd/docs/source/asf/pyanalysis.rst | 90 ++++---- rtd/docs/source/conf.py | 2 +- rtd/docs/source/contributing/index.rst | 2 +- rtd/docs/source/images/appsysfusion.png | Bin 89171 -> 89167 bytes rtd/docs/source/images/darshanConnector.png | Bin 493933 -> 493929 bytes .../source/images/grafana/grafana_query.png | Bin 30177 -> 30176 bytes .../images/grafana/grafana_timerange.png | Bin 49678 -> 49679 bytes .../images/grafana/grafanapanel_variables.png | Bin 38177 -> 38176 bytes .../images/ldmscon/ldmscon2020tutorial.png | Bin 53283 -> 53284 bytes .../source/images/ldmscon/ldmscon2021pres.PNG | Bin 28092 -> 28091 bytes .../images/ldmscon/ldmscon2021tutorial.PNG | Bin 35569 -> 35570 bytes .../source/images/ldmscon/ldmscon2022pres.PNG | Bin 59391 -> 59390 bytes .../images/ldmscon/ldmscon2022tutorial.PNG | Bin 51297 -> 51296 bytes rtd/docs/source/images/ovis-hpc_homepage.png | Bin 53624 -> 53622 bytes rtd/docs/source/index.rst | 4 +- rtd/docs/source/ldms-index.rst | 4 +- rtd/docs/source/ldms-quickstart.rst | 208 +++++++++--------- rtd/docs/source/ldms-streams.rst | 198 ++++++++--------- .../ldms_man/Plugin_cray_dvs_sampler.rst | 6 +- rtd/docs/source/ldms_man/Plugin_jobid.rst | 12 +- rtd/docs/source/ldms_man/Plugin_papi.rst | 18 +- rtd/docs/source/ldms_man/Plugin_rapl.rst | 4 +- rtd/docs/source/ldms_man/Plugin_store_app.rst | 16 +- rtd/docs/source/ldms_man/Plugin_store_csv.rst | 80 +++---- .../source/ldms_man/Plugin_store_kafka.rst | 10 +- .../source/ldms_man/Plugin_store_rabbitkw.rst | 46 ++-- .../source/ldms_man/Plugin_store_rabbitv3.rst | 46 ++-- rtd/docs/source/ldms_man/Plugin_store_sos.rst | 50 ++--- rtd/docs/source/ldms_man/index.rst | 2 +- .../source/ldms_man/ldms-csv-anonymize.rst | 24 +- .../source/ldms_man/ldms-csv-export-sos.rst | 40 ++-- rtd/docs/source/ldms_man/ldms-plugins.rst | 4 +- .../source/ldms_man/ldms-run-static-tests.rst | 22 +- rtd/docs/source/ldms_man/ldms-static-test.rst | 92 ++++---- .../source/ldms_man/ldms_build_install.rst | 50 ++--- rtd/docs/source/ldms_man/ldms_ls.rst | 10 +- .../source/ldms_man/ldms_sampler_base.rst | 22 +- rtd/docs/source/ldms_man/ldmsctl.rst | 154 ++++++------- rtd/docs/source/ldms_man/ldmsd.rst | 14 +- rtd/docs/source/ldms_man/ldmsd_controller.rst | 176 +++++++-------- .../source/ldms_man/ldmsd_stream_publish.rst | 20 +- .../ldms_man/ldmsd_stream_subscribe.rst | 26 +-- rtd/docs/source/ldmscon.rst | 10 +- .../source/sampler_man/Plugin_app_sampler.rst | 36 +-- .../sampler_man/Plugin_aries_linkstatus.rst | 8 +- .../source/sampler_man/Plugin_aries_mmr.rst | 10 +- .../Plugin_aries_mmr_configurable.rst | 18 +- .../sampler_man/Plugin_blob_stream_writer.rst | 12 +- rtd/docs/source/sampler_man/Plugin_clock.rst | 4 +- .../sampler_man/Plugin_daos_sampler.rst | 8 +- .../sampler_man/Plugin_dcgm_sampler.rst | 10 +- rtd/docs/source/sampler_man/Plugin_dstat.rst | 26 +-- rtd/docs/source/sampler_man/Plugin_edac.rst | 8 +- .../source/sampler_man/Plugin_filesingle.rst | 4 +- .../source/sampler_man/Plugin_fptrans.rst | 4 +- .../sampler_man/Plugin_geopm_sampler.rst | 6 +- .../sampler_man/Plugin_hello_sampler.rst | 4 +- .../Plugin_ibmad_records_sampler.rst | 14 +- .../sampler_man/Plugin_ibmad_sampler.rst | 12 +- rtd/docs/source/sampler_man/Plugin_ibnet.rst | 34 +-- .../source/sampler_man/Plugin_ipmireader.rst | 14 +- .../source/sampler_man/Plugin_ipmisensors.rst | 10 +- .../source/sampler_man/Plugin_kgnilnd.rst | 4 +- .../sampler_man/Plugin_linux_proc_sampler.rst | 32 +-- .../source/sampler_man/Plugin_lnet_stats.rst | 6 +- .../source/sampler_man/Plugin_loadavg.rst | 8 +- .../sampler_man/Plugin_lustre_client.rst | 10 +- .../source/sampler_man/Plugin_lustre_mdc.rst | 12 +- .../source/sampler_man/Plugin_lustre_mdt.rst | 6 +- .../source/sampler_man/Plugin_lustre_ost.rst | 6 +- .../source/sampler_man/Plugin_meminfo.rst | 4 +- .../sampler_man/Plugin_msr_interlagos.rst | 26 +-- rtd/docs/source/sampler_man/Plugin_opa2.rst | 12 +- .../source/sampler_man/Plugin_perfevent.rst | 20 +- .../sampler_man/Plugin_procdiskstats.rst | 6 +- .../sampler_man/Plugin_procinterrupts.rst | 4 +- .../source/sampler_man/Plugin_procnet.rst | 6 +- .../source/sampler_man/Plugin_procnetdev.rst | 6 +- .../source/sampler_man/Plugin_procnetdev2.rst | 6 +- .../source/sampler_man/Plugin_procnfs.rst | 4 +- .../source/sampler_man/Plugin_procstat.rst | 4 +- .../source/sampler_man/Plugin_procstat2.rst | 6 +- .../source/sampler_man/Plugin_rdc_sampler.rst | 20 +- .../sampler_man/Plugin_sampler_atasmart.rst | 6 +- .../sampler_man/Plugin_slingshot_info.rst | 2 +- .../sampler_man/Plugin_slingshot_metrics.rst | 8 +- .../source/sampler_man/Plugin_sysclassib.rst | 8 +- .../sampler_man/Plugin_tutorial_sampler.rst | 14 +- rtd/docs/source/sampler_man/Plugin_tx2mon.rst | 8 +- .../source/sampler_man/Plugin_variable.rst | 6 +- .../sampler_man/Plugin_variorum_sampler.rst | 4 +- rtd/docs/source/sampler_man/Plugin_vmstat.rst | 4 +- rtd/docs/source/sampler_man/index.rst | 2 +- .../sampler_man/ldms-ibnet-sampler-gen.rst | 2 +- .../sampler_man/ldms-sensors-config.rst | 10 +- .../sampler_man/ldms_ibnet_schema_name.rst | 10 +- .../store_man/Plugin_avro_kafka_store.rst | 44 ++-- .../store_man/Plugin_darshan_stream_store.rst | 8 +- .../store_man/Plugin_store_flatfile.rst | 8 +- .../source/store_man/Plugin_store_sos.rst | 50 ++--- .../store_man/Plugin_store_timescale.rst | 22 +- .../store_man/Plugin_store_tutorial.rst | 12 +- .../store_man/Plugin_stream_csv_store.rst | 30 +-- rtd/docs/source/store_man/index.rst | 2 +- rtd/docs/source/ug.rst | 2 +- rtd/man2rst/Plugin_app_sampler.rst | 36 +-- rtd/man2rst/Plugin_aries_linkstatus.rst | 8 +- rtd/man2rst/Plugin_aries_mmr.rst | 10 +- rtd/man2rst/Plugin_aries_mmr_configurable.rst | 18 +- rtd/man2rst/Plugin_avro_kafka_store.rst | 44 ++-- rtd/man2rst/Plugin_blob_stream_writer.rst | 12 +- rtd/man2rst/Plugin_clock.rst | 4 +- rtd/man2rst/Plugin_cray_dvs_sampler.rst | 6 +- rtd/man2rst/Plugin_daos_sampler.rst | 8 +- rtd/man2rst/Plugin_darshan_stream_store.rst | 8 +- rtd/man2rst/Plugin_dcgm_sampler.rst | 10 +- rtd/man2rst/Plugin_dstat.rst | 26 +-- rtd/man2rst/Plugin_edac.rst | 8 +- rtd/man2rst/Plugin_filesingle.rst | 4 +- rtd/man2rst/Plugin_fptrans.rst | 4 +- rtd/man2rst/Plugin_geopm_sampler.rst | 6 +- rtd/man2rst/Plugin_hello_sampler.rst | 4 +- rtd/man2rst/Plugin_ibmad_records_sampler.rst | 14 +- rtd/man2rst/Plugin_ibmad_sampler.rst | 12 +- rtd/man2rst/Plugin_ibnet.rst | 34 +-- rtd/man2rst/Plugin_ipmireader.rst | 14 +- rtd/man2rst/Plugin_ipmisensors.rst | 10 +- rtd/man2rst/Plugin_jobid.rst | 12 +- rtd/man2rst/Plugin_kgnilnd.rst | 4 +- rtd/man2rst/Plugin_linux_proc_sampler.rst | 32 +-- rtd/man2rst/Plugin_lnet_stats.rst | 6 +- rtd/man2rst/Plugin_loadavg.rst | 8 +- rtd/man2rst/Plugin_lustre_client.rst | 10 +- rtd/man2rst/Plugin_lustre_mdc.rst | 12 +- rtd/man2rst/Plugin_lustre_mdt.rst | 6 +- rtd/man2rst/Plugin_lustre_ost.rst | 6 +- rtd/man2rst/Plugin_meminfo.rst | 4 +- rtd/man2rst/Plugin_msr_interlagos.rst | 26 +-- rtd/man2rst/Plugin_opa2.rst | 12 +- rtd/man2rst/Plugin_papi.rst | 18 +- rtd/man2rst/Plugin_perfevent.rst | 20 +- rtd/man2rst/Plugin_procdiskstats.rst | 6 +- rtd/man2rst/Plugin_procinterrupts.rst | 4 +- rtd/man2rst/Plugin_procnet.rst | 6 +- rtd/man2rst/Plugin_procnetdev.rst | 6 +- rtd/man2rst/Plugin_procnetdev2.rst | 6 +- rtd/man2rst/Plugin_procnfs.rst | 4 +- rtd/man2rst/Plugin_procstat.rst | 4 +- rtd/man2rst/Plugin_procstat2.rst | 6 +- rtd/man2rst/Plugin_rapl.rst | 4 +- rtd/man2rst/Plugin_rdc_sampler.rst | 20 +- rtd/man2rst/Plugin_sampler_atasmart.rst | 6 +- rtd/man2rst/Plugin_slingshot_info.rst | 2 +- rtd/man2rst/Plugin_slingshot_metrics.rst | 8 +- rtd/man2rst/Plugin_store_app.rst | 16 +- rtd/man2rst/Plugin_store_csv.rst | 80 +++---- rtd/man2rst/Plugin_store_flatfile.rst | 8 +- rtd/man2rst/Plugin_store_kafka.rst | 10 +- rtd/man2rst/Plugin_store_rabbitkw.rst | 46 ++-- rtd/man2rst/Plugin_store_rabbitv3.rst | 46 ++-- rtd/man2rst/Plugin_store_sos.rst | 50 ++--- rtd/man2rst/Plugin_store_timescale.rst | 22 +- rtd/man2rst/Plugin_store_tutorial.rst | 12 +- rtd/man2rst/Plugin_stream_csv_store.rst | 30 +-- rtd/man2rst/Plugin_sysclassib.rst | 8 +- rtd/man2rst/Plugin_tutorial_sampler.rst | 14 +- rtd/man2rst/Plugin_tx2mon.rst | 8 +- rtd/man2rst/Plugin_variable.rst | 6 +- rtd/man2rst/Plugin_variorum_sampler.rst | 4 +- rtd/man2rst/Plugin_vmstat.rst | 4 +- rtd/man2rst/ldms-csv-anonymize.rst | 24 +- rtd/man2rst/ldms-csv-export-sos.rst | 40 ++-- rtd/man2rst/ldms-ibnet-sampler-gen.rst | 2 +- rtd/man2rst/ldms-plugins.rst | 4 +- rtd/man2rst/ldms-run-static-tests.rst | 22 +- rtd/man2rst/ldms-sensors-config.rst | 10 +- rtd/man2rst/ldms-static-test.rst | 92 ++++---- rtd/man2rst/ldms_build_install.rst | 50 ++--- rtd/man2rst/ldms_ibnet_schema_name.rst | 10 +- rtd/man2rst/ldms_ls.rst | 10 +- rtd/man2rst/ldms_sampler_base.rst | 22 +- rtd/man2rst/ldmsctl.rst | 154 ++++++------- rtd/man2rst/ldmsd.rst | 14 +- rtd/man2rst/ldmsd_controller.rst | 176 +++++++-------- rtd/man2rst/ldmsd_stream_publish.rst | 20 +- rtd/man2rst/ldmsd_stream_subscribe.rst | 26 +-- rtd/pandoc_man_2_rst.py | 6 +- rtd/update-branches.py | 39 ---- 192 files changed, 1933 insertions(+), 1972 deletions(-) delete mode 100644 rtd/update-branches.py diff --git a/rtd/README.rst b/rtd/README.rst index ef558b878..80b6a640f 100644 --- a/rtd/README.rst +++ b/rtd/README.rst @@ -33,14 +33,14 @@ Instructions and documentation on how to use ReadTheDocs can be found here: > git add > git commit -m "add message" > git push - -Adding A New File + +Adding A New File ****************** For any new RST files created, please include them in rtd/docs/src/index.rst under their corresponding sections. All RST files not included in index.rst will not populate on the offical webpage (e.g. readthedocs). Paper Lock ************ This is for claiming any sections you are working on so there is no overlap. -Please USE paper.lock to indicate if you are editing an existing RST file. +Please USE paper.lock to indicate if you are editing an existing RST file. diff --git a/rtd/docs/source/asf/asf-quickstart.rst b/rtd/docs/source/asf/asf-quickstart.rst index 4ce601b01..4c96fd6e9 100644 --- a/rtd/docs/source/asf/asf-quickstart.rst +++ b/rtd/docs/source/asf/asf-quickstart.rst @@ -5,7 +5,7 @@ Create A Simple Analysis ------------------------ To start, please create a folder called ``graf_analysis`` in your home directory and copy the following contents to a python file called ``dsosTemplate.py``: -* This is a python analysis that queries the DSOS database and returns a DataFrame of the ``meminfo`` schema metrics along with the ``timestamp``, ``component_id`` and ``job_id``. +* This is a python analysis that queries the DSOS database and returns a DataFrame of the ``meminfo`` schema metrics along with the ``timestamp``, ``component_id`` and ``job_id``. dsosTemplate.py: @@ -20,7 +20,7 @@ dsosTemplate.py: class dsosTemplate(Analysis): def __init__(self, cont, start, end, schema='meminfo', maxDataPoints=4096): super().__init__(cont, start, end, schema, 1000000) - + def get_data(self, metrics, filters=[],params=None): try: self.sel = f'select {",".join(metrics)} from {self.schema}' @@ -36,37 +36,37 @@ dsosTemplate.py: a, b, c = sys.exc_info() print(str(e)+' '+str(c.tb_lineno)) -.. note:: - - If you want to use this analysis module in a Grafana dashboard, you will need to ask your administrator to copy your new analysis module(s) into the directory that Grafana points to. This is because Grafana is setup to look at a specific path directory to query from. +.. note:: + + If you want to use this analysis module in a Grafana dashboard, you will need to ask your administrator to copy your new analysis module(s) into the directory that Grafana points to. This is because Grafana is setup to look at a specific path directory to query from. Test Analysis via Terminal Window ---------------------------------- -You can easily test your module without the Grafana interface by creating a python script that mimics the Grafana query and formats the returned JSON into a timeseries dataframe or table. +You can easily test your module without the Grafana interface by creating a python script that mimics the Grafana query and formats the returned JSON into a timeseries dataframe or table. -First, create the following file in the same directory as your python analysis (i.e. ``/user/home/graf_analysis/``) and label it ``testDSOSanalysis.py``. +First, create the following file in the same directory as your python analysis (i.e. ``/user/home/graf_analysis/``) and label it ``testDSOSanalysis.py``. * This python script imitates the Grafana query that calls your analysis module and will return a timeseries DataFrame of the ``Active`` and ``Inactive`` meminfo metrics. .. code-block :: python #!/usr/bin/python3 - + import time,sys from sosdb import Sos from grafanaFormatter import DataFormatter from table_formatter import table_formatter from time_series_formatter import time_series_formatter from dsosTemplate import dsosTemplate - + sess = Sos.Session("//config/dsos.conf") cont = '' cont = sess.open(cont) - + model = dsosTemplate(cont, time.time()-300, time.time(), schema='meminfo', maxDataPoints=4096) - + x = model.get_data(['Active','Inactive'], filters=['job_id'], params='') - + #fmt = table_formatter(x) fmt = time_series_formatter(x) x = fmt.ret_json() @@ -84,7 +84,7 @@ First, create the following file in the same directory as your python analysis ( .. note:: - All imports are python scripts that need to reside in the same directory as the test analysis module in order for it to run successfully. + All imports are python scripts that need to reside in the same directory as the test analysis module in order for it to run successfully. Then, run the python script with the current python verion installed. In this case it would be ``python3 `` @@ -103,7 +103,7 @@ Create A New Dashboard To create a new dashboard, click on the + sign on the left side of the home page and hit dashboard. This will create a blank dashboard with an empty panel in it. Hit the add query button on the panel to begin configuring the query to be sent to an analysis module.  .. note:: - + For more information on how to navigate around the Grafana dashboard and what the variables and advanced settings do, please see `Grafana Panel `_ and `Grafana Usage `_. * Next, add your analysis by filling out the required fields shown below: diff --git a/rtd/docs/source/asf/grafanapanel.rst b/rtd/docs/source/asf/grafanapanel.rst index 1bf335e3d..fb408debe 100644 --- a/rtd/docs/source/asf/grafanapanel.rst +++ b/rtd/docs/source/asf/grafanapanel.rst @@ -10,7 +10,7 @@ Configuring the Query and Visualization Once you right click on the panel title and select edit, the panel settings will appear. The first tab is for configuring the query. There are 8 fields in the query field defined below: -* Query Type - type of query to perform. The most commonly used in "analysis" which calls an analysis module. "metrics" is used to return raw data without any analysis module. +* Query Type - type of query to perform. The most commonly used in "analysis" which calls an analysis module. "metrics" is used to return raw data without any analysis module. * Query Format - the type of visualization to be used on the dataset. It is used by Grafana Formatter to properly JSON-ify the data returned from the analysis module. Can be either time_series, table, or heatmap. * Analysis - required if you choose analysis query type. Specifies the python module to call to transofrm the data. * Container - the name of the container to be used. This can be either the full path to the container or the Django backend get_container function can be changed to customize for site settings. @@ -19,7 +19,7 @@ Once you right click on the panel title and select edit, the panel settings will * Extra Params - (Optional) pass in an arbitrary string into the analysis module * Filters - (Optional) include a no-sql like syntax for filtering your query, can be a comma separated list of filters i.e. component_id == 5,job_id > 0 -The second tab in the panel settings is for visualization. Graph, Table, and Heatmap are the available visualizations for a query output. +The second tab in the panel settings is for visualization. Graph, Table, and Heatmap are the available visualizations for a query output. Text, which uses Markdown language, could also be used for Dashboard descriptions or details. If you use a graph visualization, the query Format should be time_series. If you use a table visualization, the query Format should be table. @@ -31,7 +31,7 @@ Dashboard Variables and Advanced Settings ------------------------------------------- .. image:: ../images/grafana/grafanapanel_variables.png -Often we want users to be able to change inputs into the queries, however users cannot change edit queries. What they can edit in Grafana are variables, which are listed at the top of the dashboard. These variables can be referenced with a ``$`` in front of the variable name. For example, we can let the user switch SOS containers they are interested by creating a variable called container and then putting ``$container`` in the container field of the query. To create variables, go to the dashboard settings (gear button at the top right) and go to variables. Here you can create new variables. Common variable types are text boxes, for users to fill in, or queries. We can actually create a pre-populated list of options for certain fields by querying the container. Below are the queryable metrics what information to put in the query field. +Often we want users to be able to change inputs into the queries, however users cannot change edit queries. What they can edit in Grafana are variables, which are listed at the top of the dashboard. These variables can be referenced with a ``$`` in front of the variable name. For example, we can let the user switch SOS containers they are interested by creating a variable called container and then putting ``$container`` in the container field of the query. To create variables, go to the dashboard settings (gear button at the top right) and go to variables. Here you can create new variables. Common variable types are text boxes, for users to fill in, or queries. We can actually create a pre-populated list of options for certain fields by querying the container. Below are the queryable metrics what information to put in the query field. * Container - select the custom option in the **Type** field and add the name of the container being used to query from in the **custom options** field. * Schema - ``query=schema&container=`` @@ -40,8 +40,8 @@ Often we want users to be able to change inputs into the queries, however users * Component IDs - ``query=components&container=&schema=`` * Jobs - ``query=jobs&container=&schema=`` -You can put variables in queries as well. For example, if you already have a $container variable, you can set the schema variable query to be ``query=schema&container=$container``. Then the ``$schema`` variable can be used in other queries. +You can put variables in queries as well. For example, if you already have a $container variable, you can set the schema variable query to be ``query=schema&container=$container``. Then the ``$schema`` variable can be used in other queries. -In the dashboard settings you can also change the dashboard name and folder location and load previously saved versions. +In the dashboard settings you can also change the dashboard name and folder location and load previously saved versions. Other than the container variable, all other variables bulleted above are set to query in the **Type** field diff --git a/rtd/docs/source/asf/index.rst b/rtd/docs/source/asf/index.rst index e44f2aaff..0b067db4d 100644 --- a/rtd/docs/source/asf/index.rst +++ b/rtd/docs/source/asf/index.rst @@ -1,22 +1,22 @@ -.. image:: ../images/appsysfusion.png - :width: 300 - :height: 125 - :align: center - -ASF -==== -AppSysFusion provides analysis and visualization capabilities aimed at serving insights from HPC monitoring data gathered with LDMS, though could be generalized outside of that scope. -It combines a Grafana front-end with a Django back-end to perform in-query analyses on raw data and return transformed information back to the end user. -By performing in-query analyses, only data of interest to the end-user is operated on rather than the entirety of the dataset for all analyses for all time. -This saves significant computation and storage resources with the penalty of slightly higher query times. -These analyses are modular python scripts that can be easily added or changed to suit evolving needs. -The current implementation is aimed at querying DSOS databases containing LDMS data, though efforts are in progress to abstract this functionality out to other databases and datatypes. - -.. toctree:: - :maxdepth: 2 - - asf-quickstart - asf-tutorial - grafanapanel - grafanause - pyanalysis +.. image:: ../images/appsysfusion.png + :width: 300 + :height: 125 + :align: center + +ASF +==== +AppSysFusion provides analysis and visualization capabilities aimed at serving insights from HPC monitoring data gathered with LDMS, though could be generalized outside of that scope. +It combines a Grafana front-end with a Django back-end to perform in-query analyses on raw data and return transformed information back to the end user. +By performing in-query analyses, only data of interest to the end-user is operated on rather than the entirety of the dataset for all analyses for all time. +This saves significant computation and storage resources with the penalty of slightly higher query times. +These analyses are modular python scripts that can be easily added or changed to suit evolving needs. +The current implementation is aimed at querying DSOS databases containing LDMS data, though efforts are in progress to abstract this functionality out to other databases and datatypes. + +.. toctree:: + :maxdepth: 2 + + asf-quickstart + asf-tutorial + grafanapanel + grafanause + pyanalysis diff --git a/rtd/docs/source/asf/pyanalysis.rst b/rtd/docs/source/asf/pyanalysis.rst index 378599889..d14ea5aa8 100644 --- a/rtd/docs/source/asf/pyanalysis.rst +++ b/rtd/docs/source/asf/pyanalysis.rst @@ -3,26 +3,26 @@ Python Analysis Creation Analysis I/O ----------- -An analysis module is a python script and has a general template. There is a class, which must be called the same name as the python script itself, and two class functions: ``__init__`` and ``get_data``. The module is first initialized and then ``get_data`` is called. This should return a pandas DataFrame or a NumSOS DataSet (preferably the former if you are using python3). Below are the variables passed from the Grafana interface to these class functions. +An analysis module is a python script and has a general template. There is a class, which must be called the same name as the python script itself, and two class functions: ``__init__`` and ``get_data``. The module is first initialized and then ``get_data`` is called. This should return a pandas DataFrame or a NumSOS DataSet (preferably the former if you are using python3). Below are the variables passed from the Grafana interface to these class functions. ``__init__`` * ``cont`` - A Sos.Container object which contains the path information to the SOS container specified in the Grafana query * ``start`` - The beginning of the time range of the Grafana query (in epoch time). * ``end`` - The end of the time range of the Grafana query (in epoch time). * ``schema`` - The LDMS schema specified by the Grafana query (e.g. meminfo). - * ``maxDataPoints`` - the maximum number of points that Grafana can display on the user's screen. + * ``maxDataPoints`` - the maximum number of points that Grafana can display on the user's screen. ``get_data`` * ``metrics`` - a python list of metrics specified by the Grafana query (e.g. ['Active','MemFree']). - * ``job_id`` - a string of the job_id specified by the Grafana query. + * ``job_id`` - a string of the job_id specified by the Grafana query. * ``user_name`` - a string of the user name specified by the Grafana query. * ``params`` - a string of the extra parameters specified by the Grafana query (e.g. 'threshold = 10'). * ``filters`` - a python list of filter strings for the DSOS query (e.g. ['job_id == 30','component_id < 604']). - +/t Example Analysis Module ------------------------------------- -Below is a basic analysis that simply queries the database and returns the DataFrame of the metrics passed in along with the timestamp, component_id, and job_id for each metric. +Below is a basic analysis that simply queries the database and returns the DataFrame of the metrics passed in along with the timestamp, component_id, and job_id for each metric. .. code-block :: python @@ -35,7 +35,7 @@ Below is a basic analysis that simply queries the database and returns the DataF class dsosTemplate(Analysis): def __init__(self, cont, start, end, schema='job_id', maxDataPoints=4096): super().__init__(cont, start, end, schema, 1000000) - + def get_data(self, metrics, filters=[],params=None): try: sel = f'select {",".join(metrics)} from {self.schema}' @@ -51,23 +51,23 @@ Below is a basic analysis that simply queries the database and returns the DataF a, b, c = sys.exc_info() print(str(e)+' '+str(c.tb_lineno)) -In the ``__init__`` function, most things are set to be self variables to access them later in the ``get_data`` using the ``super()`` function. The ``super()`` function also sets up a variable called ``self.query`` which is a ``Sos.SqlQuery`` object. The 1000000 in the ``super()`` function sets the block size for this ``self.query`` object. An optimal block size is dependent on the query, however 1 million has been sufficiently performant to this point. +In the ``__init__`` function, most things are set to be self variables to access them later in the ``get_data`` using the ``super()`` function. The ``super()`` function also sets up a variable called ``self.query`` which is a ``Sos.SqlQuery`` object. The 1000000 in the ``super()`` function sets the block size for this ``self.query`` object. An optimal block size is dependent on the query, however 1 million has been sufficiently performant to this point. -In the ``get_data`` function we create a select clause for the DSOS query by joining the metrics and schema variables. The ``self.get_where`` is a graf_analysis class function which takes filter parameters and makes an SQL-like where clause string with ``self.start`` and ``self.end`` as timestamp boundaries. There is also the orderby variable which we are setting as ``time_job_comp`` here. This refers to what index we should use when querying the database. Our SOS databases are setup to use permutations of ``timestamp``, ``job ID``, and ``component ID`` as multi-indices. Depending on your filter, you may want to use a different multi-index. +In the ``get_data`` function we create a select clause for the DSOS query by joining the metrics and schema variables. The ``self.get_where`` is a graf_analysis class function which takes filter parameters and makes an SQL-like where clause string with ``self.start`` and ``self.end`` as timestamp boundaries. There is also the orderby variable which we are setting as ``time_job_comp`` here. This refers to what index we should use when querying the database. Our SOS databases are setup to use permutations of ``timestamp``, ``job ID``, and ``component ID`` as multi-indices. Depending on your filter, you may want to use a different multi-index. -The ``self.get_all_data`` takes the Sos.SqlQuery object, ``self.query``, and calls ``self.query.next``. This returns a block size number of records that match the query from database defined by the cont variable. If there are more than a block size number of records, it continues calling ``self.query.next`` and appending the results to a pandas DataFrame until all data is returned. +The ``self.get_all_data`` takes the Sos.SqlQuery object, ``self.query``, and calls ``self.query.next``. This returns a block size number of records that match the query from database defined by the cont variable. If there are more than a block size number of records, it continues calling ``self.query.next`` and appending the results to a pandas DataFrame until all data is returned. -Additional analysis can be added where the "Fun stuff here!" comment is. +Additional analysis can be added where the "Fun stuff here!" comment is. -With the example parameters specified in the last section, our select statement here would be ``select Active,MemFree from meminfo where timestamp > start and timestamp < end and job_id == 30 and component_id < 604 order_by time_job_comp``. +With the example parameters specified in the last section, our select statement here would be ``select Active,MemFree from meminfo where timestamp > start and timestamp < end and job_id == 30 and component_id < 604 order_by time_job_comp``. .. note:: - ``job_id`` and ``user_name`` must exist in the schema passed in for this command to work. + ``job_id`` and ``user_name`` must exist in the schema passed in for this command to work. Testing an Analysis Module -------------------------- -This section goes over how to test your python analysis module as a user. +This section goes over how to test your python analysis module as a user. You do not need to query from the Grafana interface to test your module. Below is a simple code which mimics the Grafana pipeline and prints the JSON returned to Grafana. @@ -86,22 +86,22 @@ Then you can imitate the Grafana query to call your analysis module using a pyth .. code-block :: python #!/usr/bin/python3 - + import time,sys from sosdb import Sos from grafanaFormatter import DataFormatter from table_formatter import table_formatter from time_series_formatter import time_series_formatter from dsosTemplate import dsosTemplate - + sess = Sos.Session("//config/dsos.conf") cont = '' cont = sess.open(cont) - + model = dsosTemplate(cont, time.time()-300, time.time(), schema='meminfo', maxDataPoints=4096) - + x = model.get_data(['Active']) - + #fmt = table_formatter(x) fmt = time_series_formatter(x) x = fmt.ret_json() @@ -111,54 +111,54 @@ Then you can imitate the Grafana query to call your analysis module using a pyth * If you would like to query all metrics then replace ``Active`` with ``*``. * To query a specific job_id: set ``job_id`` to you job_id with ``==``. -* To query from a specific time range: update the start time, ``time.time()-300`` and end time, ``time.time()`` to an epoch timestamp. +* To query from a specific time range: update the start time, ``time.time()-300`` and end time, ``time.time()`` to an epoch timestamp. * To add a string metric, filter or parameter, you must include a double quote, ``"``, before and after the string (i.e. ``filters=['user=="myusername"']``) .. note:: - The ``params`` can be any number or string that you want to define in your analysis module to better manage, output or analyze the data. For example, you can program your module to return specific analyses such as the average with ``params='analysis=average'`` by parsing the arguement, using ``if`` statements to determine what analysis to apply to the data and, to make things cleaner, a function to perform these calculations in. - + The ``params`` can be any number or string that you want to define in your analysis module to better manage, output or analyze the data. For example, you can program your module to return specific analyses such as the average with ``params='analysis=average'`` by parsing the arguement, using ``if`` statements to determine what analysis to apply to the data and, to make things cleaner, a function to perform these calculations in. +/t Required Scripts //////////////// The following scripts are needed to run the python analysis module. If these python scripts or modules **do not exist on your system and you have no way of accessing them** then please continue. Otherwise, you can skip this section -**If you do not have access to these existing scripts** then please create them in the same directory as your python analysis module. +**If you do not have access to these existing scripts** then please create them in the same directory as your python analysis module. .. note:: - + If Grafana and SOS are installed on your system then please ask your system administator where these files reside on the system so that you can copy them to your home directory. grafanaFormatter: .. code:: RST - + from sosdb import Sos from sosdb.DataSet import DataSet import numpy as np import pandas as pd import copy - + class RowIter(object): def __init__(self, dataSet): self.dset = dataSet self.limit = dataSet.get_series_size() self.row_no = 0 - + def __iter__(self): return self - + def cvt(self, value): if type(value) == np.datetime64: return [ value.astype(np.int64) / 1000 ] return value - + def __next__(self): if self.row_no >= self.limit: raise StopIteration res = [ self.cvt(self.dset[[col, self.row_no]]) for col in range(0, self.dset.series_count) ] self.row_no += 1 return res - + class DataFormatter(object): def __init__(self, data): self.result = [] @@ -169,16 +169,16 @@ grafanaFormatter: 'pandas.core.frame' : self.fmt_dataframe, 'builtins' : self.fmt_builtins } - + def ret_json(self): return self.fmt_data[self.fmt]() - + def fmt_dataset(self): pass - + def fmt_dataframe(self): pass - + def fmt_builtins(self): pass @@ -192,13 +192,13 @@ table_formatter: import numpy as np import pandas as pd import copy - + class table_formatter(DataFormatter): def fmt_dataset(self): # Format data from sosdb DataSet object if self.data is None: return {"columns" : [{ "text" : "No papi jobs in time range" }] } - + self.result = { "type" : "table" } self.result["columns"] = [ { "text" : colName } for colName in self.data.series ] rows = [] @@ -206,16 +206,16 @@ table_formatter: rows.append(row) self.result["rows"] = rows return self.result - + def fmt_dataframe(self): if self.data is None: return {"columns" : [{ "text" : "No papi jobs in time range" }] } - + self.result = { "type" : "table" } self.result["columns"] = [ { "text" : colName } for colName in self.data.columns ] self.result["rows"] = self.data.to_numpy() return self.result - + def fmt_builtins(self): if self.data is None: return { "columns" : [], "rows" : [], "type" : "table" } @@ -225,20 +225,20 @@ table_formatter: time_series_formatter: .. code:: RST - + from graf_analysis.grafanaFormatter import DataFormatter from sosdb.DataSet import DataSet from sosdb import Sos import numpy as np import pandas as pd import copy - + class time_series_formatter(DataFormatter): def fmt_dataset(self): # timestamp is always last series if self.data is None: return [ { "target" : "", "datapoints" : [] } ] - + for series in self.data.series: if series == 'timestamp': continue @@ -249,11 +249,11 @@ time_series_formatter: self.result.append(plt_dict) del ds return self.result - + def fmt_dataframe(self): if self.data is None: return [ { "target" : "", "datapoints" : [] } ] - + for series in self.data.columns: if series == 'timestamp': continue @@ -261,7 +261,7 @@ time_series_formatter: plt_dict['datapoints'] = self.fmt_datapoints([series, 'timestamp']) self.result.append(plt_dict) return self.result - + def fmt_datapoints(self, series): ''' Format dataframe to output expected by grafana ''' aSet = [] @@ -290,7 +290,7 @@ time_series_formatter: aRow.append(v) aSet.append(aRow) return aSet - + def fmt_builtins(self): if self.data is None: return [ { "target" : "", "datapoints" : [] } ] diff --git a/rtd/docs/source/conf.py b/rtd/docs/source/conf.py index 40df1768b..7de1dbac6 100644 --- a/rtd/docs/source/conf.py +++ b/rtd/docs/source/conf.py @@ -22,7 +22,7 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3/', None), 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), - + # Link to the "apis" of the "hpc-ovis" project and subprojects "ovis-hpc": ("https://ovis-hpc.readthedocs.io/en/latest/", None), "sos": ("https://ovis-hpc.readthedocs.io/projects/sos/en/latest/", None), diff --git a/rtd/docs/source/contributing/index.rst b/rtd/docs/source/contributing/index.rst index 2181c2155..77499458d 100644 --- a/rtd/docs/source/contributing/index.rst +++ b/rtd/docs/source/contributing/index.rst @@ -3,7 +3,7 @@ Contributing to LDMS .. toctree:: :maxdepth: 2 - + samplerwrite storewrite docreqs diff --git a/rtd/docs/source/images/appsysfusion.png b/rtd/docs/source/images/appsysfusion.png index 270ed05b6d0e1b3549a9ee9830ff777fe93f299e..28769ee143c0904538d131dfdff7f27de225828f 100644 GIT binary patch delta 31 pcmV+)0O0@AxCPI+1+b2|vyZq0IwgZVECLsN*fN?4tknS&M%#&nPX#YEbar@s1OrK4FVjqq& a&FyF8Z0`?c24WT@~ diff --git a/rtd/docs/source/images/grafana/grafana_timerange.png b/rtd/docs/source/images/grafana/grafana_timerange.png index cde8593b0886c29eb34464c39a0575d2e8c59822..73af4aff79e04bfd910415f9fe8630c60c4c715f 100644 GIT binary patch delta 17 YcmeBcVeW5X-eAzoq+hbxu=%bX06KOCwg3PC delta 16 XcmeBgVeV^T-eAzo$hq05`K}!RGa3dQ diff --git a/rtd/docs/source/images/grafana/grafanapanel_variables.png b/rtd/docs/source/images/grafana/grafanapanel_variables.png index f68b69720b334e6c3f98363aac782aa98df78127..cc978d890f24b2c2e2e88cac39ee5427a812c7a8 100644 GIT binary patch delta 14 WcmZ3uifO?rrVUx4o3lf=*#iJEWd?Bo delta 16 YcmZ3mifQ30rVUx4j0&5xLburi06I4YkpKVy diff --git a/rtd/docs/source/images/ldmscon/ldmscon2020tutorial.png b/rtd/docs/source/images/ldmscon/ldmscon2020tutorial.png index 72231a86f8434f6bfc2b314b7fd94194dbafa6ee..29807db12fccc2cdc80cce7007064c833e9d77e2 100644 GIT binary patch delta 17 ZcmZ3yfO*LR<_$?aO!_68lX(st1^_*v2KE2| delta 16 YcmZ3ofO+u(<_$?ajGUWOcn%x}05&HETmS$7 diff --git a/rtd/docs/source/images/ldmscon/ldmscon2021pres.PNG b/rtd/docs/source/images/ldmscon/ldmscon2021pres.PNG index d577ec28ac348299cc36417ba88b9304f338a2ee..6643f2a7df34563539fdf6282a36324ff3491a4d 100644 GIT binary patch delta 14 VcmdmUn{oGT#tpG4o8waAH~}|o27Lek delta 16 Xcmdmen{m%=#tpG4j0&4$Q{p%QL2d?} diff --git a/rtd/docs/source/images/ldmscon/ldmscon2021tutorial.PNG b/rtd/docs/source/images/ldmscon/ldmscon2021tutorial.PNG index b7ee78a2792694ce09dee13e5597d6acff246262..3e8631434c1c6bd078bc42cd64f495f55801eb17 100644 GIT binary patch delta 17 Zcmex3mFd$|rVShPnDk3FZ`5;$0RT!Z2Xg=b delta 16 Ycmew~mFeSDrVShP7&$j@(sPIb071Y8*#H0l diff --git a/rtd/docs/source/images/ldmscon/ldmscon2022pres.PNG b/rtd/docs/source/images/ldmscon/ldmscon2022pres.PNG index e59802f6fb8fedfc1325444c6ed83839b468a248..ace4b18917e458f36e0cde8725f15881b371e346 100644 GIT binary patch delta 35 qcmex=p84N-<_$Y6nDk3F@3IgO*nHaKw8G};nR6pRBBqBb}1qXhssAqR8- delta 16 YcmaDbf%)MC<_#%x85K6C%>APU074T7lmGw# diff --git a/rtd/docs/source/images/ovis-hpc_homepage.png b/rtd/docs/source/images/ovis-hpc_homepage.png index 6b44e4c1693ef7ab40765f770daab98295b936dc..a50e74bc60cd92c9f296dc91584a51dbb14778fe 100644 GIT binary patch delta 32 ocmeydi22(h<_$YknDk3F?^M~syE)H@Uukn)$W{N%Axl(r0rM0M ASF - + Other Projects ==================================== @@ -56,7 +56,7 @@ Other Projects `sos `_ `baler `_ - + diff --git a/rtd/docs/source/ldms-index.rst b/rtd/docs/source/ldms-index.rst index 8982e3157..0c5f28033 100644 --- a/rtd/docs/source/ldms-index.rst +++ b/rtd/docs/source/ldms-index.rst @@ -1,4 +1,4 @@ -LDMS +LDMS ====== .. image:: images/ovis-hpc_homepage.png @@ -41,6 +41,6 @@ To join the LDMS Users Group Mailing List: https://github.com/ovis-hpc/ovis-wiki :maxdepth: 2 :caption: Contributing to LDMS - contributing/index + contributing/index diff --git a/rtd/docs/source/ldms-quickstart.rst b/rtd/docs/source/ldms-quickstart.rst index b28a24a3b..1670aa793 100644 --- a/rtd/docs/source/ldms-quickstart.rst +++ b/rtd/docs/source/ldms-quickstart.rst @@ -4,7 +4,7 @@ LDMS Quick Start Installation ***************** -AlmaLinux8 +AlmaLinux8 ------------ Prerequisites @@ -60,9 +60,9 @@ The following steps were ran on AlmaLinux8 arm64v8 sudo dnf install -y python38-devel sudo dnf install -y python38-Cython sudo dnf install -y python38-libs - -RHEL 9 + +RHEL 9 ------------ Prerequisites @@ -74,28 +74,28 @@ Prerequisites * libtool * python3 (or higher) * python3-devel (or higher) -* cython +* cython * bison * flex Prerequisite Installation --------------------------- -The following steps were ran on a basic RHEL 9 instance via AWS. +The following steps were ran on a basic RHEL 9 instance via AWS. .. code-block:: RST sudo yum update -y sudo yum install automake -y - sudo yum install openssl-devel -y + sudo yum install openssl-devel -y sudo yum install pkg-config -y sudo yum install libtool -y sudo yum install python3 -y sudo yum install python3-devel.x86_64 -y sudo yum install python3-Cython -y - sudo yum install make -y - sudo yum install bison -y + sudo yum install make -y + sudo yum install bison -y sudo yum install flex -y - + LDMS Source Installation Instructions -------------------------- @@ -106,31 +106,31 @@ Getting the Source * This example shows cloning into $HOME/Source/ovis-4 and installing into $HOME/ovis/4.4.2 .. code-block:: RST - + mkdir $HOME/Source mkdir $HOME/ovis cd $HOME/Source git clone -b OVIS-4.4.2 https://github.com/ovis-hpc/ovis.git ovis-4 - + Building the Source ----------------------- * Run autogen.sh .. code-block:: RST - cd $HOME/Source/ovis + cd $HOME/Source/ovis ./autogen.sh * Configure and Build (Builds default linux samplers. Build installation directory is prefix): .. code-block:: RST - + mkdir build cd build ../configure --prefix=$HOME/ovis/4.4.2 make make install - + Basic Configuration and Running ******************************* * Set up environment: @@ -143,12 +143,12 @@ Basic Configuration and Running export ZAP_LIBPATH=$OVIS/lib/ovis-ldms export PATH=$OVIS/sbin:$OVIS/bin:$PATH export PYTHONPATH=$OVIS/lib/python3.8/site-packages - + Sampler *********************** * Edit a new configuration file, named `sampler.conf`, to load the `meminfo` and `vmstat` samplers. For this example, it can be saved anywhere, but it will be used later to start the LDMS Daemon (`ldmsd`) -The following configuration employs generic hostname, uid, gid, component id, and permissions octal set values. +The following configuration employs generic hostname, uid, gid, component id, and permissions octal set values. Sampling intervals are set using a "microsecond" time unit (i.e., 1 sec=1e+6 µs), and are adjustable, as needed. Some suggestions include: @@ -180,7 +180,7 @@ Some suggestions include: .. code-block:: RST :linenos: - + # Meminfo Sampler Plugin using 1 second sampling interval load name=meminfo config name=meminfo producer=host1 instance=host1/meminfo component_id=1 schema=meminfo job_set=host1/jobinfo uid=12345 gid=12345 perm=0755 @@ -189,8 +189,8 @@ Some suggestions include: load name=vmstat config name=vmstat producer=host1 instance=host1/vmstat component_id=1 schema=vmstat job_set=host1/jobinfo uid=0 gid=0 perm=0755 start name=vmstat interval=10000000 offset=0 - -As an alternative to the configuration above, one may, instead, export environmental variables to set LDMS's runtime configuration by using variables to reference those values in the sampler configuration file. + +As an alternative to the configuration above, one may, instead, export environmental variables to set LDMS's runtime configuration by using variables to reference those values in the sampler configuration file. The following setup will set the samplers to collect at 1 second, (i.e., 1000000 µs) intervals: @@ -213,18 +213,18 @@ The following setup will set the samplers to collect at 1 second, (i.e., 1000000 config name=vmstat producer=${HOSTNAME} instance=${HOSTNAME}/vmstat component_id=${COMPONENT_ID} schema=vmstat job_set=${HOSTNAME}/jobinfo uid=0 gid=0 perm=0755 start name=vmstat interval=${SAMPLE_INTERVAL} offset=${SAMPLE_OFFSET} -* Run a daemon using munge authentication: +* Run a daemon using munge authentication: .. code-block:: RST - + ldmsd -x sock:10444 -c sampler.conf -l /tmp/demo_ldmsd.log -v DEBUG -a munge -r $(pwd)/ldmsd.pid - + Or in non-cluster environments where munge is unavailable: .. code-block:: RST - + ldmsd -x sock:10444 -c sampler.conf -l /tmp/demo_ldmsd.log -v DEBUG -r $(pwd)/ldmsd.pid - + .. note:: For the rest of these instructions, omit the "-a munge" if you do not have munge running. This will also write out DEBUG-level information to the specified (-l) log. @@ -235,7 +235,7 @@ Or in non-cluster environments where munge is unavailable: ldms_ls -h localhost -x sock -p 10444 -a munge ldms_ls -h localhost -x sock -p 10444 -v -a munge ldms_ls -h localhost -x sock -p 10444 -l -a munge - + .. note:: Note the use of munge. Users will not be able to query a daemon launched with munge if not querying with munge. Users will only be able to see sets as allowed by the permissions in response to `ldms_ls`. @@ -244,12 +244,12 @@ Example (note permissions and update hint): .. code-block:: RST ldms_ls -h localhost -x sock -p 10444 -l -v -a munge - + Output: .. code-block:: RST - host1/vmstat: consistent, last update: Mon Oct 22 16:58:15 2018 -0600 [1385us] + host1/vmstat: consistent, last update: Mon Oct 22 16:58:15 2018 -0600 [1385us] APPLICATION SET INFORMATION ------ updt_hint_us : 5000000:0 METADATA -------- @@ -274,8 +274,8 @@ Output: D u64 app_id 0 D u64 nr_free_pages 32522123 ... - D u64 pglazyfree 1082699829 - host1/meminfo: consistent, last update: Mon Oct 22 16:58:15 2018 -0600 [1278us] + D u64 pglazyfree 1082699829 + host1/meminfo: consistent, last update: Mon Oct 22 16:58:15 2018 -0600 [1278us] APPLICATION SET INFORMATION ------ updt_hint_us : 5000000:0 METADATA -------- @@ -303,7 +303,7 @@ Output: D u64 MemAvailable 129556912 ... D u64 DirectMap1G 134217728 - + Aggregator Using Data Pull *********************** @@ -323,20 +323,20 @@ Aggregator Using Data Pull updtr_add name=policy_h2 interval=2000000 offset=100000 updtr_prdcr_add name=policy_h2 regex=host2 updtr_start name=policy_h2 - + * On host3, set up the environment as above and run a daemon: .. code-block:: RST ldmsd -x sock:10445 -c agg11.conf -l /tmp/demo_ldmsd.log -v ERROR -a munge - + * Run `ldms_ls` on the aggregator node to see set listing: .. code-block:: RST ldms_ls -h localhost -x sock -p 10445 -a munge - + Output: .. code-block:: RST @@ -345,12 +345,12 @@ Output: host1/vmstat host2/meminfo host2/vmstat - + You can also run `ldms_ls` to query the ldms daemon on the remote node: .. code-block:: RST - ldms_ls -h host1 -x sock -p 10444 -a munge + ldms_ls -h host1 -x sock -p 10444 -a munge Output: @@ -369,7 +369,7 @@ Aggregator Using Data Push * Make a configuration file (called agg11_push.conf) to cause the two samplers to push their data to the aggregator as they update. * Note that the prdcr configs remain the same as above but the updater_add includes the additional options: push=onchange auto_interval=false. - + * Note that the updtr_add interval has no effect in this case but is currently required due to syntax checking .. code-block:: RST @@ -381,20 +381,20 @@ Aggregator Using Data Push updtr_add name=policy_all interval=5000000 push=onchange auto_interval=false updtr_prdcr_add name=policy_all regex=.* updtr_start name=policy_all - - -* On host3, set up the environment as above and run a daemon: + + +* On host3, set up the environment as above and run a daemon: .. code-block:: RST ldmsd -x sock:10445 -c agg11_push.conf -l /tmp/demo_ldmsd_log -v DEBUG -a munge - + * Run ldms_ls on the aggregator node to see set listing: .. code-block:: RST - ldms_ls -h localhost -x sock -p 10445 -a munge - + ldms_ls -h localhost -x sock -p 10445 -a munge + Output: .. code-block:: RST @@ -403,9 +403,9 @@ Output: host1/vmstat host2/meminfo host2/vmstat - -Two Aggregators Configured as Failover Pairs + +Two Aggregators Configured as Failover Pairs *********************** * Use same sampler configurations as above * Make a configuration file (called agg11.conf) to aggregate from one sampler with the following contents: @@ -419,38 +419,38 @@ Two Aggregators Configured as Failover Pairs updtr_start name=policy_all failover_config host=host3 port=10446 xprt=sock type=active interval=1000000 peer_name=agg12 timeout_factor=2 failover_start - + * On host3, set up the environment as above and run two daemons as follows: .. code-block:: RST ldmsd -x sock:10445 -c agg11.conf -l /tmp/demo_ldmsd_log -v ERROR -n agg11 -a munge ldmsd -x sock:10446 -c agg12.conf -l /tmp/demo_ldmsd_log -v ERROR -n agg12 -a munge - + * Run ldms_ls on each aggregator node to see set listing: .. code-block:: RST - ldms_ls -h localhost -x sock -p 10445 -a munge + ldms_ls -h localhost -x sock -p 10445 -a munge host1/meminfo host1/vmstat ldms_ls -h localhost -x sock -p 10446 -a munge host2/meminfo host2/vmstat - + * Kill one daemon: .. code-block:: RST kill -SIGTERM - + * Make sure it died * Run ldms_ls on the remaining aggregator to see set listing: .. code-block:: RST - ldms_ls -h localhost -x sock -p 10446 -a munge - + ldms_ls -h localhost -x sock -p 10446 -a munge + Output: .. code-block:: RST @@ -464,60 +464,60 @@ Set Groups *********************** A set group is an LDMS set with special information to represent a group of sets inside ldmsd. A set group would appear as a regular LDMS set to other LDMS applications, but ldmsd and `ldms_ls` will treat it as a collection of LDMS sets. If ldmsd updtr updates a set group, it also subsequently updates all the member sets. Performing ldms_ls -l on a set group will also subsequently perform a long-query all the sets in the group. -To illustrate how a set group works, we will configure 2 sampler daemons with set groups and 1 aggregator daemon that updates and stores the groups in the following subsections. +To illustrate how a set group works, we will configure 2 sampler daemons with set groups and 1 aggregator daemon that updates and stores the groups in the following subsections. Creating a set group and inserting sets into it *********************** -The following is a configuration file for our s0 LDMS daemon (sampler #0) that collects sda disk stats in the s0/sda set and lo network usage in the s0/lo set. The s0/grp set group is created to contain both s0/sda and s0/lo. +The following is a configuration file for our s0 LDMS daemon (sampler #0) that collects sda disk stats in the s0/sda set and lo network usage in the s0/lo set. The s0/grp set group is created to contain both s0/sda and s0/lo. .. code-block:: RST ### s0.conf - load name=procdiskstats + load name=procdiskstats config name=procdiskstats device=sda producer=s0 instance=s0/sda - start name=procdiskstats interval=1000000 offset=0 - - load name=procnetdev - config name=procnetdev ifaces=lo producer=s0 instance=s0/lo - start name=procnetdev interval=1000000 offset=0 - - setgroup_add name=s0/grp producer=s0 interval=1000000 offset=0 - setgroup_ins name=s0/grp instance=s0/sda,s0/lo + start name=procdiskstats interval=1000000 offset=0 + + load name=procnetdev + config name=procnetdev ifaces=lo producer=s0 instance=s0/lo + start name=procnetdev interval=1000000 offset=0 + + setgroup_add name=s0/grp producer=s0 interval=1000000 offset=0 + setgroup_ins name=s0/grp instance=s0/sda,s0/lo -The following is the same for s1 sampler daemon, but with different devices (sdb and eno1). +The following is the same for s1 sampler daemon, but with different devices (sdb and eno1). .. code-block:: RST ### s1.conf - load name=procdiskstats + load name=procdiskstats config name=procdiskstats device=sdb producer=s1 instance=s1/sdb - start name=procdiskstats interval=1000000 offset=0 - - load name=procnetdev - config name=procnetdev ifaces=eno1 producer=s1 instance=s1/eno1 - start name=procnetdev interval=1000000 offset=0 - - setgroup_add name=s1/grp producer=s1 interval=1000000 offset=0 - setgroup_ins name=s1/grp instance=s1/sdb,s1/eno1 - -The s0 LDMS daemon is listening on port 10000 and the s1 LDMS daemon is listening on port 10001. + start name=procdiskstats interval=1000000 offset=0 + + load name=procnetdev + config name=procnetdev ifaces=eno1 producer=s1 instance=s1/eno1 + start name=procnetdev interval=1000000 offset=0 + + setgroup_add name=s1/grp producer=s1 interval=1000000 offset=0 + setgroup_ins name=s1/grp instance=s1/sdb,s1/eno1 + +The s0 LDMS daemon is listening on port 10000 and the s1 LDMS daemon is listening on port 10001. Perform `ldms_ls` on a group *********************** Performing `ldms_ls -v` or `ldms_ls -l` on a LDMS daemon hosting a group will perform the query on the set representing the group itself as well as iteratively querying the group's members. -Example: +Example: .. code-block:: RST ldms_ls -h localhost -x sock -p 10000 - + Output: .. code-block:: RST ldms_ls -h localhost -x sock -p 10000 -v s0/grp | grep consistent - + Output: .. code-block:: RST @@ -525,44 +525,44 @@ Output: s0/grp: consistent, last update: Mon May 20 15:44:30 2019 -0500 [511879us] s0/lo: consistent, last update: Mon May 20 16:13:16 2019 -0500 [1126us] s0/sda: consistent, last update: Mon May 20 16:13:17 2019 -0500 [1176us] - + .. code-block:: RST ldms_ls -h localhost -x sock -p 10000 -v s0/lo | grep consistent # only query lo set from set group s0 - + .. note:: - The update time of the group set is the time that the last set was inserted into the group. + The update time of the group set is the time that the last set was inserted into the group. Update / store with set group *********************** -The following is an example of an aggregator configuration to match-update only the set groups, and their members, with storage policies: +The following is an example of an aggregator configuration to match-update only the set groups, and their members, with storage policies: .. code-block:: RST - # Stores - load name=store_csv + # Stores + load name=store_csv config name=store_csv path=csv # strgp for netdev, csv file: "./csv/net/procnetdev" - strgp_add name=store_net plugin=store_csv container=net schema=procnetdev - strgp_prdcr_add name=store_net regex=.* - strgp_start name=store_net + strgp_add name=store_net plugin=store_csv container=net schema=procnetdev + strgp_prdcr_add name=store_net regex=.* + strgp_start name=store_net # strgp for diskstats, csv file: "./csv/disk/procdiskstats" - strgp_add name=store_disk plugin=store_csv container=disk schema=procdiskstats - strgp_prdcr_add name=store_disk regex=.* - strgp_start name=store_disk - - # Updater that updates only groups - updtr_add name=u interval=1000000 offset=500000 - updtr_match_add name=u regex=ldmsd_grp_schema match=schema - updtr_prdcr_add name=u regex=.* - updtr_start name=u - + strgp_add name=store_disk plugin=store_csv container=disk schema=procdiskstats + strgp_prdcr_add name=store_disk regex=.* + strgp_start name=store_disk + + # Updater that updates only groups + updtr_add name=u interval=1000000 offset=500000 + updtr_match_add name=u regex=ldmsd_grp_schema match=schema + updtr_prdcr_add name=u regex=.* + updtr_start name=u + Performing `ldms_ls` on the LDMS aggregator daemon exposes all the sets (including groups) .. code-block:: RST ldms_ls -h localhost -x sock -p 9000 - + Output: .. code-block:: RST @@ -573,13 +573,13 @@ Output: s0/sda s0/lo s0/grp - + Performing `ldms_ls -v` on a LDMS daemon hosting a group again but only querying the group and its members: .. code-block:: RST ldms_ls -h localhost -x sock -p 9000 -v s1/grp | grep consistent - + Output: .. code-block:: RST @@ -587,12 +587,12 @@ Output: s1/grp: consistent, last update: Mon May 20 15:42:34 2019 -0500 [891643us] s1/sdb: consistent, last update: Mon May 20 16:38:38 2019 -0500 [1805us] s1/eno1: consistent, last update: Mon May 20 16:38:38 2019 -0500 [1791us] - - -The following is an example of the CSV output: + + +The following is an example of the CSV output: .. code-block:: RST - + > head csv/*/* .. code-block:: RST @@ -607,7 +607,7 @@ The following is an example of the CSV output: 1558387834.001121,1121,s0,0,0,0,197797,0,9132,0,5382606,0,69312,0,522561,0,446083,0,418086168,0,966856,0,0,0,213096,0,1036080,0,1327776668,0,1380408297,0 1558387835.001179,1179,s0,0,0,0,197797,0,9132,0,5382606,0,69312,0,522561,0,446083,0,418086168,0,966856,0,0,0,213096,0,1036080,0,1327776668,0,1380408297,0 1558387835.001193,1193,s1,0,0,0,108887,0,32214,0,1143802,0,439216,0,1,0,0,0,8,0,44,0,0,0,54012,0,439240,0,1309384656,0,1166016512,0 - + ==> csv/net/procnetdev <== #Time,Time_usec,ProducerName,component_id,job_id,app_id,rx_bytes#lo,rx_packets#lo,rx_errs#lo,rx_drop#lo,rx_fifo#lo,rx_frame#lo,rx_compressed#lo,rx_multicast#lo,tx_bytes#lo,tx_packets#lo,tx_errs#lo,tx_drop#lo,tx_fifo#lo,tx_colls#lo,tx_carrier#lo,tx_compressed#lo 1558387831.001798,1798,s0,0,0,0,12328527,100865,0,0,0,0,0,0,12328527,100865,0,0,0,0,0,0 diff --git a/rtd/docs/source/ldms-streams.rst b/rtd/docs/source/ldms-streams.rst index 40818c40b..3f4659a05 100644 --- a/rtd/docs/source/ldms-streams.rst +++ b/rtd/docs/source/ldms-streams.rst @@ -4,7 +4,7 @@ Streams-enabled Application Data Collectors Caliper *********************** -This section covers the basic steps on how to compile, build and use the caliperConnector. +This section covers the basic steps on how to compile, build and use the caliperConnector. **What Is Caliper?** @@ -21,22 +21,22 @@ Build the Caliper program with the application you wish to analyze. No modificat One built, you will need to poin the $LD_LIBRARY_PATH to Caliper's library: .. code-block:: RST - + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/lib64 Now, to enable LDMS data collection, set (or export) the following list of caliper variables to ``ldms`` when executing a program. An example is shown below: .. code-block:: RST - + CALI_LOOP_MONITOR_ITERATION_INTERVAL=10 ./caliper_example.o 400 CALI_SERVICES_ENABLE=loop_monitor,mpi,ldms -The ``CALI_LOOP_MONITOR_ITERATION_INTERVAL`` collects measurements every n loop iterations of the acpplicaiton and the ``CALI_SERVICES_ENABLE`` define which services will be combined to collect the data. +The ``CALI_LOOP_MONITOR_ITERATION_INTERVAL`` collects measurements every n loop iterations of the acpplicaiton and the ``CALI_SERVICES_ENABLE`` define which services will be combined to collect the data. Once done, you will just need to execute your program and you will have application data collected by Caliper and LDMS. .. note:: - + The MPI service (i.e., mpi) is required when enabling LDMS because it is used for associating the MPI rank data collected by LDMS. LDMS Expected Output @@ -44,7 +44,7 @@ LDMS Expected Output LDMS collects a set of runtime timeseries data of the application in parallel with Caliper. Below is an example output of the data collect, formatted into a JSON string: .. code-block:: - + {"job_id":11878171,"ProducerName":“n1","rank":0,"timestamp":1670373198.056455,"region":"init","time":33.172237 } {"job_id":11878171,"ProducerName":"n1","rank":0,"timestamp":1670373198.056455,"region":"initialization","time":33.211929 } {"job_id":11878171,"ProducerName":“n1","rank":0,"timestamp":1670373198.056455,"region":"main","time":44.147736 } @@ -60,11 +60,11 @@ Any data collected by LDMS should have the same fields as the one shown above an Darshan *********************** -This section covers basics steps on how to compile, build and use the Darshan-LDMS Integration code (i.e. darshanConnector). The following application tests are part of the Darshan program and can be found under ``/darshan/darshan-test/regression/test-cases/src/``. +This section covers basics steps on how to compile, build and use the Darshan-LDMS Integration code (i.e. darshanConnector). The following application tests are part of the Darshan program and can be found under ``/darshan/darshan-test/regression/test-cases/src/``. **What Is Darshan?** -A lightweight I/O characterization tool that transparently captures application I/O behavior from HPC applications with minimal overhead. +A lightweight I/O characterization tool that transparently captures application I/O behavior from HPC applications with minimal overhead. **What Is The darshanConnector?** @@ -74,7 +74,7 @@ A Darshan-LDMS functionality that utilizes LDMS Streams to collect Darshan’s o :caption: The above diagrams provieds a high level visualization of the darshanConnector. During the Darshan initialization, the connector (on the left-hand side) checks to see if darshan has been built against the ldms library and if it has it will initialize a connection to the LDMS stream daemon when the DARSHAN_LDMS_ENABLE is set. Once initialized, the connecter will know which module data we want to collect by checking which environment variables are set. For example, if MPI-IO_ENABLE_LDMS is set, that specific I/O event data will be collected. The runtime data collection and JSON message formatting is then performed in the darshan ldms connector send function. This function is triggered whenever an I/O event occurs. The data is then published to LDMS streams interface and sent to through the LDMS Transport to be stored into a database. As you can see at the very bottom left is the JSON formatted message. Meanwhile, on the right, darshan is running as usual by initializing their modules, collecting the I/O event data for these modules, aggregating and calculating the data and then outputting the information into a Darshan log file. As you can see, the LDMS Streams implementation does not interfere with Darshan .. note:: - + LDMS must already be installed on the system or locally. If it is not, then please following ``Getting The Source`` and ``Building The Source`` in the `LDMS Quickstart Guide `_. If the Darshan-LDMS code is already deployed on your system, please skip to `Run An LDMS Streams Daemon`_ **Metric Definitions** @@ -111,7 +111,7 @@ Below are the list of Darshan metrics that are currently being collected by the * ``cnt:`` The count of the operations ("op" field) performed per module per rank. Resets to 0 after each "close" operation. * ``seg:`` Contains the following array metrics from the operation ("op" field): - + ``pt_sel: HDF5 number of different access selections. reg_hslab: HDF5 number of regular hyperslabs. irreg_hslab: HDF5 number of irregular hyperslabs. @@ -138,9 +138,9 @@ All data fields which that not change throughout the entire application run (i.e Compile and Build with LDMS --------------------------- 1. Run the following to build Darshan and link against an existing LDMS library on the system. - + .. code-block:: RST - + git clone https://github.com/darshan-hpc/darshan.git cd darshan && mkdir build/ ./prepare.sh && cd build/ @@ -149,16 +149,16 @@ Compile and Build with LDMS --prefix=/darshan/ \ --with-JOB_ID-env= \ --enable-ldms-mod \ - --with-ldms= + --with-ldms= make && make install .. note:: - * This configuration is specific to the system. should be replaced by the compiler wrapper for your MPI Library, (e.g., ``mpicc`` for Open MPI, or ``cc`` for Cray Development Environment MPI wrappers). + * This configuration is specific to the system. should be replaced by the compiler wrapper for your MPI Library, (e.g., ``mpicc`` for Open MPI, or ``cc`` for Cray Development Environment MPI wrappers). * If running an MPI program, make sure an MPI library is installed/loaded on the system. For more information on how to install and build the code across various platforms, please visit `Darshan's Runtime Installation Page `_ * ``--with-jobid-env=`` expects a string that is the environment variable that the hosted job scheduler utilizes on the HPC system. (e.g., Slurm would use ``--with-jobid-env=SLURM_JOB_ID``) - -2. **OPTIONAL** To build HDF5 module for Darshan, you must first load the HDF5 modulefile with ``module load hdf5-parallel``, then run configure as follows: + +2. **OPTIONAL** To build HDF5 module for Darshan, you must first load the HDF5 modulefile with ``module load hdf5-parallel``, then run configure as follows: .. code-block:: RST @@ -167,28 +167,28 @@ Compile and Build with LDMS --prefix=/darshan/ \ --with-jobid-env= \ --enable-ldms-mod \ - --with-ldms= + --with-ldms= --enable-hdf5-mod \ - --with-hdf5= + --with-hdf5= make && make install 2a. **OPTIONAL** If you do not have HDF5 installed on your system, you may install Python's ``h5py`` package with: .. code-block:: RST - + sudo apt-get install -y hdf5-tools libhdf5-openmpi-dev openmpi-bin # we need to build h5py with the system HDF5 lib backend export HDF5_MPI="ON" CC=cc python -m pip install --no-binary=h5py h5py .. note:: - + If the HDF5 library is installed this way, you do not need to include the ``--with-hdf5`` flag during configuration. For more information on other methods and HDF5 versions to install, please visit `Darshan's Runtime Installation Page `_. - + Run an LDMS Streams Daemon --------------------------- -This section will go over how to start and configure a simple LDMS Streams deamon to collect the Darshan data and store to a CSV file. +This section will go over how to start and configure a simple LDMS Streams deamon to collect the Darshan data and store to a CSV file. If an LDMS Streams daemon is already running on the system then please skip to `Test the Darshan-LDMS Integrated Code (Multi Node)`_. 1. First, initialize an ldms streams daemon on a compute node as follows: @@ -202,7 +202,7 @@ If an LDMS Streams daemon is already running on the system then please skip to ` .. code-block:: RST - LDMS_INSTALL= + LDMS_INSTALL= export LD_LIBRARY_PATH="$LDMS_INSTALL/lib/:$LDMS_INSTALL/lib:$LD_LIBRARY_PATH" export LDMSD_PLUGIN_LIBPATH="$LDMS_INSTALL/lib/ovis-ldms/" export ZAP_LIBPATH="$LDMS_INSTALL/lib/ovis-ldms" @@ -214,19 +214,19 @@ If an LDMS Streams daemon is already running on the system then please skip to ` export HOSTNAME="localhost" .. note:: - + LDMS must already be installed on the system or locally. If it is not, then please follow ``Getting The Source`` and ``Building The Source`` in the `LDMS Quickstart Guide `_. 3. Next, create a file called **"darshan\_stream\_store.conf"** and add the following content to it: .. code-block:: RST - + load name=hello_sampler config name=hello_sampler producer=${HOSTNAME} instance=${HOSTNAME}/hello_sampler stream=darshanConnector component_id=${COMPONENT_ID} start name=hello_sampler interval=${SAMPLE_INTERVAL} offset=${SAMPLE_OFFSET} - + load name=stream_csv_store - config name=stream_csv_store path=./streams/store container=csv stream=darshanConnector rolltype=3 rollover=500000 + config name=stream_csv_store path=./streams/store container=csv stream=darshanConnector rolltype=3 rollover=500000 4. Next, run the LDSM Streams daemon with the following command: @@ -235,7 +235,7 @@ If an LDMS Streams daemon is already running on the system then please skip to ` ldmsd -x sock:10444 -c darshan_stream_store.conf -l /tmp/darshan_stream_store.log -v DEBUG -r ldmsd.pid .. note:: - + To check that the ldmsd daemon is connected running, run ``ps auwx | grep ldmsd | grep -v grep``, ``ldms_ls -h -x sock -p -a none -v`` or ``cat /tmp/darshan_stream_store.log``. Where is the node where the LDMS daemon exists and is the port number it is listening on. Test the Darshan-LDMS Integrated Code (Multi Node) @@ -252,7 +252,7 @@ Set The Environment export LD_PRELOAD=$DARSHAN_INSTALL_PATH/lib/libdarshan.so export LD_LIBRARY_PATH=$DARSHAN_INSTALL_PATH/lib:$LD_LIBRARY_PATH # optional. Please visit Darshan's webpage for more information. - export DARSHAN_MOD_ENABLE="DXT_POSIX,DXT_MPIIO" + export DARSHAN_MOD_ENABLE="DXT_POSIX,DXT_MPIIO" # uncomment if hdf5 is enabled #export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/include/hdf5/openmpi @@ -264,22 +264,22 @@ Set The Environment export DARSHAN_LDMS_HOST= export DARSHAN_LDMS_PORT=10444 export DARSHAN_LDMS_AUTH=none - + # enable LDMS data collection. No runtime data collection will occur if this is not exported. export DARSHAN_LDMS_ENABLE= - - # determine which modules we want to publish to ldmsd - #export DARSHAN_LDMS_ENABLE_MPIIO= - #export DARSHAN_LDMS_ENABLE_POSIX= + + # determine which modules we want to publish to ldmsd + #export DARSHAN_LDMS_ENABLE_MPIIO= + #export DARSHAN_LDMS_ENABLE_POSIX= #export DARSHAN_LDMS_ENABLE_STDIO= - #export DARSHAN_LDMS_ENABLE_HDF5= + #export DARSHAN_LDMS_ENABLE_HDF5= #export DARSHAN_LDMS_ENABLE_ALL= #export DARSHAN_LDMS_VERBOSE= -.. note:: - - The ```` is set to the node name the LDMS Streams daemon is running on (e.g. the node we previous ssh'd into). Make sure the ``LD_PRELOAD`` and at least one of the ``DARSHAN_LDMS_ENABLE_*`` variables are set. If not, no data will be collected by LDMS. - +.. note:: + + The ```` is set to the node name the LDMS Streams daemon is running on (e.g. the node we previous ssh'd into). Make sure the ``LD_PRELOAD`` and at least one of the ``DARSHAN_LDMS_ENABLE_*`` variables are set. If not, no data will be collected by LDMS. + .. note:: ``DARSHAN_LDMS_VERBOSE`` outputs the JSON formatted messages sent to the LDMS streams daemon. The output will be sent to STDERR. @@ -289,12 +289,12 @@ Execute Test Application Now we will test the darshanConnector with Darshan's example ``mpi-io-test.c`` code by setting the following environment variables: .. code-block:: RST - + export PROG=mpi-io-test export DARSHAN_TMP=/tmp/darshan-ldms-test export DARSHAN_TESTDIR=/darshan/darshan-test/regression export DARSHAN_LOGFILE_PATH=$DARSHAN_TMP - + Now ``cd`` to the executable and test the appilcation with the darshanConnector enabled. .. code-block:: RST @@ -305,15 +305,15 @@ Now ``cd`` to the executable and test the appilcation with the darshanConnector srun ${PROG} -f $DARSHAN_TMP/${PROG}.tmp.dat Once the application is complete, to view the data please skip to `Check Results`_. - -Test the Darshan-LDMS Integrated Code (Single Node) + +Test the Darshan-LDMS Integrated Code (Single Node) ---------------------------------- The section goes over step-by-step instructions on how to compile and execute the ``mpi-io-test.c`` program under ``darshan/darshan-test/regression/test-cases/src/``, collect the data with the LDMS streams daemon and store it to a CSV file on a single login node. This section is for those who will not be running their applications on a cluster (i.e. no compute nodes). 1. Set Environment Variables for Darshan, LDMS and Darshan-LDMS Integrated code (i.e. darshanConnector). .. code-block:: RST - + # Darshan export DARSHAN_INSTALL_PATH= export LD_PRELOAD=/lib/libdarshan.so @@ -324,10 +324,10 @@ The section goes over step-by-step instructions on how to compile and execute th # uncomment if hdf5 is enabled #export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/include/hdf5/openmpi #export HDF5_LIB=/libhdf5.so - + # LDMS - LDMS_INSTALL= + LDMS_INSTALL= export LD_LIBRARY_PATH="$LDMS_INSTALL/lib/:$LDMS_INSTALL/lib:$LD_LIBRARY_PATH" export LDMSD_PLUGIN_LIBPATH="$LDMS_INSTALL/lib/ovis-ldms/" export ZAP_LIBPATH="$LDMS_INSTALL/lib/ovis-ldms" @@ -337,7 +337,7 @@ The section goes over step-by-step instructions on how to compile and execute th export SAMPLE_INTERVAL="1000000" export SAMPLE_OFFSET="0" export HOSTNAME="localhost" - + # darshanConnector export DARSHAN_LDMS_STREAM=darshanConnector export DARSHAN_LDMS_XPRT=sock @@ -347,10 +347,10 @@ The section goes over step-by-step instructions on how to compile and execute th # enable LDMS data collection. No runtime data collection will occur if this is not exported. export DARSHAN_LDMS_ENABLE= - - # determine which modules we want to publish to ldmsd - #export DARSHAN_LDMS_ENABLE_MPIIO= - #export DARSHAN_LDMS_ENABLE_POSIX= + + # determine which modules we want to publish to ldmsd + #export DARSHAN_LDMS_ENABLE_MPIIO= + #export DARSHAN_LDMS_ENABLE_POSIX= #export DARSHAN_LDMS_ENABLE_STDIO= #export DARSHAN_LDMS_ENABLE_HDF5= #export DARSHAN_LDMS_ENABLE_ALL= @@ -362,13 +362,13 @@ The section goes over step-by-step instructions on how to compile and execute th 2. Generate the LDMSD Configuration File and Start the Daemon -.. code-block:: RST +.. code-block:: RST cat > darshan_stream_store.conf << EOF load name=hello_sampler config name=hello_sampler producer=${HOSTNAME} instance=${HOSTNAME}/hello_sampler stream=darshanConnector component_id=${COMPONENT_ID} start name=hello_sampler interval=${SAMPLE_INTERVAL} offset=${SAMPLE_OFFSET} - + load name=stream_csv_store config name=stream_csv_store path=./streams/store container=csv stream=darshanConnector rolltype=3 rollover=500000 EOF @@ -376,19 +376,19 @@ The section goes over step-by-step instructions on how to compile and execute th ldmsd -x sock:10444 -c darshan_stream_store.conf -l /tmp/darshan_stream_store.log -v DEBUG # check daemon is running ldms_ls -p 10444 -h localhost -v - + 3. Set Up Test Case Variables -.. code-block:: RST +.. code-block:: RST export PROG=mpi-io-test export DARSHAN_TMP=/tmp/darshan-ldms-test export DARSHAN_TESTDIR=/darshan/darshan-test/regression export DARSHAN_LOGFILE_PATH=$DARSHAN_TMP - + 4. Run Darshan's mpi-io-test.c program -.. code-block:: RST +.. code-block:: RST cd darshan/darshan-test/regression/test-cases/src $DARSHAN_TESTDIR/test-cases/src/${PROG}.c -o $DARSHAN_TMP/${PROG} @@ -396,10 +396,10 @@ The section goes over step-by-step instructions on how to compile and execute th ./${PROG} -f $DARSHAN_TMP/${PROG}.tmp.dat Once the application is complete, to view the data please skip to `Check Results`_. - -Pre-Installed Darshan-LDMS + +Pre-Installed Darshan-LDMS --------------------------- -If both the Darshan-LDMS integrated code (i.e., darshanConnector) and LDMS are already installed, and a system LDMS streams daemon is running, then there are two ways to enable the LDMS functionality: +If both the Darshan-LDMS integrated code (i.e., darshanConnector) and LDMS are already installed, and a system LDMS streams daemon is running, then there are two ways to enable the LDMS functionality: 1. Set the environment via sourcing the ``darshan_ldms.env`` script  @@ -415,20 +415,20 @@ If both the Darshan-LDMS integrated code (i.e., darshanConnector) and LDMS are a In order to enable the darshanConnector code on the system, just source the following env script: .. code-block:: RST - + module use /projects/ovis/modules/ source /projects/ovis/modules//darshan_ldms.env **OPTIONAL**: Add a "-v" when sourcing this file to enable verbose: .. code-block:: RST - + $ source /projects/ovis/modules//darshan_ldms.env -v This will output json messages collected by ldms to the terminal window. .. note:: - + The STDIO data will NOT be collected by LDMS. This is to prevent any recursive LDMS function calls.  2. Load Module @@ -437,10 +437,10 @@ This will output json messages collected by ldms to the terminal window. If you do not wish to set the environment using the env script from above, you can always load the ``darshan_ldms`` modulefile, as follows: .. code-block:: RST - + module use /projects/ovis/modules/ module load darshan_ldms - + **OPTIONAL**: If you decide to load the module, you will need to turn on verbose by setting the following environment variable in your run script: .. code-block:: RST @@ -464,8 +464,8 @@ If you want to collect all types of data then set all *_ENABLE_LDMS variables: export DARSHAN_LDMS_ENABLE_HDF5="" .. note:: - - All Darshan binary log-files (i.e. .darshan) will be saved to ``$LOGFILE_PATH_DARSHAN``, as specified at build time and exported in the user environment. + + All Darshan binary log-files (i.e. .darshan) will be saved to ``$LOGFILE_PATH_DARSHAN``, as specified at build time and exported in the user environment. .. code-block:: RST @@ -517,21 +517,21 @@ Once the module is loaded and the environment is set, you will just need to run If runtime errors or issues occur, then this is most likely due to incompatibility issues with the application build, or the Darshan-LDMS build that is using ``LD_PRELOAD``. You may debug the issue, as follows: - 1. Unset the ``LD_PRELOAD`` environment variable (e.g., ``unset LD_PRELOAD``), then run the application with: ``mpiexec -env LD_PRELOAD $DARSHAN_INSTALL_PATH/lib/libdarshan.so`` or ``srun --export=LD_PRELOAD=$DARSHAN_INSTALL_PATH/lib/libdarshan.so``. - For more information please see section 5.2 in `Darshan's Runtime Installation Page `_. + 1. Unset the ``LD_PRELOAD`` environment variable (e.g., ``unset LD_PRELOAD``), then run the application with: ``mpiexec -env LD_PRELOAD $DARSHAN_INSTALL_PATH/lib/libdarshan.so`` or ``srun --export=LD_PRELOAD=$DARSHAN_INSTALL_PATH/lib/libdarshan.so``. + For more information please see section 5.2 in `Darshan's Runtime Installation Page `_. - 2. If you are still running into runtime issues, please send an email to ldms@sandia.gov and provide: - a) mpi-io, hdf5, pnetcdf, compiler version (if applicable) used to build your application - b) Contents of your environment variables: $PATH, $LIBRARY_PATH, $LD_LIBRARY_PATH and $LD_PRELOAD. + 2. If you are still running into runtime issues, please send an email to ldms@sandia.gov and provide: + a) mpi-io, hdf5, pnetcdf, compiler version (if applicable) used to build your application + b) Contents of your environment variables: $PATH, $LIBRARY_PATH, $LD_LIBRARY_PATH and $LD_PRELOAD. Check Results ------------- LDMS Output //////////// -This section provides the expected output of an application run with the data published to LDMS streams daemon with a CSV storage plugin (see section `Run An LDMS Streams Daemon`_). +This section provides the expected output of an application run with the data published to LDMS streams daemon with a CSV storage plugin (see section `Run An LDMS Streams Daemon`_). -* If you are publishing to a Local Streams Daemon (compute or login nodes) to collect the Darshan data, then compare the generated ``csv`` file to the one shown below in this section. +* If you are publishing to a Local Streams Daemon (compute or login nodes) to collect the Darshan data, then compare the generated ``csv`` file to the one shown below in this section. * If you are publishing to a System Daemon, that aggregates the data and stores to a Scalable Object Store (SOS), please skip this section and go to the :doc:`SOS Quickstart Guide ` for more information about viewing and accessing data from this database. @@ -540,7 +540,7 @@ LDMS Log File * Once the application has completed, run ``cat /tmp/hello_stream_store.log`` in the terminal window where the ldmsd is running (compute node). You should see a similar output to the one below. .. code-block:: RST - + cat /tmp/hello_stream_store.log Fri Feb 18 11:35:23 2022: INFO : stream_type: JSON, msg: "{ "job_id":53023,"rank":3,"ProducerName":"nid00052","file":"darshan-output/mpi-io-test.tmp.dat","record_id":1601543006480890062,"module":"POSIX","type":"MET","max_byte":-1,"switches":-1,"flushes":-1,"cnt":1,"op":"opens_segment","seg":[{"data_set":"N/A","pt_sel":-1,"irreg_hslab":-1,"reg_hslab":-1,"ndims":-1,"npoints":-1,"off":-1,"len":-1,"dur":0.00,"timestamp":1645209323.082951}]}", msg_len: 401, entity: 0x155544084aa0 Fri Feb 18 11:35:23 2022: INFO : stream_type: JSON, msg: "{ "job_id":53023,"rank":3,"ProducerName":"nid00052","file":"N/A","record_id":1601543006480890062,"module":"POSIX","type":"MOD","max_byte":-1,"switches":-1,"flushes":-1,"cnt":1,"op":"closes_segment","seg":[{"data_set":"N/A","pt_sel":-1,"irreg_hslab":-1,"reg_hslab":-1,"ndims":-1,"npoints":-1,"off":-1,"len":-1,"dur":0.00,"timestamp":1645209323.083581}]}", msg_len: 353, entity: 0x155544083f60 @@ -553,7 +553,7 @@ CSV File .. code-block:: RST - #module,uid,ProducerName,switches,file,rank,flushes,record_id,exe,max_byte,type,job_id,op,cnt,seg:off,seg:pt_sel,seg:dur,seg:len,seg:ndims,seg:reg_hslab,seg:irreg_hslab,seg:data_set,seg:npoints,seg:timestamp,seg:total,seg:start + #module,uid,ProducerName,switches,file,rank,flushes,record_id,exe,max_byte,type,job_id,op,cnt,seg:off,seg:pt_sel,seg:dur,seg:len,seg:ndims,seg:reg_hslab,seg:irreg_hslab,seg:data_set,seg:npoints,seg:timestamp,seg:total,seg:start POSIX,99066,n9,-1,/lustre//darshan-ldms-output/mpi-io-test_lC.tmp.out,278,-1,9.22337E+18,/lustre//darshan-ldms-output/mpi-io-test,-1,MET,10697754,open,1,-1,-1,0.007415,-1,-1,-1,-1,N/A,-1,1662576527,0.007415,0.298313 MPIIO,99066,n9,-1,/lustre//darshan-ldms-output/mpi-io-test_lC.tmp.out,278,-1,9.22337E+18,/lustre//darshan-ldms-output/mpi-io-test,-1,MET,10697754,open,1,-1,-1,0.100397,-1,-1,-1,-1,N/A,-1,1662576527,0.100397,0.209427 POSIX,99066,n11,-1,/lustre//darshan-ldms-output/mpi-io-test_lC.tmp.out,339,-1,9.22337E+18,/lustre//darshan-ldms-output/mpi-io-test,-1,MET,10697754,open,1,-1,-1,0.00742,-1,-1,-1,-1,N/A,-1,1662576527,0.00742,0.297529 @@ -567,12 +567,12 @@ Compare With Darshan Log File(s) //////////////////////////////// Parse the Darshan binary file using Darshan's standard and DXT (only if the ``DXT Module`` is enabled) parsers. -.. code-block:: RST +.. code-block:: RST $DARSHAN_INSTALL_PATH/bin/darshan-parser --all $LOGFILE_PATH_DARSHAN/.darshan > $DARSHAN_TMP/${PROG}.darshan.txt $DARSHAN_INSTALL_PATH/bin/darshan-dxt-parser --show-incomplete $LOGFILE_PATH_DARSHAN/.darshan > $DARSHAN_TMP/${PROG}-dxt.darshan.txt -Now you can view the log(s) with ``cat $DARSHAN_TMP/${PROG}.darshan.txt`` or ``cat $DARSHAN_TMP/${PROG}-dxt.darshan.txt`` and compare them to the data collected by LDMS. +Now you can view the log(s) with ``cat $DARSHAN_TMP/${PROG}.darshan.txt`` or ``cat $DARSHAN_TMP/${PROG}-dxt.darshan.txt`` and compare them to the data collected by LDMS. The ``producerName``, file path and record_id of each job should match and, if ``dxt`` was enabled, the individual I/O statistics of each rank (i.e., start time and number of I/O operations). @@ -589,9 +589,9 @@ Setup and Configuration ---------------------- **The KokkosConnector** -A Kokkos-LDMS functionality that utilizes LDMS Streams to collect Kokkos related data during runtime. Kokkos sampler, provided by the Kokkos-tools library, controls the sampling rate and provides the option to sample data using a count-based push. It then formats the data to a JSON message and *publishes* it to an LDMS streams interface. +A Kokkos-LDMS functionality that utilizes LDMS Streams to collect Kokkos related data during runtime. Kokkos sampler, provided by the Kokkos-tools library, controls the sampling rate and provides the option to sample data using a count-based push. It then formats the data to a JSON message and *publishes* it to an LDMS streams interface. -.. warning:: +.. warning:: To use kokkosConnector, all users will need to install Kokkos-Tools. You can find their repository and instructions on installing it here: https://github.com/kokkos/kokkos-tools @@ -599,14 +599,14 @@ The following environmental variables are needed in an application's runscript t .. code-block:: RST - export KOKKOS_LDMS_HOST="localhost" - export KOKKOS_LDMS_PORT="412" + export KOKKOS_LDMS_HOST="localhost" + export KOKKOS_LDMS_PORT="412" export KOKKOS_PROFILE_LIBRARY="/kokkos-tools/common/kokkos_sampler/kp_sampler.so;/ovis/kokkosConnector/kp_kernel_ldms.so" export KOKKOS_SAMPLER_RATE=101 export KOKKOS_LDMS_VERBOSE=0 export KOKKOS_LDMS_AUTH="munge" export KOKKOS_LDMS_XPRT="sock" - + * The KOKKOS_SAMPLER_RATE variable determines the rate of messages pushed to streams and collected. Please note that it is in best practice to set this to a prime number to avoid collecting information from the same kernels. * The KOKKOS_LDMS_VERBOSE variable can be set to 1 for debug purposes which prints all collected kernel data to the console. @@ -614,10 +614,10 @@ How To Make A Data Connector ***************************** In order to create a data connector with LDMS to collect runtime timeseries application data, you will need to utilize LDMS's Streams Functionality. This section will provide the necessary functions and Streams API required to make the data connector. -The example (code) below is pulled from the Darshan-LDMS Integration code. +The example (code) below is pulled from the Darshan-LDMS Integration code. .. note:: - + The LDMS Streams functionality uses a push-based method to reduce memory consumed and data loss on the node. Include the following LDMS files @@ -625,7 +625,7 @@ Include the following LDMS files * First, the following libaries will need to be included in the program as these contain all the functions that the data connector will be using/calling. .. code-block:: RST - #include + #include #include #include @@ -634,7 +634,7 @@ Initialize All Necessary Variables * Next, the following variables will need to be initialized globally or accessible by the Streams API Functions described in the next section: -.. code-block:: RST +.. code-block:: RST #define SLURM_NOTIFY_TIMEOUT 5 ldms_t ldms_g; @@ -647,7 +647,7 @@ Initialize All Necessary Variables Copy "Hello Sampler" Streams API Functions ------------------------------------------ -Next, copy the ``ldms_t setup_connection`` and ``static void event_cb`` functions listed below. These functions originated from the `ldmsd_stream_subscribe.c `_ code. +Next, copy the ``ldms_t setup_connection`` and ``static void event_cb`` functions listed below. These functions originated from the `ldmsd_stream_subscribe.c `_ code. The ``setup_connection`` contains LDMS API calls that connects to the LDMS daemon and the ``static void event_cb`` is a callback function to check the connection status of the LDMS Daemon. @@ -731,8 +731,8 @@ Initialize and Connect to LDMSD Once the above functions have been copied, the ``setup_connection`` will need to be called in order to establish a connection an LDMS Streams Daemon. .. note:: - - The LDMS Daemon is configured with the `Streams Plugin `_ and should already be running on the node. The host is set to the node the daemon is running on and port is set to the port the daemon is listening to. Below you will find an example of the Darshan Connector for reference. + + The LDMS Daemon is configured with the `Streams Plugin `_ and should already be running on the node. The host is set to the node the daemon is running on and port is set to the port the daemon is listening to. Below you will find an example of the Darshan Connector for reference. .. code-block:: RST @@ -765,7 +765,7 @@ Once the above functions have been copied, the ``setup_connection`` will need to pthread_mutex_unlock(ln_lock); return; } - + The environment variables ``DARSHAN_LDMS_X`` are used to define the stream name (configured in the daemon), transport type (sock, ugni, etc.), host, port and authentication of the LDMSD. In this specific example, the stream name is set to "darshanConnector" so the environment variable, ``DARSHAN_LDMS_STREAM`` is exported as follows: ``export DARSHAN_LDMS_STREAM=darshanConnector`` .. note:: @@ -773,10 +773,10 @@ The environment variables ``DARSHAN_LDMS_X`` are used to define the stream name .. note:: If you run into the following error: ``error:unknown type name 'sem_t'`` then you will need to add the following libraries to your code: - + * ``#include `` * ``#include `` - + Publish Event Data to LDMSD ------------------------------------- Now we will create a function that will collect all relevent application events and publish to the LDMS Streams Daemon. In the Darshan-LDMS Integration, the following Darshan's I/O traces for each I/O event (i.e. open, close, read, write) are collected along with the absolute timestamp (for timeseries data) for each I/O event: @@ -809,7 +809,7 @@ Now we will create a function that will collect all relevent application events out_1: return; } - + .. note:: For more information about the various Darshan I/O traces and metrics collected, please visit `Darshan's Runtime Installation Page `_ and `Darshan LDMS Metrics Collected `_ pages. @@ -818,7 +818,7 @@ Once this function is called, it initializes a connection to the LDMS Streams Da There are various types of formats that can be used to publish the data (i.e. JSON, string, etc.) so please review the `Defining A Format`_ section for more information. -Collect Event Data +Collect Event Data ///////////////////////// To collect the application data in real time (and using the example given in this section), the ``void darshan_ldms_connector_send(arg1, arg2, arg3,....)`` will be placed in all sections of the code where we want to publish a message. From the Darshan-LDMS Integration code we would have: @@ -826,11 +826,11 @@ To collect the application data in real time (and using the example given in thi .. code-block:: RST darshan_ldms_connector_send(rec_ref->file_rec->counters[MPIIO_COLL_OPENS] + rec_ref->file_rec->counters[MPIIO_INDEP_OPENS], "open", -1, -1, -1, -1, -1, __tm1, __tm2, __ts1, __ts2, rec_ref->file_rec->fcounters[MPIIO_F_META_TIME], "MPIIO", "MET"); - -This line of code is placed within multiple macros (`MPIIO_RECORD_OPEN/READ/WRITE `_) in Darshan's MPIIO module. + +This line of code is placed within multiple macros (`MPIIO_RECORD_OPEN/READ/WRITE `_) in Darshan's MPIIO module. * Doing this will call the function everytime Darshan detects an I/O event from the application (i.e. read, write, open, close). Once called, the arguements will be passed to the function, added to the JSON formatted message and pushed to the LDMS daemon. -.. note:: - +.. note:: + For more information about how to store the published data from and LDMS Streams Daemon, please see the Stream CSV Store plugin man pages on a system where LDMS Docs are installed: ``man Plugin_stream_csv_store`` diff --git a/rtd/docs/source/ldms_man/Plugin_cray_dvs_sampler.rst b/rtd/docs/source/ldms_man/Plugin_cray_dvs_sampler.rst index 0a4fb56e2..7788bea20 100644 --- a/rtd/docs/source/ldms_man/Plugin_cray_dvs_sampler.rst +++ b/rtd/docs/source/ldms_man/Plugin_cray_dvs_sampler.rst @@ -48,17 +48,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be cray_dvs_sampler schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`cray_dvs_sampler\`. conffile= - | + | | Optional path to the configuration file DATA AND THE CONFIGURATION FILE diff --git a/rtd/docs/source/ldms_man/Plugin_jobid.rst b/rtd/docs/source/ldms_man/Plugin_jobid.rst index 8627358d6..73cb07581 100644 --- a/rtd/docs/source/ldms_man/Plugin_jobid.rst +++ b/rtd/docs/source/ldms_man/Plugin_jobid.rst @@ -38,29 +38,29 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be jobid. producer= - | + | | The producer name value. instance= - | + | | The name of the metric set. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`vmstat\`. component_id= - | + | | Optional component identifier. Defaults to zero. with_jobid= - | + | | Option to lookup job_id with set or 0 if not. The job_id column will always appear, but populated witn zero. diff --git a/rtd/docs/source/ldms_man/Plugin_papi.rst b/rtd/docs/source/ldms_man/Plugin_papi.rst index e50e7a8e8..d4e5cfa54 100644 --- a/rtd/docs/source/ldms_man/Plugin_papi.rst +++ b/rtd/docs/source/ldms_man/Plugin_papi.rst @@ -44,37 +44,37 @@ LDMSCTL CONFIGURATION ATTRIBUTE SYNTAX configuration line name= - | + | | This MUST be spapi. producer= - | + | | The producer string value. instance= - | + | | The name of the metric set schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. component_id= - | + | | Optional component identifier. Defaults to zero. with_jobid= - | + | | Option to collect job id with set or 0 if not. events= - | + | | Comma separated list of events. Available events can be determined using papi_avail command if papi is installed on system. pid - The PID for the process being monitored - | + | NOTES ============= @@ -91,7 +91,7 @@ EXAMPLES ================ The following is a short example that measures 4 events. - | + | | Total CPU cycles | Total CPU instructions | Total branch instructions diff --git a/rtd/docs/source/ldms_man/Plugin_rapl.rst b/rtd/docs/source/ldms_man/Plugin_rapl.rst index 42a5674d4..8705cc98d 100644 --- a/rtd/docs/source/ldms_man/Plugin_rapl.rst +++ b/rtd/docs/source/ldms_man/Plugin_rapl.rst @@ -42,11 +42,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be rapl. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`rapl\`. diff --git a/rtd/docs/source/ldms_man/Plugin_store_app.rst b/rtd/docs/source/ldms_man/Plugin_store_app.rst index 5b77ec790..237233afa 100644 --- a/rtd/docs/source/ldms_man/Plugin_store_app.rst +++ b/rtd/docs/source/ldms_man/Plugin_store_app.rst @@ -8,12 +8,12 @@ Plugin_store_app :depth: 3 .. -NAME +NAME ================== ldmsd_store_app - LDMSD store_app storage plugin -SYNOPSIS +SYNOPSIS ====================== **load** **name**\ =\ **store_app** @@ -27,7 +27,7 @@ SYNOPSIS **strgp_prdcr_add** **name**\ =\ *STRGP_NAME* **regex**\ =\ *PRDCR_REGEX* -DESCRIPTION +DESCRIPTION ========================= **``store_app``** is an LDMSD storage plugin for storing data from the @@ -66,21 +66,21 @@ attributes: - **``job_time_rank``**: (indexed) the join of **``job_id``**, **``timestamp``**, and **``rank``**. -CONFIG OPTIONS +CONFIG OPTIONS ============================ -name +name The name of the plugin instance to configure. -path +path The path to the directory that contains SOS containers (one container per strgp). -perm +perm The octal mode (e.g. 0777) that is used in SOS container creation. The default is **0660**. -EXAMPLES +EXAMPLES ====================== :: diff --git a/rtd/docs/source/ldms_man/Plugin_store_csv.rst b/rtd/docs/source/ldms_man/Plugin_store_csv.rst index c0fea3629..a0aad9e78 100644 --- a/rtd/docs/source/ldms_man/Plugin_store_csv.rst +++ b/rtd/docs/source/ldms_man/Plugin_store_csv.rst @@ -72,16 +72,16 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_csv. opt_file= - | + | | The options for the plugin and specific instances will be read from the named file. See OPTIONS FILE. path= - | + | | This option is required; the config line or the options file must supply a default value. The output files will be put into a directory whose root is specified by the path argument. This @@ -93,7 +93,7 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX time e.g., XXX/meminfo_ctr/meminfo-123456789. altheader=<0/!0> - | + | | Distinguishes whether or not to write the header to a separate file than the data file. 0 = same file. Any non-zero is a separate file. Default is the same file. If a separate file is @@ -104,7 +104,7 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX written at each rollover. Default is altheader=0. typeheader= - | + | | Controls the presence and format of a .KIND file. The kind CSV file gives type information on each metric (or metric array). For example, if the metric file is named meminfo, the kind file @@ -132,18 +132,18 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX contrast to the repetition employed by format 0. ietfcsv=<0/1> - | + | | Turns on (1) or off (0) use of IETF 4180 quoting for header column names. userdata=<0/!0> - | + | | Distinguishes whether or not to write each metrics' user data along with each data value. 0 = no write. Any non-zero means to write the values. Default is to not write. buffer=<0/1/N> - | + | | Distinguishes whether or not to buffer the data for the writeout. 0 = does not buffer. 1 enables buffering with the system determining the flush. N will flush after approximately N @@ -151,13 +151,13 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX these it is. Default is system controlled buffering (1). buffertype=<3/4> - | + | | If buffer=N then buffertype determines if the buffer parameter refers to kB of writeout or number of lines. The values are the same as in rolltype, so only 3 and 4 are applicable. rolltype= - | + | | By default, the store does not rollover and the data is written to a continously open filehandle. Rolltype and rollover are used in conjunction to enable the store to manage rollover, including @@ -165,26 +165,26 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX roll occurs. Valid options are: 1 - | + | | wake approximately every rollover seconds and roll. Rollover is suppressed if no data at all has been written and rollempty=0. 2 - | + | | wake daily at rollover seconds after midnight (>=0) and roll. Rollover is suppressed if no data at all has been written and rollempty=0. 3 - | + | | roll after approximately rollover records are written. 4 roll after approximately rollover bytes are written. 5 - | + | | wake at rollover seconds after midnight (>=0) and roll, then repeat every rollagain (> rollover) seconds during the day. For example "rollagain=3600 rollover=0 rolltype=5" rolls @@ -192,18 +192,18 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX been written and rollempty=0. rollover= - | + | | Rollover value controls the frequency of rollover (e.g., number of bytes, number of records, time interval, seconds after midnight). Note that these values are estimates. rollempty=0 - | + | | Turn off rollover of empty files. Default value is 1 (create extra empty files). create_perm= - | + | | Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The default permission is 0600 for data files. The mode specified can include execute bits @@ -212,18 +212,18 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX and 0644 for data files. create_uid= - | + | | Specify a new user id for data files. If unspecified, no change in user ownership is made. Changes in ownership of the files do not affect intermediate directories. create_gid= - | + | | Specify a new group id for data files. If unspecified, no change in group ownership is made. rename_template= - | + | | This option relocates closed CSV files, typically to a subdirectory, for processing by other tools that watch directories. The metapath template is applied to define a new @@ -240,14 +240,14 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX output directory and handle completed files. rename_perm= - | + | | Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The permissions are changed before the rename and even if the rename fails. This option is applied only if rename_template is applied. rename_uid= - | + | | Specify a new user id for the file. If unspecified, no change in user ownership is made. Changes in ownership of the files do not affect intermediate directories that might be created following @@ -255,28 +255,28 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX applied. rename_gid= - | + | | Specify a new group id for the file. If unspecified, no change in group ownership is made. This option is applied only if rename_template is applied. expand_array= - | + | | The default is false. Each array element is stored in a column. True means that all elements are stored in a single column. array_sep= - | + | | Specify a character to separate array elements. If exand_array is true, the value is ignored. array_lquote= - | + | | Specify the left-quote character if expand_array is true. If expand_array is false, the value is ignored. array_rquote= - | + | | Specify the right-quote character if expand_array is true. If expand_array is false, the value is ignored. @@ -312,21 +312,21 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_csv. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). They also are used to match any specific config lines. schema= - | + | | The container and the schema determine where the output files will be written (see path above). You can have multiples of the same sampler, but with different schema (which means they will @@ -334,7 +334,7 @@ output files via identification of the container and schema. containers (and therefore files). decomposition= - | + | | Optionally use set-to-row decomposition with the specified configuration file in JSON format. See more about decomposition in ldmsd_decomposition(7). @@ -373,38 +373,38 @@ The following % escape sequence replacements are performed on the rename_template value for file renamings: %P - | + | | plugin name %C - | + | | container name %S - | + | | schema name %T - | + | | file type (DATA, HEADER, KIND, UNITS, CNAMES, PYNAMES) %B - | + | | basename(closed-file-name) %D - | + | | dirname(closed-file-name) %{ENV_VAR_NAME} - | + | | getenv(ENV_VAR_NAME). The use of undefined or empty environment vars yields an empty substitution, not an error. Characters in the environment variable are restricted to: 'A-Za-z0-9%@()+-\_./:='; other characters present will prevent the rename. %s - | + | | timestamp suffix, if it exists. NOTES diff --git a/rtd/docs/source/ldms_man/Plugin_store_kafka.rst b/rtd/docs/source/ldms_man/Plugin_store_kafka.rst index ff10f685c..08c58605c 100644 --- a/rtd/docs/source/ldms_man/Plugin_store_kafka.rst +++ b/rtd/docs/source/ldms_man/Plugin_store_kafka.rst @@ -40,7 +40,7 @@ PLUGIN CONFIGURATION Configuration Options: **name=**\ *store_kafka* - | + | | The name of the plugin. This must be **store_kafka**. **path=**\ *KAFKA_CONFIG_JSON_FILE* @@ -63,20 +63,20 @@ STRGP CONFIGURATION strgp options: **name=**\ *NAME* - | + | | The name of the strgp. **plugin=**\ store_kafka - | + | | The plugin must be store_kafka. **container=**\ *KAFKA_SERVER_LIST* - | + | | A comma-separated list of Kafka servers (host[:port]). For example: container=localhost,br1.kf:9898. **decomposition=**\ *DECOMP_CONFIG_JSON_FILE* - | + | | Set-to-row decomposition configuration file (JSON format). See more about decomposition in **ldmsd_decomposition**\ (7). diff --git a/rtd/docs/source/ldms_man/Plugin_store_rabbitkw.rst b/rtd/docs/source/ldms_man/Plugin_store_rabbitkw.rst index ac632cb66..b14499155 100644 --- a/rtd/docs/source/ldms_man/Plugin_store_rabbitkw.rst +++ b/rtd/docs/source/ldms_man/Plugin_store_rabbitkw.rst @@ -41,69 +41,69 @@ user, and pwfile are shared across all metric sets. | These parameters are: name= - | + | | This MUST be store_rabbitkw. routing_key - | + | | The routing key shared by all metric sets is . host= - | + | | The rabbitmq server host. The default is localhost. port= - | + | | The server port on the nearest rabbitmq host. The default is 5672. exchange= - | + | | The amqp exchange to publish with is . The default is amq.topic. This must preexist; the plugin will no cause its creation. vhost= - | + | | The virtual host to be used is . The default is "/". user= - | + | | The amqp username is . The default is "guest". pwfile= - | + | | The file contains the amqp user password in the format 'secretword=password. The default password "guest" is assumed if no file is specified. retry= - | + | | If amqp connection fails due to network or server issue, retry every seconds. Default is 60. heartbeat= - | + | | Heartbeat interval used to detect failed connections. timeout= - | + | | Timeout to use for connections, in milliseconds. Default is 1000. extraprops= - | + | | Turn on (y) or off (n) the use of extra properties with all messages. If AMQP-based filtering is not planned, 'n' will reduce message sizes slightly. logmsg= - | + | | Enable (y) or disable (n, the default) logging all message metric content at the DEBUG level. This is a debugging option. useserver= - | + | | Enable (y, the default) or disable (n) calls to the amqp server; this is a debugging option. @@ -114,16 +114,16 @@ STORE ATTRIBUTE SYNTAX | name= schema= container= name= - | + | | This MUST be store_rabbitkw. schema= - | + | | The name of the metric group, independent of the host name. The schema will be used as a header in messages if extraprops is y. container= - | + | | The container will be used as a header in messages if extraprops is y. @@ -137,12 +137,12 @@ The properties follow the AMQP standard, with LDMS specific interpretations: timestamp - | + | | The sample collection time in MICROSECONDS UTC. Divide by 1,000,000 to get seconds UTC. app_id - | + | | The app_id is LDMS. Optional AMQ event contents @@ -151,19 +151,19 @@ Optional AMQ event contents These fields and headers are present if extraprops=y is configured. content_type - | + | | <"text/plain"> for all. reply_to - | + | | The metric set instance name. container - | + | | The container configuration name. schema - | + | | The schema configuration name. PAYLOAD FORMAT diff --git a/rtd/docs/source/ldms_man/Plugin_store_rabbitv3.rst b/rtd/docs/source/ldms_man/Plugin_store_rabbitv3.rst index e8600a282..e1c59cc35 100644 --- a/rtd/docs/source/ldms_man/Plugin_store_rabbitv3.rst +++ b/rtd/docs/source/ldms_man/Plugin_store_rabbitv3.rst @@ -40,48 +40,48 @@ and pwfile are shared across all metric sets. | These parameters are: name= - | + | | This MUST be store_rabbitv3. root= - | + | | The routing key prefix shared by all metric sets will be . host= - | + | | The rabbitmq server host. The default is localhost. port= - | + | | The server port on the nearest rabbitmq host. The default is 5672. exchange= - | + | | The amqp exchange to publish with is . The default is amq.topic. vhost= - | + | | The virtual host to be used is . The default is "/". user= - | + | | The amqp username is . The default is "guest". pwfile= - | + | | The file contains the amqp user password in the format 'secretword=password. The default password "guest" is assumed if no file is specified. extraprops= - | + | | Turn on (y) or off (n) the use of extra properties with all messages. mint - | + | | The number of seconds between emission of time and host invariant (meta) metrics. @@ -92,15 +92,15 @@ STORE ATTRIBUTE SYNTAX | name= schema= container= name= - | + | | This MUST be store_rabbitv3. schema= - | + | | The name of the metric group, independent of the host name. container= - | + | | The container will be used in the routing key. The current routing key patterns is: .... @@ -121,16 +121,16 @@ The properties follow the AMQP standard, with LDMS specific interpretations: timestamp - | + | | The sample collection time in MICROSECONDS UTC. Divide by 1,000,000 to get seconds UTC. type - | + | | The ldms metric data type. app_id - | + | | The app_id is the integer component_id, if it has been defined by the sampler. @@ -140,36 +140,36 @@ Optional AMQ event contents These fields and headers are present if extraprops=y is configured. content_type - | + | | <"text/plain"> for all. reply_to - | + | | The producer name. metric - | + | | The label registered by the sampler plugin, which might be anything. metric_name_amqp - | + | | The label modified to work as a routing key, not necessarily easily read. metric_name_least - | + | | The label modified to work as a programming variable name, possibly shortened and including a hash suffix. Not expected to be fully human-readable in all cases. It will be the same across runs for metric sets whose content labels do not vary across runs. container - | + | | The container configuration name. schema - | + | | The schema configuration name. PAYLOAD FORMAT diff --git a/rtd/docs/source/ldms_man/Plugin_store_sos.rst b/rtd/docs/source/ldms_man/Plugin_store_sos.rst index 41fc197a6..f09e6e2d7 100644 --- a/rtd/docs/source/ldms_man/Plugin_store_sos.rst +++ b/rtd/docs/source/ldms_man/Plugin_store_sos.rst @@ -38,11 +38,11 @@ STORE_SOS INIT CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_sos. path= - | + | | The store will be put into a directory whose root is specified by the path argument. This directory must exist; the store will be created. The full path to the store will be @@ -61,27 +61,27 @@ container and schema for a store. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_sos. name= - | + | | The policy name for this strgp. container= - | + | | The container and schema define the store as described above (see path). schema= - | + | | The container and schema define the store as described above (see path). You can have multiples of the same path and container, but with different schema (which means they will have different metrics) and they will be stored in the same store. decomposition= - | + | | Optionally use set-to-row decomposition with the specified configuration file in JSON format. See more about decomposition in ldmsd_decomposition(7). @@ -101,11 +101,11 @@ into XXX. Any commands given with no argument, will return usage info. | Create a partition. **-C** ** - | + | | Path to the container **-s** *state* - | + | | State of the new partition (case insensitive). Default is OFFLINE. Optional parameter. Valid options are: @@ -117,7 +117,7 @@ into XXX. Any commands given with no argument, will return usage info. or deleted. **part_name** - | + | | Name of the partition **sos_part_delete** @@ -126,11 +126,11 @@ into XXX. Any commands given with no argument, will return usage info. OFFLINE state to be deleted. **-C** ** - | + | | Path to the container **name** - | + | | Name of the parition **sos_part_modify** @@ -138,11 +138,11 @@ into XXX. Any commands given with no argument, will return usage info. | Modify the state of a partition. **-C** ** - | + | | Path to the container **-s** *state* - | + | | State of the new partition (case insensitive). Default is OFFLINE. Optional parameter. Valid options are: @@ -154,24 +154,24 @@ into XXX. Any commands given with no argument, will return usage info. or deleted. **part_name** - | + | | Name of the partition **sos_part_move** - | + | | Move a partition to another storage location. -C -p part_name **-C** ** - | + | | Path to the container **-p** ** - | + | | The new path. **part_name** - | + | | Name of the partition USING SOS COMMANDS TO LOOK AT DATA IN A PARTITION @@ -185,7 +185,7 @@ command options are below. Example usage is in the example section. | Print a directory of the schemas. **-C** ** - | + | | Path to the container **sos_cmd** @@ -193,7 +193,7 @@ command options are below. Example usage is in the example section. | Show debug information for the container **-C** ** - | + | | Path to the container **sos_cmd** @@ -201,22 +201,22 @@ command options are below. Example usage is in the example section. | Print data from a container **-C** ** - | + | | Path to the container **-q** Used to query **-S** ** - | + | | Schema querying against **-X** ** - | + | | Variable that is indexed to use in the query. **-V** ** - | + | | One or more vars to output. NOTES diff --git a/rtd/docs/source/ldms_man/index.rst b/rtd/docs/source/ldms_man/index.rst index 215eb7f51..be58cdc9d 100644 --- a/rtd/docs/source/ldms_man/index.rst +++ b/rtd/docs/source/ldms_man/index.rst @@ -4,5 +4,5 @@ LDMS Man Pages .. toctree:: :maxdepth: 1 :glob: - + * diff --git a/rtd/docs/source/ldms_man/ldms-csv-anonymize.rst b/rtd/docs/source/ldms_man/ldms-csv-anonymize.rst index da3d0f443..fe14aebe4 100644 --- a/rtd/docs/source/ldms_man/ldms-csv-anonymize.rst +++ b/rtd/docs/source/ldms_man/ldms-csv-anonymize.rst @@ -36,12 +36,12 @@ OPTIONS ====================== --input= - | + | | Args is a file name or space-separated list of file names to be processed. Filenames cannot contain whitespace. --out-dir= - | + | | Path is a directory (must pre-exist and should not be the same as any directory containing the input) which will be filled with the changed files. The original files will are not changed. If an @@ -49,30 +49,30 @@ OPTIONS may be lost or corrupted. --col-sep= - | + | | Split columns at this character. The default is comma. ---save-maps= - | +--save-maps= + | | The path prefix for the generated map files. If the resulting map filenames coincide with an existing file, the existing file is overwritten. --imap= - | + | | An integer mapping file to preload. It must contain two columns of integers and magic. Normally it is the output of a prior run. See MAPS below. --nmap= - | + | | A name mapping file to preload. It must contain two columns of names and magic. Normally it is the output of a prior run. Each real name is replaced with 'n' and a sequential number. See MAPS below. --pmap= - | + | | A path element mapping file to preload. It must contain two columns of path elements and magic. Normally it is the output of a prior run. Path elements are unqualified subdirectory names. Each unique @@ -81,7 +81,7 @@ OPTIONS application identities. See MAPS below. --hmap= - | + | | A host name mapping file to preload. It must contain columns of host elements and magic. It may be host name fragment information or the output of a prior run. Any hostname found in the input data @@ -90,7 +90,7 @@ OPTIONS See MAPS below. --gen-args=[,M:H]\*, - | + | | Creating the M:C specification needed in a data transformation run can be done by first using the argument generation mode. Given a file starting with a header line of column names and the list of @@ -98,11 +98,11 @@ OPTIONS M:C arguments needed for the data transformation. --debug - | + | | Echo some details of the transformation as it runs. --seed - | + | | Supply a seed to the random number generator. No random values are used at this time in the processing, however. diff --git a/rtd/docs/source/ldms_man/ldms-csv-export-sos.rst b/rtd/docs/source/ldms_man/ldms-csv-export-sos.rst index 36d5b5bd2..28864c4bc 100644 --- a/rtd/docs/source/ldms_man/ldms-csv-export-sos.rst +++ b/rtd/docs/source/ldms_man/ldms-csv-export-sos.rst @@ -35,7 +35,7 @@ OPTIONS ======================= --data= - | + | | DATA is a file name of a LDMS .HEADER, .KIND, or data file. The file name and at least the first line of the file are digested to determine the content and the column types. LDMS CSV file name @@ -45,71 +45,71 @@ OPTIONS must also be gzipped. --blacklist= - | + | | BLACKLIST is the name of a file with column names to exclude from the schema, one per line. leading # comments allowed in the file. --whitelist= - | + | | WHITELIST is the name of a file with column names to include in the schema, one per line. leading # comments allowed in the file. Any other columns found are excluded. --exclude= - | + | | LIST is a string of metric names separated by commas. Columns named are excluded from the generated schema. --include= - | + | | LIST is a string of metric names separated by commas. Columns named are included in the generated schema and all other columns found are excluded. --schema-name= - | + | | NAME overrides the default schema name determined from the data file name. --schema-file= - | + | | Use an existing schema file FILE instead of generating a schema. When not specified, a schema file is always generated. Schema files may not be gzipped. ---map-file= - | +--map-file= + | | Override the output map file name derived from the data file name. ---alias-file= - | +--alias-file= + | | Provide the list of metrics to rename when creating or matching a schema discovered from a header line. ---strip-udata - | +--strip-udata + | | Suppress output of .userdata fields and remove .value suffix from schema element names. --guess - | + | | Guess the ldms data column types. (can be slow on large files) --maxlines= - | + | | Parse no more than MAXLINES to guess data types with the --guess option. The default if unspecified is 100000 lines. --assume= - | + | | Assume all unknown data columns are type ASSUME. --verbose - | + | | Show process debugging details. --widen - | + | | Widen numeric types discovered to 64 bits. METRIC FILTERING @@ -194,11 +194,11 @@ Other examples --schema-file=fptrans.SCHEMASOS # test input guess when x.14 does not exist - ldms-csv-export-sos --data=x.HEADER.14 --guess + ldms-csv-export-sos --data=x.HEADER.14 --guess # test input guess when y.KIND.14 does not exist but y.14 does ldms-csv-export-sos --data=y.HEADER.14 \ - --guess --maxlines=4000 + --guess --maxlines=4000 # test input guess and widen ldms-csv-export-sos --data=y.HEADER.14 \ diff --git a/rtd/docs/source/ldms_man/ldms-plugins.rst b/rtd/docs/source/ldms_man/ldms-plugins.rst index 47448cbca..ef9d57b0e 100644 --- a/rtd/docs/source/ldms_man/ldms-plugins.rst +++ b/rtd/docs/source/ldms_man/ldms-plugins.rst @@ -32,11 +32,11 @@ The names all, store, and sampler are interpreted as described in ldmsd(8). -b - | + | | Produce brief output, omitting usages. -n - | + | | Produce names only. EXAMPLES diff --git a/rtd/docs/source/ldms_man/ldms-run-static-tests.rst b/rtd/docs/source/ldms_man/ldms-run-static-tests.rst index 78c6e8452..774b543f7 100644 --- a/rtd/docs/source/ldms_man/ldms-run-static-tests.rst +++ b/rtd/docs/source/ldms_man/ldms-run-static-tests.rst @@ -40,11 +40,11 @@ OPTIONS ========================= -l - | + | | List the enabled plugins. -h - | + | | List help message. LANGUAGE @@ -60,19 +60,19 @@ Uses the current set environment to run. Environment may need to be configured before excuting this test script. input - | + | | The name of the input file as specified when ldms-static-test.sh is invoked for each enabled plugin. testname - | + | | The base name (directories stripped) of the input file name. This variable makes it possible to use similar input across many test files when the name of the input file is the same as the plugin tested. strict - | + | | If the variable "strict" is used for KILL_LDMSD (ldms-static-test(8)) the script will output "FAIL $testname" and return an XFAIL to indicate an expected failure only if the test @@ -81,7 +81,7 @@ strict test.$testname.log under the default output location of test_dir. file - | + | | The file "static-test-list" located in ldms/scripts/ defines a list of samplers that are expected to fail. If there is a failed test and the sampler is listed in this file, then run-static-test.sh @@ -89,7 +89,7 @@ file list to meet their needs. bypass <1,0> - | + | | This variable assignment is used to determine an expected failure (1) or normal failure (0) of a sampler plugin. This variable is set to (1) if the sampler is listed in $file and set to (0) otherwise. @@ -106,13 +106,13 @@ FILES ======================= *$input_file.$i* - | + | | For each value of i specifed to start an ldmsd, a configuration file named $input_file.$i must also exist. This configuration file is used when starting the daemon. *test_dir* - | + | | Test output directory of ldms-static-test.sh. The default output location is \`pwd`/ldmstest/$testname. @@ -120,11 +120,11 @@ GENERATED FILES ================================= *$test_dir/test.$testname.log* - | + | | The log file containing stderr and stdout of ldms-static-test.sh. *$test_dir/test.$testname.tgz* - | + | | Location of the compressed file logfile. SEE ALSO diff --git a/rtd/docs/source/ldms_man/ldms-static-test.rst b/rtd/docs/source/ldms_man/ldms-static-test.rst index 34ae91ec9..486a91f55 100644 --- a/rtd/docs/source/ldms_man/ldms-static-test.rst +++ b/rtd/docs/source/ldms_man/ldms-static-test.rst @@ -37,11 +37,11 @@ OPTIONS ==================== -l - | + | | List the canned tests available. -h - | + | | List help message. LANGUAGE @@ -51,7 +51,7 @@ The following macro language is provided as extensions on bash. Other bash use is also possible, but not recommended. DAEMONS - | + | | Give all the numbers that will be used in the LDMSD invocations anywhere in the test. This causes port variables to be defined so that any daemon can connect to any other by referencing $portN as @@ -59,7 +59,7 @@ DAEMONS aggregation relationships of LDMSD calls may be infeasible. LDMSD [conf-options] - | + | | This starts a number of daemons described by daemon-numbers. The numbers can be a given list, such as "1 2 3". The environment of each daemon (and its config script) will contain the variable i set @@ -70,25 +70,25 @@ LDMSD [conf-options] See CONFIGURATION OPTIONS below for the explanation of [conf-options]. MESSAGE [arguments] - | + | | The expanded arguments are logged. LDMS_LS [ldms_ls_args] - | + | | This invokes ldms_ls on the k-th ldmsd. KILL_LDMSD [strict] - | + | | Kills the listed daemons. If optional keyword strict is supplied, then missing daemons will cause the bypass variable to be set and the script to include an error code when it exits. SLEEP - | + | | Sleeps n seconds and logs a message about it. JOBDATA jobfile [daemon-numbers] - | + | | Creates jobfile with data for the jobid plugin to parse. If daemon numbers are specified, creates a jobfile.$k for each value of k listed in daemon-numbers. Each file will have unique numeric @@ -96,32 +96,32 @@ JOBDATA jobfile [daemon-numbers] slurm-plugin sampler binary format. vgon - | + | | Turns on use of valgrind for any ldmsd or ldms_ls subsequently started. vgoff - | + | | Turns off use of valgrind for any ldmsd or ldms_ls subsequently started. file_created - | + | | Verifies the existence and readability of filename. rollover_created - | + | | Verifies the existence and readability of rollover files matching pattern filename.[0-9]\*. bypass=<0,1> - | + | | This variable assignment disables (1) or enables (0) all the macros described above. Typical use is to skip one or more operations while debugging a test script. KILL_LDMSD_STRICT=<0,1> - | + | | This variable allows the script author to control whether KILL_LDMSD is strict by default or not. If enabled (1), the script will exit with error code 1 following a failed KILL_LDMSD. If @@ -131,7 +131,7 @@ KILL_LDMSD_STRICT=<0,1> the numeric arguments to KILL_LDMSD also sets KILL_LDMSD_STRICT=1. portbase= - | + | | The listening port numbers assigned to the daemons will be K+i, where i is as described for macro LDMSD. It is a good idea (to support automated testing) if portbase is set in so @@ -145,13 +145,13 @@ The LDMSD command supports the following options. Note that all -P options are processed before all -p options in a single LDMSD call. -p - | + | | The prolog file is included before the usually expected input file. The location of prolog files is handled as are the test input files. See FILES below. Multiple -p options are allowed. -P - | + | | The looped-prolog-file is included before the usually expected input file, once for each value in daemon-csl. Daemon-csl is a comma separated list of daemon numbers, e.g. a complete argument @@ -162,14 +162,14 @@ The location of looped prolog files is handled as are the test input files. See FILES below. Multiple -P options are allowed. -c - | + | | Where multiple daemon numbers are specified, the input generated for the first number is cloned to all subsequent daemons. See FILES. This allows a single file to serve many similar daemon instances in scale testing. -s - | + | | After an ldmsd is started, wait wait_microseconds before checking for the daemon PID file to exist. The appropriate wait time is variable depending on the complexity of the configuration. If not @@ -182,31 +182,31 @@ The following variables can be set in the script to affect the launch of ldmsd: LDMSD_EXTRA - | + | | If set, these arguments are are appended to the ldmsd launch. Typical use is to specify "-m MEMSIZE" or other unusual arguments. The following flags are always determined for the user and must not be present in LDMSD_EXTRA: -x -c -l -v -r. VG - | + | | If valgrind is used (see vgon, vgoff), then $VG is the name of the debugging tool wrapped around the launch of ldmsd. The default is 'valgrind'. VGARGS - | + | | If valgrind is used (see vgon, vgoff), then $VGARGS is appended to the default valgrind arguments. VGTAG - | + | | If valgrind is used (see vgon, vgoff), then $VGTAG is inserted in the valgrind output file name when defined. A good practice is for VGTAG to start with ".". KILL_NO_TEARDOWN - | + | | Set KILL_NO_TEARDOWN=1 to suppress attempting configuration cleanup during KILL_LDMSD. If set, ldmsd internal cleanup() function will attempt partial cleanup, but possibly leave active data structures @@ -214,52 +214,52 @@ KILL_NO_TEARDOWN the input file and the configuration file. i - | + | | Daemon configuration files and commands can refer to ${i} where i is the integer daemon number supplied via LDMSD for the specific daemon using the script. portN - | + | | Daemon configuration files and commands can refer to ${portN} where N is any value of 'i' described above. portN is the data port number of the N-th daemon. input - | + | | The name of the input file as specified when invoking this command. testname - | + | | The base name (directories stripped) of the input file name. This variable makes it possible to use similar input across many test files when the name of the input file is the same as the plugin tested. TESTDIR - | + | | Root directory of the testing setup. STOREDIR - | + | | A directory that should be used for store output configuration. LOGDIR - | + | | A directory that should be used for log outputs. LDMS_AUTH_FILE - | + | | Secret file used for daemon communication. XPRT - | + | | The transport used. It may be specified in the environment to override the default 'sock', and it is exported to the executed daemon environment. HOST - | + | | The host name used for a specific interface. It may be specified in the environment to override the default 'localhost', and it is exported to the executed daemon environment. @@ -281,7 +281,7 @@ FILES ================== *$input_file.$i* - | + | | For each value of i specifed to start an ldmsd, a configuration file named $input_file.$i must also exist. This configuration file is used when starting the daemon. @@ -293,12 +293,12 @@ the first are ignored. Where prologs are also specified, the regular prolog inclusion process is applied to the first file. *[test_dir]* - | + | | If test_dir is supplied, it is used as the test output directory. The default output location is \`pwd`/ldmstest/$testname. *$docdir/examples/static-test/$input_file* - | + | | If input_file is not found in the current directory, it is checked for in $docdir/examples/static-test/$input_file. @@ -311,35 +311,35 @@ GENERATED FILES for ldms_ls of the kth daemon with PID %p, if valgrind is active. *$test_dir/logs/$k.txt* - | + | | The log for the kth daemon. *$test_dir/logs/teardown.$k.txt* - | + | | The teardown log for the kth daemon. *$test_dir/run/conf.$k* - | + | | The input for the kth daemon. *$test_dir/run/revconf.$k* - | + | | The input for the kth daemon teardown. *$test_dir/run/env.$k* - | + | | The environment present for the kth daemon. *$test_dir/run/start.$k* - | + | | The start command of the kth daemon. *$test_dir/store/* - | + | | The root of store output locations. *$test_dir/run/ldmsd/secret* - | + | | The secret file for authentication. SEE ALSO diff --git a/rtd/docs/source/ldms_man/ldms_build_install.rst b/rtd/docs/source/ldms_man/ldms_build_install.rst index 0ede02ebd..e212f2ca7 100644 --- a/rtd/docs/source/ldms_man/ldms_build_install.rst +++ b/rtd/docs/source/ldms_man/ldms_build_install.rst @@ -94,20 +94,20 @@ A number of top level "enable|disable-feature" options exist. The defaults are chosen for a generic linux build to work by default. **--enable|disable-rpath** - | + | | Disable this. Do not hardcode runtime library paths. **--enable|disable-ldms** - | + | | Enable this. Default enabled. **--enable|disable-sos** - | + | | Used to enable or disable sos. Enable only if you are going to use the store_sos plugin. Default disable. **--enable|disable-ocm|baler|me|komondor** - | + | | Disable all of these. All default disabled. OVIS_LIB LEVEL OPTIONS @@ -117,19 +117,19 @@ A number of top level "enable|disable-feature" options exist. The defaults are chosen for a generic linux build to work by default. **--enable|disable-auth** - | + | | Enables or disables authentication. Default enabled. **--enable|disable-sock** - | + | | Enables or disables the sock transport. Default enabled. **--enable|disable-rdma** - | + | | Enables or disables the rdma transport. Default disabled **--enable|disable-ugni** - | + | | Enables or disables the ugni transport. The is cray-specific for rdma over gemini or aries. Default disabled. @@ -144,11 +144,11 @@ General Options --------------- **--enable|disable-ovis_auth** - | + | | If --enable, then disable/enable authentication. Default enabled. **--enable|disable-python** - | + | | Enable the ldms python api and the configuration tools that depend on the API. Default: enabled if python and cython detected. **--enable|disable-readline** @@ -158,41 +158,41 @@ General Options method), then this can be disabled. **--with-libevent**\ *[=path]* - | + | | Specify libevent path [default=/usr] Generic Sampler Options ----------------------- **--enable|disable-meminfo|procinterrupts|procnfs|procnetdev|vmstat** - | + | | Enable or disable generic linux samplers for data in /proc. Default enabled. **--enable|disable-lustre** - | + | | Enable or disable the lustre module. Default enabled. Cray-specific Sampler Options ----------------------------- **--enable|disable-kgnilnd** - | + | | Enable the kgnilnd sampler. Default disabled. **--enable|disable-cray_system_sampler** - | + | | Enable or disable the cray_system_sampler module. Default disabled. If you enable this, then consider the following options: **--enable-gemini-gpcdr** - | + | | Enable the gemini-gpcdr version of the cray_system_sampler. Default disabled. Both the gemini and aries versions can be built simultaneously. **--enable-aries-gpcdr** - | + | | Enable the aries-gpcdr version of the cray_system_sampler. Default disabled. For the Aries, we recommended getting the HSN metrics via aries-mmr, instead of the aries-gpcdr sampler. Still @@ -201,39 +201,39 @@ Cray-specific Sampler Options be built simultaneously. **--enable-cray-nvidia**\ OR\ **--with-cray-nvidia-inc**\ [=path] - | + | | For gemini systems with gpus, Enable the cray-nvidia metric sampling in the cray_gemini_r_sampler. You need not specify --enable-cray-nvidia if you are instead specifying the path to the include file via --with-cray-nvidia-inc. **--enable|disable-lustre** - | + | | Enable or disable the lustre module for use in the cray_system_sampler. Default enabled. **--with-rca**\ *[=path]* - | + | | Specify the path to the rca includes via --with-rca [default=/usr]. **--with-krca**\ *[=path]* - | + | | Specify the path to the krca includes via --with-krca [default=/usr]. **--with-cray-hss-devel**\ *[=path]* - | + | | Specify the path to the hss-devel includes via --with-cray-hss-devel [default=/usr]. **--enable|disable-aries-mmr** - | + | | Enable or disable the aries-mmr module. Default disabled. If you enable this, then consider the following options: **--with-aries-libgpcd**\ *LIBDIR,INCDIR* - | + | | Locations of gpcd library and headers for aries_mmr sampler. E.g. --with-aries-libgpcd=/special/libs,/private/headerdir @@ -241,7 +241,7 @@ Store Options ------------- **--enable|disable-csv** - | + | | Enable the csv stores (store_csv and store_function_csv). Default enable. **--enable|disable-sos** | Enable or disable the sos stores. Enable this only if you are going diff --git a/rtd/docs/source/ldms_man/ldms_ls.rst b/rtd/docs/source/ldms_man/ldms_ls.rst index e04c63011..39dfc5024 100644 --- a/rtd/docs/source/ldms_man/ldms_ls.rst +++ b/rtd/docs/source/ldms_man/ldms_ls.rst @@ -58,19 +58,19 @@ OPTIONS If the NAME is specified on the command line without -E/-S/-I, only information for that instance = NAME is displayed. **-E** *NAME* - | + | | Indicates that the NAME is a regular expression. **-S** *NAME* - | + | | Indicates that the NAME is a schema name. **-I** *NAME* - | + | | Indicates that the NAME is an instance name. This is the default. **-h** *HOST* - | + | | HOST to query. Default is localhost. **-x** *TRANSPORT* @@ -95,7 +95,7 @@ If the NAME is specified on the command line without -E/-S/-I, only information plugin manual for the option details. **-m** *MEMORY_SIZE* - | + | | MEMORY_SIZE is the size of memory reserved for metric sets. This value has precedence over the value of the LDMS_LS_MEM_SZ environment variable. The given size must be less than 1 petabytes. diff --git a/rtd/docs/source/ldms_man/ldms_sampler_base.rst b/rtd/docs/source/ldms_man/ldms_sampler_base.rst index aefcf2c1e..8af324bb7 100644 --- a/rtd/docs/source/ldms_man/ldms_sampler_base.rst +++ b/rtd/docs/source/ldms_man/ldms_sampler_base.rst @@ -41,54 +41,54 @@ CONFIGURATION ATTRIBUTE SYNTAX [component_id=] [schema=] [job_set= job_id= app_id= job_start= job_end=] -| +| | configuration line name= - | + | | This will be the name of the plugin being loaded. producer= - | + | | A unique name for the host providing the data. instance= - | + | | A unique name for the metric set. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. Defaults to the sampler name. component_id= - | + | | Optional unique number for the component being monitored, Defaults to zero. job_set= - | + | | The instance name of the set containing the job data, default is 'job_info'. job_id= - | + | | The name of the metric containing the Job Id, default is 'job_id'. app_id= - | + | | The name of the metric containing the Application Id, default is 'app_id'. job_start= - | + | | The name of the metric containing the Job start time, default is 'job_start'. job_end= - | + | | The name of the metric containing the Job end time, default is 'job_end'. diff --git a/rtd/docs/source/ldms_man/ldmsctl.rst b/rtd/docs/source/ldms_man/ldmsctl.rst index 5263501ca..fcb2d25d1 100644 --- a/rtd/docs/source/ldms_man/ldmsctl.rst +++ b/rtd/docs/source/ldms_man/ldmsctl.rst @@ -103,7 +103,7 @@ Load a plugin | **load** attr= **name** *name* - | + | | The plugin name List the usage of the loaded plugins @@ -117,7 +117,7 @@ unload a plugin | **term** attr= **name** *name* - | + | | The plugin name Send a configuration command to the specified plugin. @@ -126,51 +126,51 @@ Send a configuration command to the specified plugin. **config** attr= **name** *name* - | + | | The plugin name **attr=value** - | + | | Plugin specific attr=value tuples **Attributes specific for sampler plugins (Some sampler plugins may have additional** attributes) **producer** *producer* - | + | | A unique name for the host providing the data **instance** *instance* - | + | | The set instance name. The name must be unique among all metric sets in all LDMS daemons. **[component_id** *component_id*\ **]** - | + | | A unique number for the comopnent being monitored. The default is zero. **[schema** *schema*\ **]** - | + | | The name of the metric set schema. **[job_set** *job_set*\ **]** - | + | | The set instance name of the set containing the job data. The default is 'job_info'. **[uid** *uid*\ **]** - | + | | The user id of the set's owner. The default is the returned value of geteuid(). **[gid** *gid*\ **]** - | + | | The group id of the set's owner. The default is the returned value of getegid(). **[perm** *perm*\ **]** - | + | | The sampler plugin instance access permission. The default is 0440. @@ -180,15 +180,15 @@ Start a sampler plugin **start** attr= **name** *name* - | + | | The plugin name. **interval** *interval* - | + | | The sample interval in microseconds. **[offset** *offset*\ **]** - | + | | Offset (shift) from the sample mark in microseconds. Offset can be positive or negative with magnitude up to 1/2 the sample interval. If this offset is specified, including 0, collection @@ -201,7 +201,7 @@ Stop a sampler plugin **stop** attr= **name** *name* - | + | | The plugin name. PRODUCER COMMAND SYNTAX @@ -213,29 +213,29 @@ Add a producer to the aggregator | **prdcr_add** attr= **name** *name* - | + | | The producer name. The producer name must be unique in an aggregator. It is independent of any attributes specified for the metric sets or hosts. **xprt** *xprt* - | + | | The transport name [sock, rdma, ugni] **host** *host* - | + | | The hostname of the host **type** *conn_type* - | + | | The connection type [active, passive] **interval** *interval* - | + | | The connection retry interval **[perm** *permission*\ **]** - | + | | The permission to modify the producer in the future Delete a producer from the aggregator @@ -245,7 +245,7 @@ Delete a producer from the aggregator | **prdcr_del** attr= **name** *name* - | + | | The producer name Start a producer @@ -254,11 +254,11 @@ Start a producer **prdcr_start** attr= **name** *name* - | + | | The producer name **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -268,11 +268,11 @@ Start all producers matching a regular expression **prdcr_start_regex** attr= **regex** *regex* - | + | | A regular expression **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -282,7 +282,7 @@ Stop a producer **prdcr_stop** attr= **name** *name* - | + | | The producer name Stop all producers matching a regular expression @@ -291,7 +291,7 @@ Stop all producers matching a regular expression **prdcr_stop_regex** attr= **regex** *regex* - | + | | A regular expression Query producer status @@ -300,7 +300,7 @@ Query producer status **prdcr_status** attr= **[name** *name*\ **]** - | + | | The producer name. If none is given, the statuses of all producers are reported. @@ -310,11 +310,11 @@ Subscribe for stream data from all matching producers **prdcr_subsribe** **regex** *regex* - | + | | The regular expression matching producer name **stream** *stream* - | + | | The stream name UPDATER COMMAND SYNTAX @@ -326,21 +326,21 @@ Add an updater process that will periodically sample producer metric sets **updtr_add** attr= **name** *name* - | + | | The update policy name. The policy name should be unique. It is independent of any attributes specified for the metric sets or hosts. **interval** *interval* - | + | | The update/collect interval **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. **[push** *onchange|true*\ **]** - | + | | Push mode: 'onchange' and 'true'. 'onchange' means the Updater will get an update whenever the set source ends a transaction or pushes the update. 'true' means the Updater will receive an @@ -355,7 +355,7 @@ Add an updater process that will periodically sample producer metric sets *false*. **[perm** *permission*\ **]** - | + | | The permission to modify the updater in the future Remove an updater from the configuration @@ -364,7 +364,7 @@ Remove an updater from the configuration **updtr_del** attr= **name** *name* - | + | | The update policy name Add a match condition that specifies the sets to update. @@ -373,15 +373,15 @@ Add a match condition that specifies the sets to update. **updtr_match_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -392,15 +392,15 @@ Remove a match condition from the Updater. **updtr_match_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -413,11 +413,11 @@ This is required before starting the updater. **updtr_prdcr_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Remove matching producers to an updater policy @@ -426,11 +426,11 @@ Remove matching producers to an updater policy **updtr_prdcr_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Start updaters. @@ -439,16 +439,16 @@ Start updaters. **updtr_start** attr= **name** *name* - | + | | The update policy name **[interval** *interval*\ **]** - | + | | The update interval in micro-seconds. If this is not specified, the previously configured value will be used. Optional. **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. Stop an updater. @@ -459,7 +459,7 @@ The Updater must be stopped in order to change it's configuration. **updtr_stop** attr= **name** *name* - | + | | The update policy name Query the updater status @@ -468,7 +468,7 @@ Query the updater status **updtr_status** attr= **[name** *name*\ **]** - | + | | The updater name. If none is given, the statuses of all updaters are reported. @@ -481,23 +481,23 @@ Create a Storage Policy and open/create the storage instance. **strgp_add** attr= **name** *name* - | + | | The unique storage policy name. **plugin** *plugin* - | + | | The name of the storage backend. **container** *container* - | + | | The storage backend container name. **schema** *schema* - | + | | The schema name of the metric set to store. **[perm** *permission*\ **]** - | + | | The permission to modify the storage in the future Remove a Storage Policy @@ -508,7 +508,7 @@ Remove a Storage Policy | **strgp_del** attr= **name** *name* - | + | | The storage policy name Add a regular expression used to identify the producers this storage policy will apply to. @@ -519,11 +519,11 @@ Add a regular expression used to identify the producers this storage policy will | **strgp_prdcr_add** attr= **name** *name* - | + | | The storage policy name **regex** *name* - | + | | A regular expression matching metric set producers. Remove a regular expression from the producer match list @@ -535,7 +535,7 @@ Remove a regular expression from the producer match list | The storage policy name **regex** *regex* - | + | | The regex of the producer to remove. Add the name of a metric to store @@ -547,7 +547,7 @@ Add the name of a metric to store | The storage policy name **metric** *metric* - | + | | The metric name. If the metric list is NULL, all metrics in the metric set will be stored. @@ -560,7 +560,7 @@ Remove a metric from the set of stored metrics. | The storage policy name **metric** *metric* - | + | | The metric to remove Start a storage policy. @@ -587,7 +587,7 @@ Query the storage policy status **strgp_status** attr= **[name** *name*\ **]** - | + | | The storage policy name. If none is given, the statuses of all storage policies are reported. @@ -610,11 +610,11 @@ Publish data to the named stream **plublish** attr= **name** *name* - | + | | The stream name **data** *data* - | + | | The data to publish Subscribe to a stream @@ -623,7 +623,7 @@ Subscribe to a stream **subscribe** attr= **name** *name* - | + | | The stream name LDMS DAEMON COMMAND SYNTAX @@ -669,7 +669,7 @@ MISC COMMAND SYNTAX Display the list of available commands -------------------------------------- -| +| | **help** | [*command]* @@ -679,19 +679,19 @@ Display the list of available commands Set the user data value for a metric in a metric set. ----------------------------------------------------- -| +| | **udata** attr= **set** *set* - | + | | The sampler plugin name **metric** *metric* - | + | | The metric name **udata** *udata* - | + | | The desired user-data. This is a 64b unsigned integer. Set the user data of multiple metrics using regular expression. @@ -703,19 +703,19 @@ Set the user data of multiple metrics using regular expression. | **udata_regex** attr= **set** *set* - | + | | The metric set name. **regex** *regex* - | + | | A regular expression to match metric names to be set **base** *base* - | + | | The base value of user data (uint64) **[incr** *incr*\ **]** - | + | | Increment value (int). The default is 0. If incr is 0, the user data of all matched metrics are set to the base value. Optional. diff --git a/rtd/docs/source/ldms_man/ldmsd.rst b/rtd/docs/source/ldms_man/ldmsd.rst index b848acb5f..567b354aa 100644 --- a/rtd/docs/source/ldms_man/ldmsd.rst +++ b/rtd/docs/source/ldms_man/ldmsd.rst @@ -148,7 +148,7 @@ General/Configuration Options: configuration. See also REORDERED COMMANDS below. **-m, --set_memory** *MEMORY_SIZE* - | + | | MEMORY_SIZE is the maximum size of pre-allocated memory for metric sets. The given size must be less than 1 petabytes. For example, 20M or 20mb are 20 megabytes. The default is adequate for most @@ -160,7 +160,7 @@ General/Configuration Options: value at ldmsd exit. **-n, --daemon_name** *NAME* - | + | | The name of the daemon. By default, it is "*HOSTNAME:PORT*". The failover feature uses the daemon name to verify the buddy name, and the producer name of kernel metric sets is the daemon name. @@ -180,7 +180,7 @@ Communication Options: ---------------------- **-x** *XPRT:PORT:HOST* - | + | | Specifies the transport type to listen on. May be specified more than once for multiple transports. The XPRT string is one of 'rdma', 'sock', or 'ugni' (CRAY XE/XK/XC). A transport specific @@ -212,7 +212,7 @@ Log Verbosity Options: ---------------------- **-l, --log_file** *LOGFILE* - | + | | LOGFILE is the path to the log file for status messages. Default is stdout unless given. The syslog facility is used if LOGFILE is exactly "syslog". Silence can be obtained by specifying /dev/null @@ -220,7 +220,7 @@ Log Verbosity Options: below. **-v, --log_level** *LOG_LEVEL* - | + | | LOG_LEVEL can be one of DEBUG, INFO, ERROR, CRITICAL or QUIET. The default level is ERROR. QUIET produces only user-requested output. (Note: this has changed from the previous release where q @@ -230,7 +230,7 @@ Log Verbosity Options: Truncate the log file if it already exists. **-L,**\ *--log_config* ** \| \| ** - | + | | Append configuration replay messages or configuration debugging messages to the log indicated by -l (when PATH is omitted) or to the file named PATH. Bit values of CINT correspond to: @@ -265,7 +265,7 @@ Thread Options: --------------- **-P, --worker_threads** *THR_COUNT* - | + | | THR_COUNT is the number of event threads to start. SPECIFYING COMMAND-LINE OPTIONS IN CONFIGURATION FILES diff --git a/rtd/docs/source/ldms_man/ldmsd_controller.rst b/rtd/docs/source/ldms_man/ldmsd_controller.rst index e34e16015..8451c0775 100644 --- a/rtd/docs/source/ldms_man/ldmsd_controller.rst +++ b/rtd/docs/source/ldms_man/ldmsd_controller.rst @@ -59,7 +59,7 @@ LDMSD_CONTROLLER OPTIONS The LDMS authentication plugin. Please see **ldms_authentication**\ (7) for more information. -**-A,--auth-arg** *NAME=VALUE* +**-A,--auth-arg** *NAME=VALUE* Options *NAME*\ =\ *VALUE* Passing the *NAME*\ =\ *VALUE* option to the LDMS Authentication plugin. This command line option can be given multiple times. Please see **ldms_authentication**\ (7) for more @@ -67,11 +67,11 @@ LDMSD_CONTROLLER OPTIONS options. **--source** *SOURCE* - | + | | Path to the config file **--script** *SCRIPT* - | + | | Execute the script and send the output commands to the connected ldmsd @@ -98,7 +98,7 @@ Load a plugin | **load** attr= **name** *name* - | + | | The plugin name List the usage of the loaded plugins @@ -112,7 +112,7 @@ unload a plugin | **term** attr= **name** *name* - | + | | The plugin name Send a configuration command to the specified plugin. @@ -121,51 +121,51 @@ Send a configuration command to the specified plugin. **config** attr= **name** *name* - | + | | The plugin name **attr=value** - | + | | Plugin specific attr=value tuples **Attributes specific for sampler plugins (Some sampler plugins may have additional** attributes) **producer** *producer* - | + | | A unique name for the host providing the data **instance** *instance* - | + | | The set instance name. The name must be unique among all metric sets in all LDMS daemons. **[component_id** *component_id*\ **]** - | + | | A unique number for the comopnent being monitored. The default is zero. **[schema** *schema*\ **]** - | + | | The name of the metric set schema. **[job_set** *job_set*\ **]** - | + | | The set instance name of the set containing the job data. The default is 'job_info'. **[uid** *uid*\ **]** - | + | | The user id of the set's owner. The default is the returned value of geteuid(). **[gid** *gid*\ **]** - | + | | The group id of the set's owner. The default is the returned value of getegid(). **[perm** *perm*\ **]** - | + | | The sampler plugin instance access permission. The default is 0440. @@ -175,15 +175,15 @@ Start a sampler plugin **start** attr= **name** *name* - | + | | The plugin name. **interval** *interval* - | + | | The sample interval in microseconds. **[offset** *offset*\ **]** - | + | | Offset (shift) from the sample mark in microseconds. Offset can be positive or negative with magnitude up to 1/2 the sample interval. If this offset is specified, including 0, collection @@ -196,7 +196,7 @@ Stop a sampler plugin **stop** attr= **name** *name* - | + | | The plugin name. AUTHENTICATION COMMAND SYNTAX @@ -209,16 +209,16 @@ Add an authentication domain ATTRIBUTES* ... ] **name**\ =\ *NAME* - | + | | The name of the authentication domain. This is the name referred to by **listen** and **prdcr_add** commands. **plugin**\ =\ *none*\ \|\ *ovis*\ \|\ *munge* - | + | | The LDMS Authentication Plugin for this domain. [ ... *PLUGIN ATTRIBUTES* ... ] - | + | | Arbitrary plugin attributes. Please consult the manual of the authentication plugin for more information. @@ -233,22 +233,22 @@ Instruct ldmsd to listen to a port [**host**\ =\ *HOST*] [**auth**\ =\ *AUTH_REF*] **port**\ =\ *PORT* - | + | | The port to listen to. Also, please be sure not to use ephemeral port (ports in the range of **/proc/sys/net/ip4/ip_local_port_range**). **xprt**\ =\ *sock*\ \|\ *rdma*\ \|\ *ugni*\ \|\ *fabric* - | + | | The type of the transport. **host**\ =\ *HOST* - | + | | An optional hostname or IP address to bind. If not given, listen to all addresses (0.0.0.0 or PORT). **auth**\ =\ *AUTH_REF* - | + | | Instruct **ldmsd** to use *AUTH_REF* (a name reference to **auth** object created by **auth_add** command) to authenticate connections on this port. If not given, the port uses the @@ -264,33 +264,33 @@ Add a producer to the aggregator | **prdcr_add** attr= **name** *name* - | + | | The producer name. The producer name must be unique in an aggregator. It is independent of any attributes specified for the metric sets or hosts. **xprt** *xprt* - | + | | The transport name [sock, rdma, ugni] **host** *host* - | + | | The hostname of the host **type** *conn_type* - | + | | The connection type [active, passive] **interval** *interval* - | + | | The connection retry interval **[perm** *permission*\ **]** - | + | | The permission to modify the producer in the future **[auth** *AUTH_REF*\ **]** - | + | | Instruct **ldmsd** to use *AUTH_REF* (a name reference to **auth** object created by **auth_add** command) with the connections to this producer. If not given, the default @@ -304,7 +304,7 @@ Delete a producer from the aggregator | **prdcr_del** attr= **name** *name* - | + | | The producer name Start a producer @@ -313,11 +313,11 @@ Start a producer **prdcr_start** attr= **name** *name* - | + | | The producer name **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -327,11 +327,11 @@ Start all producers matching a regular expression **prdcr_start_regex** attr= **regex** *regex* - | + | | A regular expression **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -341,7 +341,7 @@ Stop a producer **prdcr_stop** attr= **name** *name* - | + | | The producer name Stop all producers matching a regular expression @@ -350,7 +350,7 @@ Stop all producers matching a regular expression **prdcr_stop_regex** attr= **regex** *regex* - | + | | A regular expression Query producer status @@ -359,7 +359,7 @@ Query producer status **prdcr_status** attr= **[name** *name*\ **]** - | + | | The producer name. If none is given, the statuses of all producers are reported. @@ -369,11 +369,11 @@ Subscribe for stream data from all matching producers **prdcr_subsribe** **regex** *regex* - | + | | The regular expression matching producer name **stream** *stream* - | + | | The stream name UPDATER COMMAND SYNTAX @@ -385,21 +385,21 @@ Add an updater process that will periodically sample producer metric sets **updtr_add** attr= **name** *name* - | + | | The update policy name. The policy name should be unique. It is independent of any attributes specified for the metric sets or hosts. **interval** *interval* - | + | | The update/collect interval **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. **[push** *onchange|true*\ **]** - | + | | Push mode: 'onchange' and 'true'. 'onchange' means the Updater will get an update whenever the set source ends a transaction or pushes the update. 'true' means the Updater will receive an @@ -414,7 +414,7 @@ Add an updater process that will periodically sample producer metric sets *false*. **[perm** *permission*\ **]** - | + | | The permission to modify the updater in the future Remove an updater from the configuration @@ -423,7 +423,7 @@ Remove an updater from the configuration **updtr_del** attr= **name** *name* - | + | | The update policy name Add a match condition that specifies the sets to update. @@ -432,15 +432,15 @@ Add a match condition that specifies the sets to update. **updtr_match_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -451,15 +451,15 @@ Remove a match condition from the Updater. **updtr_match_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -472,11 +472,11 @@ This is required before starting the updater. **updtr_prdcr_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Remove matching producers to an updater policy @@ -485,11 +485,11 @@ Remove matching producers to an updater policy **updtr_prdcr_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Start updaters. @@ -498,16 +498,16 @@ Start updaters. **updtr_start** attr= **name** *name* - | + | | The update policy name **[interval** *interval*\ **]** - | + | | The update interval in micro-seconds. If this is not specified, the previously configured value will be used. Optional. **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. Stop an updater. @@ -518,7 +518,7 @@ The Updater must be stopped in order to change it's configuration. **updtr_stop** attr= **name** *name* - | + | | The update policy name Query the updater status @@ -527,7 +527,7 @@ Query the updater status **updtr_status** attr= **[name** *name*\ **]** - | + | | The updater name. If none is given, the statuses of all updaters are reported. @@ -540,23 +540,23 @@ Create a Storage Policy and open/create the storage instance. **strgp_add** attr= **name** *name* - | + | | The unique storage policy name. **plugin** *plugin* - | + | | The name of the storage backend. **container** *container* - | + | | The storage backend container name. **schema** *schema* - | + | | The schema name of the metric set to store. **[perm** *permission*\ **]** - | + | | The permission to modify the storage in the future Remove a Storage Policy @@ -567,7 +567,7 @@ Remove a Storage Policy | **strgp_del** attr= **name** *name* - | + | | The storage policy name Add a regular expression used to identify the producers this storage policy will apply to. @@ -578,11 +578,11 @@ Add a regular expression used to identify the producers this storage policy will | **strgp_prdcr_add** attr= **name** *name* - | + | | The storage policy name **regex** *name* - | + | | A regular expression matching metric set producers. Remove a regular expression from the producer match list @@ -594,7 +594,7 @@ Remove a regular expression from the producer match list | The storage policy name **regex** *regex* - | + | | The regex of the producer to remove. Add the name of a metric to store @@ -606,7 +606,7 @@ Add the name of a metric to store | The storage policy name **metric** *metric* - | + | | The metric name. If the metric list is NULL, all metrics in the metric set will be stored. @@ -619,7 +619,7 @@ Remove a metric from the set of stored metrics. | The storage policy name **metric** *metric* - | + | | The metric to remove Start a storage policy. @@ -646,7 +646,7 @@ Query the storage policy status **strgp_status** attr= **[name** *name*\ **]** - | + | | The storage policy name. If none is given, the statuses of all storage policies are reported. @@ -669,11 +669,11 @@ Publish data to the named stream **plublish** attr= **name** *name* - | + | | The stream name **data** *data* - | + | | The data to publish Subscribe to a stream @@ -682,7 +682,7 @@ Subscribe to a stream **subscribe** attr= **name** *name* - | + | | The stream name LDMS DAEMON COMMAND SYNTAX @@ -728,7 +728,7 @@ MISC COMMAND SYNTAX Display the list of available commands -------------------------------------- -| +| | **help** | [*command]* @@ -738,19 +738,19 @@ Display the list of available commands Set the user data value for a metric in a metric set. ----------------------------------------------------- -| +| | **udata** attr= **set** *set* - | + | | The sampler plugin name **metric** *metric* - | + | | The metric name **udata** *udata* - | + | | The desired user-data. This is a 64b unsigned integer. Set the user data of multiple metrics using regular expression. @@ -762,19 +762,19 @@ Set the user data of multiple metrics using regular expression. | **udata_regex** attr= **set** *set* - | + | | The metric set name. **regex** *regex* - | + | | A regular expression to match metric names to be set **base** *base* - | + | | The base value of user data (uint64) **[incr** *incr*\ **]** - | + | | Increment value (int). The default is 0. If incr is 0, the user data of all matched metrics are set to the base value. Optional. diff --git a/rtd/docs/source/ldms_man/ldmsd_stream_publish.rst b/rtd/docs/source/ldms_man/ldmsd_stream_publish.rst index 9412e39a6..71055ad6a 100644 --- a/rtd/docs/source/ldms_man/ldmsd_stream_publish.rst +++ b/rtd/docs/source/ldms_man/ldmsd_stream_publish.rst @@ -31,45 +31,45 @@ COMMAND LINE SYNTAX ==================================== ldmsd_sstream_publish -x -h -p -s -a -A -t -f [-l] - | + | -x - | + | | transport of the ldmsd to which to connect. -p - | + | | port of the ldmsd to which to connect. -a - | + | | auth to connect to the ldmsd -A - | + | | auth-opts to connect to the ldmsd -s - | + | | Name of the stream (this will be used for subscribing) -t - | + | | Optional data-format. Either 'string' or 'json'. Default is string. -l - | + | | Optional line mode. Publishes file one line at a time as separate publish calls -f - | + | | File that is published. If not specified, input is copied from STDIN. -r N - | + | | Repeat the publication of the file N times, with a delay interval specifed by -i. Repeating is not supported unless the input is a file. If the -l option is given, the file and diff --git a/rtd/docs/source/ldms_man/ldmsd_stream_subscribe.rst b/rtd/docs/source/ldms_man/ldmsd_stream_subscribe.rst index 9d063fdef..5829c2c99 100644 --- a/rtd/docs/source/ldms_man/ldmsd_stream_subscribe.rst +++ b/rtd/docs/source/ldms_man/ldmsd_stream_subscribe.rst @@ -29,57 +29,57 @@ COMMAND LINE SYNTAX ====================================== ldmsd_stream_subscribe -x -h -p -s -a -A -f -D -i -R -q -E - | + | -x,--xprt - | + | | transport type on which to listen. -p,--port - | + | | port on which to listen. -h,--host - | + | | hostname or IP address of interface on which to listen. -a,--auth - | + | | authentication to expect from publishers. -A,--auth_arg - | + | | auth options if needed (for e.g. ovis auth or munge on unusual port) -s,--stream - | + | | Name of the stream to subscribe. -f,--file - | + | | File where messages delivered are written. If not specified, STDOUT. -E,--events-raw - | + | | Suppress delivery envelope information in message output. -q,--quiet - | + | | Suppress message output to file or stdout entirely. -D,--daemonize - | + | | Put the process in the background as a daemon. -R,--daemon-noroot - | + | | Prevent file system root (/) change-directory when starting the daemon. (Does nothing if -D is not present). -i,--daemon-io - | + | | Keep the input and output file descriptors attached to the daemon instead of closing them. (Does nothing if -D is not present). diff --git a/rtd/docs/source/ldmscon.rst b/rtd/docs/source/ldmscon.rst index 75b1855e2..dc06f17eb 100644 --- a/rtd/docs/source/ldmscon.rst +++ b/rtd/docs/source/ldmscon.rst @@ -4,7 +4,7 @@ The LDMS Users Group Conferences (LDMSCON) serves as a forum for users to share About ********** -You can find the general information and previous conferences following webpage: +You can find the general information and previous conferences following webpage: `LDMS Users Group Conference`_. .. _LDMS Users Group Conference: https://sites.google.com/view/ldmscon @@ -14,7 +14,7 @@ Please go to the to stay up to date on tutorials, presentations an LDMSCON2023 ************ -The following attachment contains the scripts and commands used in the LDMCON2023 Basics powerpoint presentation. +The following attachment contains the scripts and commands used in the LDMCON2023 Basics powerpoint presentation. **Please DOWNLOAD THE FOLLOWING .ZIP FILE to easily follow along with the tutorial.** @@ -31,7 +31,7 @@ Recordings of previous presentations, tutorials and information for LDMSCON2023 :width: 200 .. note:: - + **If the file directory ``ldmscon2023`` is not extracted under ``/root/``** then please keep in mind that **any reference to ``/root/``** in the powerpoint presentation, and following files, **will need to be changed to the absolute path of ``ldmscon2023/``**. * ``../conf/e3/agg_store_csv.conf`` @@ -41,9 +41,9 @@ Recordings of previous presentations, tutorials and information for LDMSCON2023 * ``../scripts/e3/store_csv.txt`` .. note:: - All files under ``../scripts/e*`` are not used in the tutorial but rather are the commands/steps used for each exercise. They demonstrate LDMS's ability to configure and initialize it's daemons with a single bash script. + All files under ``../scripts/e*`` are not used in the tutorial but rather are the commands/steps used for each exercise. They demonstrate LDMS's ability to configure and initialize it's daemons with a single bash script. -.. note:: +.. note:: These scripts must be ran in a directory that is readable and writable. Otherwise the log/data file generation will not work. LDMSCON2022 diff --git a/rtd/docs/source/sampler_man/Plugin_app_sampler.rst b/rtd/docs/source/sampler_man/Plugin_app_sampler.rst index 530ef4adc..79459414f 100644 --- a/rtd/docs/source/sampler_man/Plugin_app_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_app_sampler.rst @@ -8,12 +8,12 @@ Plugin_app_sampler :depth: 3 .. -NAME +NAME ==================== ldmsd_app_sampler - LDMSD app_sampler plugin -SYNOPSIS +SYNOPSIS ======================== **config** **name=app_sampler** **producer=**\ *PRODUCER* @@ -21,7 +21,7 @@ SYNOPSIS **component_id=\ COMPONENT_ID** ] [ **stream=\ STREAM_NAME** ] [ **metrics=\ METRICS** ] [ **cfg_file=\ PATH** ] -DESCRIPTION +DESCRIPTION =========================== **``app_sampler``** collects metrics from **``/proc/``** according @@ -38,34 +38,34 @@ metrics to monitor by specifying **``metrics``** option (comma-separated string) or writing a JSON configuration file and specifying **``cfg_file``** option (see **``EXAMPLES``** section). -CONFIG OPTIONS +CONFIG OPTIONS ============================== -name +name Must be app_sampler. -producer +producer The name of the data producer (e.g. hostname). -instance +instance This is required by sampler_base but is not used by app_sampler. So, this can be any string but must be present. -schema +schema The optional schema name (default: app_sampler). -component_id +component_id An integer identifying the component (default: *0*). -stream +stream The name of the **``ldmsd_stream``** to listen for SLURM job events. (default: slurm). -metrics +metrics The comma-separated list of metrics to monitor. The default is '' (empty), which is equivalent to monitor ALL metrics. -cfg_file +cfg_file The alternative config file in JSON format. The file is expected to have an object that may contain the following attributes: @@ -85,7 +85,7 @@ specified. Attributes other than 'stream' and 'metrics' are ignored. If the **``cfg_file``** is given, **``stream``** and **``metrics``** options are ignored. -LIST OF METRICS +LIST OF METRICS =============================== :: @@ -245,15 +245,15 @@ LIST OF METRICS /* /proc/[pid]/wchan */ wchan, -BUGS +BUGS ==================== No known bugs. -EXAMPLES +EXAMPLES ======================== -Example 1 +Example 1 ---------- Get everyting: @@ -262,7 +262,7 @@ Get everyting: config name=app_sampler -Example 2 +Example 2 ---------- Down-select and with non-default stream name: @@ -271,7 +271,7 @@ Down-select and with non-default stream name: config name=app_sampler metrics=stat_pid,stat_utime stream=mystream -Example 3 +Example 3 ---------- Down-select using config file, using default stream: diff --git a/rtd/docs/source/sampler_man/Plugin_aries_linkstatus.rst b/rtd/docs/source/sampler_man/Plugin_aries_linkstatus.rst index addcbf814..268ab7da8 100644 --- a/rtd/docs/source/sampler_man/Plugin_aries_linkstatus.rst +++ b/rtd/docs/source/sampler_man/Plugin_aries_linkstatus.rst @@ -66,25 +66,25 @@ attributes of the base class. | configuration line name= - | + | | aries_linkstatus file_send= - | + | | Location of the file with the sendlinkstatus metrics, as specified in the gpcdr configuration file. In the Cray-provided default gpcdr configuration, this will be /sys/devices/virtual/gni/gpcdr0/metricsets/linksendstatus/metrics. file_recv= - | + | | Location of the file with the recvlinkstatus metrics, as specified in the gpcdr configuration file. In the Cray-provided default gpcdr configuration, this will be /sys/devices/virtual/gni/gpcdr0/metricsets/linkrecvstatus/metrics. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. This will default to aries_linkstatus, if unspecified. diff --git a/rtd/docs/source/sampler_man/Plugin_aries_mmr.rst b/rtd/docs/source/sampler_man/Plugin_aries_mmr.rst index 65c3afa10..a16bd7d58 100644 --- a/rtd/docs/source/sampler_man/Plugin_aries_mmr.rst +++ b/rtd/docs/source/sampler_man/Plugin_aries_mmr.rst @@ -68,22 +68,22 @@ attributes of the base class. | configuration line name= - | + | | This MUST be aries_mmr, aries_nic_mmr, or aries_rtr_mmr. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. This will default to cray_aries_r or cray_gemini_r as appropriate, if unspecified. aries_rtr_id= - | + | | Optional aries router identifier. Defaults to 0 length string. file= - | + | | Configuration file of aries performance counter names that will be added in exactly as they are specified. At least one file must be specified. @@ -120,7 +120,7 @@ EXAMPLES | #RAW METRICS | AR_NIC_NETMON_ORB_EVENT_CNTR_REQ_FLITS -| +| | AR_NIC_RSPMON_NPT_EVENT_CNTR_NL_FLITS | # this is a test | AR_RTR_1_2_INQ_PRF_INCOMING_FLIT_VC0 diff --git a/rtd/docs/source/sampler_man/Plugin_aries_mmr_configurable.rst b/rtd/docs/source/sampler_man/Plugin_aries_mmr_configurable.rst index 7ed793d51..8a3c53112 100644 --- a/rtd/docs/source/sampler_man/Plugin_aries_mmr_configurable.rst +++ b/rtd/docs/source/sampler_man/Plugin_aries_mmr_configurable.rst @@ -102,11 +102,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be aries_mmr_configurable action= - | + | | Options are initialize, finalize, reset, and ls: **initialize** @@ -115,7 +115,7 @@ attributes of the base class. should be specified at this point. setfile= - | + | | Optional configuration file with the counter value assignment options. | Format: "name,type,default_value" one entry per line. @@ -133,7 +133,7 @@ attributes of the base class. These are printed out in the utility check_mmr_configurable. readfile= - | + | | Configuration file with the names of the counters to read. | Format "name,type" one entry per line. | Type is 'H' for Hex or anything else to default to uint64_t. @@ -149,28 +149,28 @@ attributes of the base class. These are printed out in the utility check_mmr_configurable. rtrid= - | + | | Optional unique rtr string identifier (e.g., c0-0c0s0a0). Defaults to 0 length string. schema= - | + | | Optional schema name. Defaults to 'aries_mmr_configurable'. **finalize** - | + | | Creates the mmr_contexts, sets the set counters to the configured values, and creates the set. Takes no arguments. If finalize fails, all state is cleared and the plugin can be configured again. **ls** - | + | | Prints out the set counter names and their configured values and also the read counter names. Takes no arguments. **reset** - | + | | Resets the set counters to their configured values. Takes no arguments. diff --git a/rtd/docs/source/sampler_man/Plugin_blob_stream_writer.rst b/rtd/docs/source/sampler_man/Plugin_blob_stream_writer.rst index 5308e9030..46f8c7475 100644 --- a/rtd/docs/source/sampler_man/Plugin_blob_stream_writer.rst +++ b/rtd/docs/source/sampler_man/Plugin_blob_stream_writer.rst @@ -38,28 +38,28 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be blob_stream_writer. path= - | + | | path to the directory of the output files container= - | + | | directory of the output file stream= - | + | | stream to which to subscribe. This argument may be repeated. Each stream will be written in a separate file pair. debug=1 - | + | | Enable logging of messages stored to the log file. timing=1 - | + | | Enable writing timestamps to a separate file. OUTPUT FORMAT diff --git a/rtd/docs/source/sampler_man/Plugin_clock.rst b/rtd/docs/source/sampler_man/Plugin_clock.rst index b1cf61767..f5af75541 100644 --- a/rtd/docs/source/sampler_man/Plugin_clock.rst +++ b/rtd/docs/source/sampler_man/Plugin_clock.rst @@ -39,11 +39,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be clock schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`clock\`. diff --git a/rtd/docs/source/sampler_man/Plugin_daos_sampler.rst b/rtd/docs/source/sampler_man/Plugin_daos_sampler.rst index 1931048ca..b8d742adb 100644 --- a/rtd/docs/source/sampler_man/Plugin_daos_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_daos_sampler.rst @@ -35,20 +35,20 @@ only covers the configuration attributes specific to this plugin; see ldms_sampler_base.man for the attributes of the base class. name= - | + | | This MUST be daos_sampler. producer=$HOSTNAME - | + | | The $HOSTNAME variable provides a good unique producer ID. engine_count=2 - | + | | The default is 2; don't change it unless the number of per-server engines is different. target_count=8 - | + | | The default is 8; don't change it unless the number of targets per engine is different. diff --git a/rtd/docs/source/sampler_man/Plugin_dcgm_sampler.rst b/rtd/docs/source/sampler_man/Plugin_dcgm_sampler.rst index 16dbe5e0a..5e2942fb9 100644 --- a/rtd/docs/source/sampler_man/Plugin_dcgm_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_dcgm_sampler.rst @@ -37,27 +37,27 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be dcgm_sampler. interval= - | + | | The sampling interval. This MUST be set to the same value that is set on the "start" line, otherwise behavior is undetermined. fields= - | + | | is a comma-separated list of integers representing DCGM field numebers that the plugin should watch. By default the plugin will watch fields 150,155. schema= - | + | | The schema name defaults to "dcgm", but it can be renamed at the user's choice. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) diff --git a/rtd/docs/source/sampler_man/Plugin_dstat.rst b/rtd/docs/source/sampler_man/Plugin_dstat.rst index dc872525d..c09096bcf 100644 --- a/rtd/docs/source/sampler_man/Plugin_dstat.rst +++ b/rtd/docs/source/sampler_man/Plugin_dstat.rst @@ -35,26 +35,26 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be dstat. producer= - | + | | The producer string value. instance= - | + | | The name of the metric set. schema= - | + | | Optional schema name. It is required by most storage backends that the same sampler on different nodes with different metric subsets needs to have a unique schema name. Use auto-schema=1 instead of schema to automatically meet the backend requirement. auto-schema= - | + | | If true, change the schema name to dstat\_$X, where $X will be a unique hex value derived from the data selection options. If both schema and auto-schema are given, for @@ -62,36 +62,36 @@ CONFIGURATION ATTRIBUTE SYNTAX plugin. component_id= - | + | | The component id numerical value. io= - | + | | Include the metrics from /proc/self/io. stat= - | + | | Include the metrics from /proc/self/stat. tick= - | + | | Include the sc_clk_tck from sysconf(3) as a metric. statm= - | + | | Include the metrics from /proc/self/statm. mmalloc= - | + | | Include the mmap memory usage metric from LDMS mmalloc. fd= - | + | | Include the number of open file descriptors found in /proc/self/fd. fdtypes= - | + | | Include the number and types of open file descriptors found in /proc/self/fd. This option may have high overhead on aggregators with many open connections. diff --git a/rtd/docs/source/sampler_man/Plugin_edac.rst b/rtd/docs/source/sampler_man/Plugin_edac.rst index 1e7d41867..e82811fe6 100644 --- a/rtd/docs/source/sampler_man/Plugin_edac.rst +++ b/rtd/docs/source/sampler_man/Plugin_edac.rst @@ -39,22 +39,22 @@ attributes of the base class. | configuration line name= - | + | | This MUST be edac. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to edac. max_mc= - | + | | The number of mc's in /sys/devices/system/edac/mc. Typically this number is 2. max_csrow= - | + | | The number of csrows in a single mc. For example, the value should be 4 for when the largest csrow looks like: /sys/devices/system/edac/mc/mc0/csrow3. Typically this number is diff --git a/rtd/docs/source/sampler_man/Plugin_filesingle.rst b/rtd/docs/source/sampler_man/Plugin_filesingle.rst index 783c394e5..73cd8f547 100644 --- a/rtd/docs/source/sampler_man/Plugin_filesingle.rst +++ b/rtd/docs/source/sampler_man/Plugin_filesingle.rst @@ -35,12 +35,12 @@ See ldms_sampler_base(7) for the common sampler options. | conf= [timing] conf= - | + | | File lines contain the source, type, and default value for each metric. See CONF FILE SYNTAX below. timing - | + | | If keyword 'timing' is included in the options, extra metrics measuring the time to collect every defined metric will be included. This allows for the discovery of slow sensors. Each diff --git a/rtd/docs/source/sampler_man/Plugin_fptrans.rst b/rtd/docs/source/sampler_man/Plugin_fptrans.rst index 1b76a97d7..84802c7ef 100644 --- a/rtd/docs/source/sampler_man/Plugin_fptrans.rst +++ b/rtd/docs/source/sampler_man/Plugin_fptrans.rst @@ -39,11 +39,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be fptrans. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, it will default to \`fptrans\`. diff --git a/rtd/docs/source/sampler_man/Plugin_geopm_sampler.rst b/rtd/docs/source/sampler_man/Plugin_geopm_sampler.rst index acdc35e8e..f9fec8474 100644 --- a/rtd/docs/source/sampler_man/Plugin_geopm_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_geopm_sampler.rst @@ -46,11 +46,11 @@ configured with the option - \``geopm_request_path=`\`. | configuration line name= - | + | | This MUST be ldms_geopm_sampler. geopm_request_path= - | + | | This parameter points to the absolute path of the ASCII file containing the list of signals that the user would like to have monitored by the sampler. @@ -69,7 +69,7 @@ configured with the option - \``geopm_request_path=`\`. than the number of available domains. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`ldms_geopm_sampler\`. diff --git a/rtd/docs/source/sampler_man/Plugin_hello_sampler.rst b/rtd/docs/source/sampler_man/Plugin_hello_sampler.rst index 3ea3d8732..24ff1ae8c 100644 --- a/rtd/docs/source/sampler_man/Plugin_hello_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_hello_sampler.rst @@ -41,11 +41,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be hello_sampler. stream= - | + | | Name of the stream to which to subscribe. BUGS diff --git a/rtd/docs/source/sampler_man/Plugin_ibmad_records_sampler.rst b/rtd/docs/source/sampler_man/Plugin_ibmad_records_sampler.rst index 83fecc763..959128b2b 100644 --- a/rtd/docs/source/sampler_man/Plugin_ibmad_records_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_ibmad_records_sampler.rst @@ -42,37 +42,37 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be ibmad_records_sampler. schema= - | + | | The schema name defaults to "ibmad", but it can be renamed at the user's choice. rate=0 - | + | | Stop the default inclusion of rate values in the set. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) include=PORTLIST - | + | | Ignore any devices and ports discovered that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the exclude option. exclude=PORTLIST - | + | | Collect all devices and ports discovered and active that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the include option. refresh_interval_sec= - | + | | (Optional) The sampler caches the list of infiniband devices, and that cache is refreshed at the beginning of a sample cycle if the refresh interval time has been exceeded. diff --git a/rtd/docs/source/sampler_man/Plugin_ibmad_sampler.rst b/rtd/docs/source/sampler_man/Plugin_ibmad_sampler.rst index 2882de6e7..d4ac045c3 100644 --- a/rtd/docs/source/sampler_man/Plugin_ibmad_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_ibmad_sampler.rst @@ -40,31 +40,31 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be ibmad_sampler. schema= - | + | | The schema name defaults to "ibmad_sampler", but it can be renamed at the user's choice. rate=0 - | + | | Stop the default inclusion of rate values in the set. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) include=PORTLIST - | + | | Ignore any devices and ports discovered that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the exclude option. exclude=PORTLIST - | + | | Collect all devices and ports discovered and active that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the include option. diff --git a/rtd/docs/source/sampler_man/Plugin_ibnet.rst b/rtd/docs/source/sampler_man/Plugin_ibnet.rst index c00d6c901..0bfe0c6c8 100644 --- a/rtd/docs/source/sampler_man/Plugin_ibnet.rst +++ b/rtd/docs/source/sampler_man/Plugin_ibnet.rst @@ -41,64 +41,64 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be ibnet. producer=. - | + | | The producer string value for the timing set. Default is the result of gethostname(). instance= - | + | | The name of the timing metric set. Default is $producer/ibnet_timing. source-list= - | + | | Lidfile is the name of a file of LID/port specifications. See PORT FILE for format details. port-name= [port-number=] - | + | | Hca is the name of the local IB interface to access the network. Num is the number of the port on the interface used to access the network. The default is 1. schema= - | + | | Optional schema base name. The default is ibnet. The name base is suffixed to create uniquely defined schema names based on the plugin options specified. component_id= - | + | | Optional component identifier for the timing set. Defaults to zero. metric-conf= - | + | | The file listing the metric groups to collect. See METRIC GROUPS below. ca_port= - | + | | The port number to use, which must be active. millis= - | + | | The number of milliseconds of the timeout on the MAD calls. Default 0, which will use the mad library timeout of 1 second. timing= - | + | | Disable timing (T=0), enable aggregate timing (T=1), or enable individual port timing(T=2) or enable port offset timing(T=3). The metric set will contain sampling process timing metrics if T > 0. node-name-map= - | + | | The file name nnmap, as used by ibnetdiscover and opensm, of a mapping from IB GUIDs to short names of IB hardware items (switch, node, etc) suitable for use in populating names of @@ -113,7 +113,7 @@ The lid/port file format is lid, hexguid, nports, plist * where hexguid is 0x...., - * nports is int, + * nports is int, * plist is ints nports long or * if range is 1-nports, * if not using a name map, names will be GUID_hex. @@ -139,20 +139,20 @@ INTERNAL METRICS ========================= port_query_time - | + | | Time in seconds spend in the single port MAD call. port_query_offset - | + | | Time in microseconds from start of all MAD calls in the current update to the end of the mad call for the specific port. ib_query_time - | + | | Time in seconds making all MAD calls in the update. ib_data_process_time - | + | | Time in seconds decoding all MAD data in the update BUGS diff --git a/rtd/docs/source/sampler_man/Plugin_ipmireader.rst b/rtd/docs/source/sampler_man/Plugin_ipmireader.rst index 6cf0e9233..a7c4cd271 100644 --- a/rtd/docs/source/sampler_man/Plugin_ipmireader.rst +++ b/rtd/docs/source/sampler_man/Plugin_ipmireader.rst @@ -42,35 +42,35 @@ attributes of the base class. | configuration line name= - | + | | This MUST be ipmireader. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`ipmireader\`. address=
- | + | | address of the host to contact. H flag in the ipmitool command. username= - | + | | username for the query. U flag in the ipmitool command. Defaults to 'admin'. password= - | + | | password for the query. P flag in the ipmitool command. Defaults to 'password'. sdrcache= - | + | | output for the sdr cache file, to improve performance. Optional. retry= - | + | | interval to retry creating set if initially fails (host down). Default 600 sec. diff --git a/rtd/docs/source/sampler_man/Plugin_ipmisensors.rst b/rtd/docs/source/sampler_man/Plugin_ipmisensors.rst index c885695c6..58d7c2a35 100644 --- a/rtd/docs/source/sampler_man/Plugin_ipmisensors.rst +++ b/rtd/docs/source/sampler_man/Plugin_ipmisensors.rst @@ -43,27 +43,27 @@ attributes of the base class. | configuration line name= - | + | | This MUST be ipmisensors. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`ipmisensors\`. address=
- | + | | address of the host to contact. h flag in the ipmi-sensors command. username= - | + | | username for the query. u flag in the ipmi-sensors command. Defaults to 'admin'. password= - | + | | password for the query. p flag in the ipmi-sensors command. Defaults to 'password'. diff --git a/rtd/docs/source/sampler_man/Plugin_kgnilnd.rst b/rtd/docs/source/sampler_man/Plugin_kgnilnd.rst index 5a1b2fd25..a238a8a00 100644 --- a/rtd/docs/source/sampler_man/Plugin_kgnilnd.rst +++ b/rtd/docs/source/sampler_man/Plugin_kgnilnd.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be kgnilnd. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`kgnilnd\`. diff --git a/rtd/docs/source/sampler_man/Plugin_linux_proc_sampler.rst b/rtd/docs/source/sampler_man/Plugin_linux_proc_sampler.rst index 9185494e9..9e2cbd280 100644 --- a/rtd/docs/source/sampler_man/Plugin_linux_proc_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_linux_proc_sampler.rst @@ -48,51 +48,51 @@ attributes of the base class. | configuration line name= - | + | | This MUST be linux_proc_sampler. instance_prefix=PREFIX - | + | | Prepend PREFIX to the set instance names. Typically a cluster name when needed to disambiguate producer names that appear in multiple clusters. (default: no prefix). exe_suffix=1 - | + | | If present, set instance names are appended with the full path of the executable. This is most likely useful for debugging configuration of the notifier up-stream using ldms_ls. (default: no such suffix) sc_clk_tck=1 - | + | | If present, include sc_clk_tck in the metric set. sc_clk_tck is the ticks per second from sysconf(\_SC_CLK_TCK). (default: not included). stream=STREAM - | + | | The name of the \`ldmsd_stream\` to listen for SLURM job events. (default: slurm). argv_sep= - | + | | Replace nul within the cmdline string with char. Special specifiers \\b,\\n,\\t,\\v,\\r,\\f are also supported. syscalls=FILE - | + | | File mapping syscall integers to symbolic names. Not needed unless syscall_name is included in the metrics. See FILES for details. metrics - | + | | The comma-separated list of metrics to monitor. The default is (empty), which is equivalent to monitor ALL metrics. cfg_file=CFILE - | + | | The alternative configuration file in JSON format. The file is expected to have an object that contains the following attributes: { "stream": "STREAM_NAME", "syscalls" : "/file", @@ -101,7 +101,7 @@ attributes of the base class. on the key=value line are ignored. argv_msg=1 - | + | | Publish the argv items to a stream named _argv, where if the schema is not specified, the default schema is linux_proc_sampler. (Default: argv_msg=0; no publication of @@ -110,13 +110,13 @@ attributes of the base class. store them. argv_fmt=<1,2> - | + | | Publish the argv items formatted as (1) a json list of strings ['argv0', 'argv1'] or (2) a json list of key/value tuples, e.g. [ {"k":0, "v":"argv[0]"}, {"k":1, "v":"argv[1]"}]. env_msg=1 - | + | | Publish the environment items to a stream named _env, where if the schema is not specified, the default SCHEMA is linux_proc_sampler. (Default: env_msg=0; no publication of the @@ -126,7 +126,7 @@ attributes of the base class. messages and store them. env_exclude=ELIST - | + | | Exclude the environment items named with regular expressions in ELIST. On the configuration key=value line, ELIST must be a file name of a file containing a list of regular expressions one per @@ -137,7 +137,7 @@ attributes of the base class. shown in EXAMPLES. fd_exclude=ELIST - | + | | Exclude the files named with regular expressions in ELIST. On the configuration key=value line, ELIST must be a file name of a file containing a list of regular expressions one per line. A @@ -147,7 +147,7 @@ attributes of the base class. array of expression strings as shown in EXAMPLES. fd_msg=N - | + | | Publish new /proc/pid/fd scan data to the _files stream every N-th sample, where if the schema is not specified, the default SCHEMA is linux_proc_sampler. (Default: fd_msg=0; no @@ -165,7 +165,7 @@ attributes of the base class. no corresponding events are generated. published_pid_dir= - | + | | Name of the directory where netlink-notifier or other notifier pids of interest may be found. This directory is scanned at sampler startup only, so that pids which were the subject of diff --git a/rtd/docs/source/sampler_man/Plugin_lnet_stats.rst b/rtd/docs/source/sampler_man/Plugin_lnet_stats.rst index 4de54af8e..949907086 100644 --- a/rtd/docs/source/sampler_man/Plugin_lnet_stats.rst +++ b/rtd/docs/source/sampler_man/Plugin_lnet_stats.rst @@ -40,17 +40,17 @@ attributes of the base class. | ldmsctl configuration line. name= - | + | | This MUST be lnet_stats. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`lnet_stats\`. file= - | + | | Optional full path name of stats file to use. If not supplied, the default search path described in NOTES is searched. Typically, this option is only used in test environments which diff --git a/rtd/docs/source/sampler_man/Plugin_loadavg.rst b/rtd/docs/source/sampler_man/Plugin_loadavg.rst index e5735364d..d51797b02 100644 --- a/rtd/docs/source/sampler_man/Plugin_loadavg.rst +++ b/rtd/docs/source/sampler_man/Plugin_loadavg.rst @@ -36,16 +36,16 @@ base class. name= [schema=] [metrics=] [force_integer] name= - | + | | This MUST be loadavg. force_integer - | + | | If present, this flag forces load metrics to be stored as integers of 100*value provided in the proc file. schema= - | + | | Optional schema name. If schema is not specified, it will be computed. The default name is loadavg if the metrics option is not supplied. The default name when metrics is specified is @@ -54,7 +54,7 @@ base class. loadavg prefix becomes loadavgi. metrics= - | + | | comma separated list of metrics to include. If not given, all are included. The complete list is load1min, load5min, load15min, runnable, scheduling_entities, newest_pid. diff --git a/rtd/docs/source/sampler_man/Plugin_lustre_client.rst b/rtd/docs/source/sampler_man/Plugin_lustre_client.rst index bc8bfd6c2..341c4ea55 100644 --- a/rtd/docs/source/sampler_man/Plugin_lustre_client.rst +++ b/rtd/docs/source/sampler_man/Plugin_lustre_client.rst @@ -46,27 +46,27 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_client. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be $producer/$llite_name. component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. perm= - | + | | Set the access permissions for the metric sets. (default 440). NOTES diff --git a/rtd/docs/source/sampler_man/Plugin_lustre_mdc.rst b/rtd/docs/source/sampler_man/Plugin_lustre_mdc.rst index 78242eb12..7a783a443 100644 --- a/rtd/docs/source/sampler_man/Plugin_lustre_mdc.rst +++ b/rtd/docs/source/sampler_man/Plugin_lustre_mdc.rst @@ -42,28 +42,28 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_mdc. producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be / component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. job_set= - | + | | Optional (defaults to "job_info"). Typically should be set to /jobid or /job_info depending on choice of job sampling plugin. mdc_timing=0 - | + | | Optionally exclude timing data from /sys/kernel/debug/lustre/mdc/\*/stats. If given, the sampler may be run by unprivileged users. If /sys/kernel/debug/ cannot be @@ -71,7 +71,7 @@ CONFIGURATION ATTRIBUTE SYNTAX mdc_timing=0 is given. auto_reset=0 - | + | | Turn off the default behavior of resetting the counters when an overflow condition is detected. Reset is implemented by writing 0 to the corresponding /proc or /sys file. diff --git a/rtd/docs/source/sampler_man/Plugin_lustre_mdt.rst b/rtd/docs/source/sampler_man/Plugin_lustre_mdt.rst index 40052185f..6c1355d51 100644 --- a/rtd/docs/source/sampler_man/Plugin_lustre_mdt.rst +++ b/rtd/docs/source/sampler_man/Plugin_lustre_mdt.rst @@ -55,17 +55,17 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_mdt. producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be $producer/$mdt_name. component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. diff --git a/rtd/docs/source/sampler_man/Plugin_lustre_ost.rst b/rtd/docs/source/sampler_man/Plugin_lustre_ost.rst index 48d2f46fe..3cd09b493 100644 --- a/rtd/docs/source/sampler_man/Plugin_lustre_ost.rst +++ b/rtd/docs/source/sampler_man/Plugin_lustre_ost.rst @@ -55,17 +55,17 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_ost. producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be $producer/$ost_name. component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. diff --git a/rtd/docs/source/sampler_man/Plugin_meminfo.rst b/rtd/docs/source/sampler_man/Plugin_meminfo.rst index 5b881c215..1cf06fd6d 100644 --- a/rtd/docs/source/sampler_man/Plugin_meminfo.rst +++ b/rtd/docs/source/sampler_man/Plugin_meminfo.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be meminfo. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`meminfo\`. diff --git a/rtd/docs/source/sampler_man/Plugin_msr_interlagos.rst b/rtd/docs/source/sampler_man/Plugin_msr_interlagos.rst index d91642795..d596b0171 100644 --- a/rtd/docs/source/sampler_man/Plugin_msr_interlagos.rst +++ b/rtd/docs/source/sampler_man/Plugin_msr_interlagos.rst @@ -288,11 +288,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be msr_interlagos action= - | + | | Options are initialize, add, finalize, halt, continue, reassign, rewrite, and ls: @@ -303,12 +303,12 @@ attributes of the base class. should be specified at this point. corespernuma= - | + | | Cores per numa node. Used to determine which and how many cores are used in setting counters that report per numa node. maxcore= - | + | | Maxcores that will be reported for all core counters and will also be used in counters that report per numa node. Must be >= actual number of cores. Any additional values will be @@ -316,7 +316,7 @@ attributes of the base class. actual number of cores. schema= - | + | | Schema name. Optional. Defaults to msr_interlagos. **add** @@ -325,12 +325,12 @@ attributes of the base class. the order the metrics are added metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. Options are listed in a separate section of this man page. **finalize** - | + | | creates the set after all the adds. No metrics may be added after this point. @@ -340,7 +340,7 @@ attributes of the base class. for all metrics for this counter. metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. metricname=all halts all. @@ -349,7 +349,7 @@ attributes of the base class. | continues collection for this counter after a halt. metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. metricname=all continues all. @@ -359,7 +359,7 @@ attributes of the base class. has been changed for this address external to ldms. metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. metricname=all rewrites all counters. @@ -370,16 +370,16 @@ attributes of the base class. counter. oldmetricname= - | + | | The name of counter to be replaced e.g., TOT_CYC newmetricname= - | + | | The name of counter that the previous variable will be replaced with e.g., TOT_INS **ls** - | + | | writes info about the intended counters to the log file. BUGS diff --git a/rtd/docs/source/sampler_man/Plugin_opa2.rst b/rtd/docs/source/sampler_man/Plugin_opa2.rst index 9b7210b1f..f8139f351 100644 --- a/rtd/docs/source/sampler_man/Plugin_opa2.rst +++ b/rtd/docs/source/sampler_man/Plugin_opa2.rst @@ -35,29 +35,29 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be opa2. producer= - | + | | The producer string value. instance= - | + | | The set_name supplied is ignored, and the name $producer/$CA/$port is used. schema= - | + | | Optional schema name. Default opa2. The same schema is used for all sets. component_id= - | + | | Optional component identifier. Defaults to zero. ports= - | + | | Port list is a comma separated list of ca_name.portnum or a '\*'. The default is '\*', which collects a set for every host fabric interface port. diff --git a/rtd/docs/source/sampler_man/Plugin_perfevent.rst b/rtd/docs/source/sampler_man/Plugin_perfevent.rst index c0d97196c..8670122c9 100644 --- a/rtd/docs/source/sampler_man/Plugin_perfevent.rst +++ b/rtd/docs/source/sampler_man/Plugin_perfevent.rst @@ -49,28 +49,28 @@ the 'init' action arguments. | configuration line name= - | + | | This MUST be perfevent. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. action=init - | + | | Perform initialization action=del metricname= - | + | | Deletes the specified event. action=ls - | + | | List the currently configured events. action=add metricname= pid= cpu= type= id= - | + | | Adds a metric to the list of configured events. | metricname | The metric name for the event @@ -88,7 +88,7 @@ the 'init' action arguments. | The event id. The pid and cpu arguments allow specifying which process and CPU to monitor: - | + | | pid == 0 and cpu == -1 | This measures the calling process/thread on any CPU. | pid == 0 and cpu >= 0 @@ -109,7 +109,7 @@ the 'init' action arguments. For more information visit: http://man7.org/linux/man-pages/man2/perf_event_open.2.html **type** - | + | | This field specifies the overall event type. It has one of the following values: | PERF_TYPE_HARDWARE @@ -133,7 +133,7 @@ the 'init' action arguments. execution of an instruction address. **id** - | + | | This specifies which event you want, in conjunction with the type field. | There are various ways to set the id field that are dependent on @@ -315,7 +315,7 @@ EXAMPLES ===================== The following is a short example that measures 4 events. - | + | | Total CPU cycles | Total CPU instructions | Total branch instructions diff --git a/rtd/docs/source/sampler_man/Plugin_procdiskstats.rst b/rtd/docs/source/sampler_man/Plugin_procdiskstats.rst index 1d30a6d2a..97185aad6 100644 --- a/rtd/docs/source/sampler_man/Plugin_procdiskstats.rst +++ b/rtd/docs/source/sampler_man/Plugin_procdiskstats.rst @@ -41,17 +41,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procdiskstats. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procdiskstats\`. device= - | + | | Comma separated list of devices BUGS diff --git a/rtd/docs/source/sampler_man/Plugin_procinterrupts.rst b/rtd/docs/source/sampler_man/Plugin_procinterrupts.rst index c03eb33ef..d8aa92f2e 100644 --- a/rtd/docs/source/sampler_man/Plugin_procinterrupts.rst +++ b/rtd/docs/source/sampler_man/Plugin_procinterrupts.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procinterrupts. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procinterrupts\`. diff --git a/rtd/docs/source/sampler_man/Plugin_procnet.rst b/rtd/docs/source/sampler_man/Plugin_procnet.rst index 3e70b7a64..eb9b758d0 100644 --- a/rtd/docs/source/sampler_man/Plugin_procnet.rst +++ b/rtd/docs/source/sampler_man/Plugin_procnet.rst @@ -42,15 +42,15 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnet. exclude_ports= - | + | | Comma separated list of ports to exclude. schema= - | + | | Optional schema name. If not specified, will default to \`procnet\`. diff --git a/rtd/docs/source/sampler_man/Plugin_procnetdev.rst b/rtd/docs/source/sampler_man/Plugin_procnetdev.rst index 1fbe74f06..b0a95b81c 100644 --- a/rtd/docs/source/sampler_man/Plugin_procnetdev.rst +++ b/rtd/docs/source/sampler_man/Plugin_procnetdev.rst @@ -40,16 +40,16 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnetdev. ifaces= - | + | | CSV list of ifaces. Order matters. Non-existent ifaces will be included and default to 0-value data. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics or ifaces have a different schema. If not specified, will default to diff --git a/rtd/docs/source/sampler_man/Plugin_procnetdev2.rst b/rtd/docs/source/sampler_man/Plugin_procnetdev2.rst index 0b55a2b8c..37f39f743 100644 --- a/rtd/docs/source/sampler_man/Plugin_procnetdev2.rst +++ b/rtd/docs/source/sampler_man/Plugin_procnetdev2.rst @@ -40,17 +40,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnetdev2. ifaces= - | + | | (Optional) A CSV list of interfaces to sample. If not specified, all available interfaces in /proc/net/dev will be reported. It is OK to specify non-existing interfaces in the ifaces list. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics or ifaces have a different schema. If not specified, will default to diff --git a/rtd/docs/source/sampler_man/Plugin_procnfs.rst b/rtd/docs/source/sampler_man/Plugin_procnfs.rst index 4ae8b96b1..49bc21ea3 100644 --- a/rtd/docs/source/sampler_man/Plugin_procnfs.rst +++ b/rtd/docs/source/sampler_man/Plugin_procnfs.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnfs. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procnfs\`. diff --git a/rtd/docs/source/sampler_man/Plugin_procstat.rst b/rtd/docs/source/sampler_man/Plugin_procstat.rst index 3416980f4..1c49164e7 100644 --- a/rtd/docs/source/sampler_man/Plugin_procstat.rst +++ b/rtd/docs/source/sampler_man/Plugin_procstat.rst @@ -39,7 +39,7 @@ See ldms_sampler_base(7) for the common sampler options. | configuration line maxcpu= - | + | | Values are 0 to N, where 0 logs only totalized data and N reserves slots for N cores. If less than N cores are found, 0-values are reported. If more than N cores are found, they are @@ -50,7 +50,7 @@ See ldms_sampler_base(7) for the common sampler options. anywhere in the cluster. sc_clk_tck=1 - | + | | Enable optional reporting of sysconf(\_SC_CLK_TCK), the scheduler ticks-per-second defined at kernel build time as CONFIG_HZ, collected from sysconf(3). Typically HPC systems use diff --git a/rtd/docs/source/sampler_man/Plugin_procstat2.rst b/rtd/docs/source/sampler_man/Plugin_procstat2.rst index 95cdfbd8f..88f6896a4 100644 --- a/rtd/docs/source/sampler_man/Plugin_procstat2.rst +++ b/rtd/docs/source/sampler_man/Plugin_procstat2.rst @@ -39,17 +39,17 @@ of the base class. | configuration line name= - | + | | This MUST be procstat2. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procstat2\`. intr_max= - | + | | (Optional). The maximum number of inerrupt numbers supported in intr_list. If not specified, intr_max will be the current number of interrupts in the intr list. diff --git a/rtd/docs/source/sampler_man/Plugin_rdc_sampler.rst b/rtd/docs/source/sampler_man/Plugin_rdc_sampler.rst index 1fe134eed..8a25f553f 100644 --- a/rtd/docs/source/sampler_man/Plugin_rdc_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_rdc_sampler.rst @@ -37,48 +37,48 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be rdc_sampler. producer=. - | + | | The producer string value for the timing set. instance= - | + | | The set instance names will be suffixed by device number (gpu%d). schema= - | + | | Optional schema base name. The default is rdc_sampler. The name base is suffixed to create uniquely defined schema names based on the plugin options specified. component_id= - | + | | Optional component identifier for the timing set. Defaults to zero. metrics=LIST - | + | | The list of values to be collected as named in rdc_field_t from rdc/rdc.h. update_freq=MICROSEC - | + | | An argument passed to rdc_field_watch. max_keep_age=SEC - | + | | An argument passed to rdc_field_watch. max_keep_samples=N - | + | | An argument passed to rdc_field_watch. warmup=K - | + | | Delay K cycles update_freq long before attempting to read data from the gpu. diff --git a/rtd/docs/source/sampler_man/Plugin_sampler_atasmart.rst b/rtd/docs/source/sampler_man/Plugin_sampler_atasmart.rst index 75d05e933..d0eb73577 100644 --- a/rtd/docs/source/sampler_man/Plugin_sampler_atasmart.rst +++ b/rtd/docs/source/sampler_man/Plugin_sampler_atasmart.rst @@ -47,17 +47,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be sampler_atasmart. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`sampler_atasmart\`. disks - | + | | A comma-separated list of disk names (e.g., /dev/sda,/dev/sda1) BUGS diff --git a/rtd/docs/source/sampler_man/Plugin_slingshot_info.rst b/rtd/docs/source/sampler_man/Plugin_slingshot_info.rst index c93f779d8..804b73ae5 100644 --- a/rtd/docs/source/sampler_man/Plugin_slingshot_info.rst +++ b/rtd/docs/source/sampler_man/Plugin_slingshot_info.rst @@ -50,7 +50,7 @@ attributes of the base class. | configuration line name= - | + | | This MUST be slingshot_info. EXAMPLES diff --git a/rtd/docs/source/sampler_man/Plugin_slingshot_metrics.rst b/rtd/docs/source/sampler_man/Plugin_slingshot_metrics.rst index efbdc6f52..9bd5c2577 100644 --- a/rtd/docs/source/sampler_man/Plugin_slingshot_metrics.rst +++ b/rtd/docs/source/sampler_man/Plugin_slingshot_metrics.rst @@ -48,18 +48,18 @@ attributes of the base class. | configuration line name= - | + | | This MUST be slingshot_metrics. counters= - | + | | (Optional) A CSV list of names of slingshot counter names. See Section COUTNER NAMES for details. If neither this option nor counters_file are specified, a default set of counters will be used. counters_files= - | + | | (Optional) A path to a file that contains a list of counter names, one per line. See Section COUNTER NAMES for details. A line will be consider a comment if the character on the line is @@ -67,7 +67,7 @@ attributes of the base class. default set of counters will be used. refresh_interval_sec= - | + | | (Optional) The sampler caches the list of slinghost devices, and that cache is refreshed at the beginning of a sample cycle if the refresh interval time has been exceeded. diff --git a/rtd/docs/source/sampler_man/Plugin_sysclassib.rst b/rtd/docs/source/sampler_man/Plugin_sysclassib.rst index 31725181a..d93c04b0b 100644 --- a/rtd/docs/source/sampler_man/Plugin_sysclassib.rst +++ b/rtd/docs/source/sampler_man/Plugin_sysclassib.rst @@ -34,20 +34,20 @@ specific to the this plugin; see ldms_sampler_base.man for the attributes of the base class. **config**\ name=\ **[schema=]**\ ports=\ **[metrics_type=]** - | + | | configuration line name= - | + | | This MUST be sysclassib. metrics_type= - | + | | Values are 0 or 1. 0 = counter data only. 1 = include rate data (per second) in addition. Default is 0. ports= - | + | | CSV list of the form CARD1.PORT1,CARD2.PORT2. Default is all discovered values. diff --git a/rtd/docs/source/sampler_man/Plugin_tutorial_sampler.rst b/rtd/docs/source/sampler_man/Plugin_tutorial_sampler.rst index c549bb570..2a0188d7a 100644 --- a/rtd/docs/source/sampler_man/Plugin_tutorial_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_tutorial_sampler.rst @@ -45,18 +45,18 @@ attributes of the base class. | configuration line name= - | + | | This MUST be tutorial_sampler. num_metrics= - | + | | Optional number of metrics for this set. Metrics will be U64. Metric names will be 'metric\_%d'. If not specified, default number of metrics is determined by DEFAULTNUMMETRICS in the source. schema= - | + | | Optional schema name. It is intended that any sets with different metrics have a different schema. If not specified, will default to \`tutorial_sampler\`. Therefore, if you are @@ -76,10 +76,10 @@ Within ldmsd_controller or a configuration file: :: config name=tutorial_sampler producer=localhost1 instance=localhost1/test1 schema=test1 component_id=1 - config name=tutorial_sampler producer=localhost1 instance=localhost1/test2 schema=test2 component_id=2 num_metrics=5 - config name=tutorial_sampler producer=localhost1 instance=localhost1/test3 schema=test3 component_id=1 num_metrics=2 - job_set=localhost1/jobid - start name=tutorial_sampler interval=1000000 + config name=tutorial_sampler producer=localhost1 instance=localhost1/test2 schema=test2 component_id=2 num_metrics=5 + config name=tutorial_sampler producer=localhost1 instance=localhost1/test3 schema=test3 component_id=1 num_metrics=2 + job_set=localhost1/jobid + start name=tutorial_sampler interval=1000000 > ldms_ls localhost1/test1: consistent, last update: Thu Oct 24 10:55:14 2019 -0600 [223680us] M u64 component_id 1 D u64 job_id 0 D u64 app_id 0 diff --git a/rtd/docs/source/sampler_man/Plugin_tx2mon.rst b/rtd/docs/source/sampler_man/Plugin_tx2mon.rst index 4acdcdd4f..0f41c1a73 100644 --- a/rtd/docs/source/sampler_man/Plugin_tx2mon.rst +++ b/rtd/docs/source/sampler_man/Plugin_tx2mon.rst @@ -37,7 +37,7 @@ tx2mon are listed here [auto-schema=] schema= - | + | | Optional schema name. It is required by most storage backends that the same sampler on different nodes with different metric subsets needs to have a unique schema name. Use auto-schema=1 @@ -45,20 +45,20 @@ tx2mon are listed here backend requirement. auto-schema= - | + | | If true, change the schema name to tx2mon\_$X, where $X will be a unique value derived from the data selection options. If both schema and auto-schema=1 are given, the schema name given is used as the base instead of "tx2mon". array= - | + | | For per-core data, report all array value elements if true. Report only maximum and minimum values if false. The default is false. extra= - | + | | For per-core data, report additional information of the internal block frequencies and the set system metrics. These additional values are static. If false, additional information will not be diff --git a/rtd/docs/source/sampler_man/Plugin_variable.rst b/rtd/docs/source/sampler_man/Plugin_variable.rst index 5144c9ad1..c0514fbeb 100644 --- a/rtd/docs/source/sampler_man/Plugin_variable.rst +++ b/rtd/docs/source/sampler_man/Plugin_variable.rst @@ -39,17 +39,17 @@ instance name. | configuration line name= - | + | | This MUST be variable. schema= - | + | | Optional schema name prefix. The string given will be suffixed with an integer N in the range 1-9 to create the schema name. The schema will also contain N integer metrics. instance= - | + | | Optional instance name prefix. The string given will be suffixed with an integer in the range 1-9 to create the instance name. If not specified, will default prefix is \`$HOST/variable\`. diff --git a/rtd/docs/source/sampler_man/Plugin_variorum_sampler.rst b/rtd/docs/source/sampler_man/Plugin_variorum_sampler.rst index 58ab0d0ed..43f01b5a8 100644 --- a/rtd/docs/source/sampler_man/Plugin_variorum_sampler.rst +++ b/rtd/docs/source/sampler_man/Plugin_variorum_sampler.rst @@ -54,11 +54,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be variorum_sampler. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`variorum_sampler\`. diff --git a/rtd/docs/source/sampler_man/Plugin_vmstat.rst b/rtd/docs/source/sampler_man/Plugin_vmstat.rst index 6e0f5864c..bbfd2f0ee 100644 --- a/rtd/docs/source/sampler_man/Plugin_vmstat.rst +++ b/rtd/docs/source/sampler_man/Plugin_vmstat.rst @@ -39,11 +39,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be vmstat. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`vmstat\`. diff --git a/rtd/docs/source/sampler_man/index.rst b/rtd/docs/source/sampler_man/index.rst index 9918a4174..b19f60e71 100644 --- a/rtd/docs/source/sampler_man/index.rst +++ b/rtd/docs/source/sampler_man/index.rst @@ -4,5 +4,5 @@ Sampler Man Pages .. toctree:: :maxdepth: 1 :glob: - + * diff --git a/rtd/docs/source/sampler_man/ldms-ibnet-sampler-gen.rst b/rtd/docs/source/sampler_man/ldms-ibnet-sampler-gen.rst index a6b770516..349134bae 100644 --- a/rtd/docs/source/sampler_man/ldms-ibnet-sampler-gen.rst +++ b/rtd/docs/source/sampler_man/ldms-ibnet-sampler-gen.rst @@ -79,7 +79,7 @@ EXAMPLES ldms-ibnet-sampler-gen --lidnames --net cluster-p-netdiscover --samplers x --out x | sort -k2 -t, > lid.host.txt - ldms-ibnet-sampler-gen --net cluster-p-netdiscover --samplers clustre-samplers --sharp 37 --annotate --out sbx + ldms-ibnet-sampler-gen --net cluster-p-netdiscover --samplers clustre-samplers --sharp 37 --annotate --out sbx :: diff --git a/rtd/docs/source/sampler_man/ldms-sensors-config.rst b/rtd/docs/source/sampler_man/ldms-sensors-config.rst index f38bd8802..f6ac04a51 100644 --- a/rtd/docs/source/sampler_man/ldms-sensors-config.rst +++ b/rtd/docs/source/sampler_man/ldms-sensors-config.rst @@ -36,29 +36,29 @@ OPTIONS ======================= --sensors= - | + | | specify an alternate location of the sensors program. The default is /usr/bin/sensors, and the PATH variable is not used to search for alternatives. --nodash - | + | | Replace all - characters in metric names with \_ characters. --lscpu= - | + | | specify an alternate location of the lscpu program. The default is /usr/bin/lscpu and the PATH variable is not used to search for alternatives. --test-lscpu= - | + | | Specify the location of a pre-collected strace log of lscpu to use instead of lscpu run on the local system. Used for testing or remote configuration. --test-sensors= - | + | | Specify the location of a pre-collected strace log of sensors to use instead of sensors run on the local system. Used for testing or remote configuration. diff --git a/rtd/docs/source/sampler_man/ldms_ibnet_schema_name.rst b/rtd/docs/source/sampler_man/ldms_ibnet_schema_name.rst index 43b75847b..b340837da 100644 --- a/rtd/docs/source/sampler_man/ldms_ibnet_schema_name.rst +++ b/rtd/docs/source/sampler_man/ldms_ibnet_schema_name.rst @@ -50,15 +50,15 @@ EXAMPLES flowctlcounters vloppackets vlopdata - vlxmitflowctlerrors - vlxmitcounters + vlxmitflowctlerrors/t + vlxmitcounters/t swportvlcong - rcvcc + rcvcc/t slrcvfecn slrcvbecn - xmitcc + xmitcc/t vlxmittimecc - smplctl + smplctl/t yields diff --git a/rtd/docs/source/store_man/Plugin_avro_kafka_store.rst b/rtd/docs/source/store_man/Plugin_avro_kafka_store.rst index 46232023c..a847bf240 100644 --- a/rtd/docs/source/store_man/Plugin_avro_kafka_store.rst +++ b/rtd/docs/source/store_man/Plugin_avro_kafka_store.rst @@ -8,12 +8,12 @@ Plugin_avro_kafka_store :depth: 3 .. -NAME +NAME ========================= avro_kafka_store - LDMSD avro_kafka_store plugin -SYNOPSIS +SYNOPSIS ============================= **config** **name=avro_kafka_store** **producer=PRODUCER** @@ -21,7 +21,7 @@ SYNOPSIS [ **encoding=\ AVRO** ] [ **kafka_conf=\ PATH** ] [ **serdes_conf=\ PATH** ] -DESCRIPTION +DESCRIPTION ================================ **``avro_kafka_store``** implements a decomposition capable LDMS metric @@ -37,26 +37,26 @@ When in *AVRO* mode, the plugin manages schema in cooperation with an Avro Schema Registry. The location of this registry is specified in a configuration file or optionally on the **``config``** command line. -CONFIG OPTIONS +CONFIG OPTIONS =================================== -mode +mode A string indicating the encoding mode: "JSON" will encode messages in JSON format, "AVRO" will encode messages using a schema and Avro Serdes. The default is "AVRO". The mode values are not case sensitive. -name +name Must be avro_kafka_store. -kafka_conf +kafka_conf A path to a configuration file in Java property format. This configuration file is parsed and used to configure the Kafka kafka_conf_t configuration object. The format of this file and the supported attributes are available here: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md. -serdes_conf +serdes_conf A path to a configuration file in Java property format. This configuration file is parsed and used to configure the Avro Serdes serdes_conf_t configuration object. The only supported option for @@ -75,37 +75,37 @@ The '%' character introduces a *format specifier* that will be substituted in the topic format string to create the topic name. The format specifiers are as follows: -%F +%F The format in which the message is serialized: "json" or "avro". -%S +%S The set parameter's *schema* name. -%I +%I The instance name of the set, e.g. "orion-01/meminfo". -%P +%P The set parameter's *producer* name, e.g. "orion-01." -%u +%u The user-name string for the owner of the set. If the user-name is not known on the system, the user-id is used. -%U +%U The user-id (uid_t) for the owner of the set. -%g +%g The group-name string for the group of the set. If the group-name is not known on the system, the group-id is used. -%G +%G The group-id (gid_t) for the group of the set. -%a +%a The access/permission bits for the set formatted as a string, e.g. "-rw-rw----". -%A +%A The access/permission bits for the set formatted as an octal number, e.g. 0440. @@ -283,10 +283,10 @@ schema-id to query the Schema registry for a schema. Once found, the client will construct a serdes from the schema definition and use this serdes to decode the message into Avro values. -EXAMPLES +EXAMPLES ============================= -kafka_conf Example File +kafka_conf Example File ------------------------ :: @@ -297,7 +297,7 @@ kafka_conf Example File # Specify the location of the Kafka broker bootstrap.server=localhost:9092 -serdes_conf Example File +serdes_conf Example File ------------------------- :: @@ -307,7 +307,7 @@ serdes_conf Example File # set to anything other than an empty string serdes.schema.url=https://localhost:9092 -Example strg_add command +Example strg_add command ------------------------- :: diff --git a/rtd/docs/source/store_man/Plugin_darshan_stream_store.rst b/rtd/docs/source/store_man/Plugin_darshan_stream_store.rst index 533135513..6195c1611 100644 --- a/rtd/docs/source/store_man/Plugin_darshan_stream_store.rst +++ b/rtd/docs/source/store_man/Plugin_darshan_stream_store.rst @@ -38,20 +38,20 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be darshan_stream_store. path= - | + | | The path to the root of the SOS container store (should be created by the user) stream= - | + | | stream to which to subscribe. mode= - | + | | The container permission mode for create, (defaults to 0660). INPUT JSON FORMAT diff --git a/rtd/docs/source/store_man/Plugin_store_flatfile.rst b/rtd/docs/source/store_man/Plugin_store_flatfile.rst index 082c8376a..71b96fe0b 100644 --- a/rtd/docs/source/store_man/Plugin_store_flatfile.rst +++ b/rtd/docs/source/store_man/Plugin_store_flatfile.rst @@ -40,21 +40,21 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_flatfile. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). They also are used to match any action=custom configuration.node/meminfo. schema= - | + | | The container and schema determines where the output files will be written (see path above). diff --git a/rtd/docs/source/store_man/Plugin_store_sos.rst b/rtd/docs/source/store_man/Plugin_store_sos.rst index 3bc2ef191..b4a20e59c 100644 --- a/rtd/docs/source/store_man/Plugin_store_sos.rst +++ b/rtd/docs/source/store_man/Plugin_store_sos.rst @@ -38,11 +38,11 @@ STORE_SOS INIT CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_sos. path= - | + | | The store will be put into a directory whose root is specified by the path argument. This directory must exist; the store will be created. The full path to the store will be @@ -61,27 +61,27 @@ container and schema for a store. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_sos. name= - | + | | The policy name for this strgp. container= - | + | | The container and schema define the store as described above (see path). schema= - | + | | The container and schema define the store as described above (see path). You can have multiples of the same path and container, but with different schema (which means they will have different metrics) and they will be stored in the same store. decomposition= - | + | | Optionally use set-to-row decomposition with the specified configuration file in JSON format. See more about decomposition in ldmsd_decomposition(7). @@ -101,11 +101,11 @@ into XXX. Any commands given with no argument, will return usage info. | Create a partition. **-C** ** - | + | | Path to the container **-s** *state* - | + | | State of the new partition (case insensitive). Default is OFFLINE. Optional parameter. Valid options are: @@ -117,7 +117,7 @@ into XXX. Any commands given with no argument, will return usage info. or deleted. **part_name** - | + | | Name of the partition **sos_part_delete** @@ -126,11 +126,11 @@ into XXX. Any commands given with no argument, will return usage info. OFFLINE state to be deleted. **-C** ** - | + | | Path to the container **name** - | + | | Name of the parition **sos_part_modify** @@ -138,11 +138,11 @@ into XXX. Any commands given with no argument, will return usage info. | Modify the state of a partition. **-C** ** - | + | | Path to the container **-s** *state* - | + | | State of the new partition (case insensitive). Default is OFFLINE. Optional parameter. Valid options are: @@ -154,24 +154,24 @@ into XXX. Any commands given with no argument, will return usage info. or deleted. **part_name** - | + | | Name of the partition **sos_part_move** - | + | | Move a partition to another storage location. -C -p part_name **-C** ** - | + | | Path to the container **-p** ** - | + | | The new path. **part_name** - | + | | Name of the partition USING SOS COMMANDS TO LOOK AT DATA IN A PARTITION @@ -185,7 +185,7 @@ command options are below. Example usage is in the example section. | Print a directory of the schemas. **-C** ** - | + | | Path to the container **sos_cmd** @@ -193,7 +193,7 @@ command options are below. Example usage is in the example section. | Show debug information for the container **-C** ** - | + | | Path to the container **sos_cmd** @@ -201,22 +201,22 @@ command options are below. Example usage is in the example section. | Print data from a container **-C** ** - | + | | Path to the container **-q** Used to query **-S** ** - | + | | Schema querying against **-X** ** - | + | | Variable that is indexed to use in the query. **-V** ** - | + | | One or more vars to output. NOTES diff --git a/rtd/docs/source/store_man/Plugin_store_timescale.rst b/rtd/docs/source/store_man/Plugin_store_timescale.rst index d020281b5..06a846870 100644 --- a/rtd/docs/source/store_man/Plugin_store_timescale.rst +++ b/rtd/docs/source/store_man/Plugin_store_timescale.rst @@ -44,37 +44,37 @@ STORE_TIMESCALE CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_timescale. user= - | + | | This option is required; It will be used as the user name to connect to timescaledb. pwfile= - | + | | This option is required; The file must have content secretword=, the password will be used as the password to connect to timescaledb. hostaddr= - | + | | This option is required; It will be used as the ip addr of timescaledb to connect to. port= - | + | | This option is required; It will be used as the port number of timescaledb to connect to. dbname= - | + | | This option is required; It will be used as the timescaledb database name to connect to. measurement_limit= - | + | | This is optional; It specifies the maximum length of the sql statement to create table or insert data into timescaledb; default 8192. @@ -91,20 +91,20 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_timescale. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). schema= - | + | | The container and the schema determine where the output files will be written (see path above). You can have multiples of the same sampler, but with different schema (which means they will diff --git a/rtd/docs/source/store_man/Plugin_store_tutorial.rst b/rtd/docs/source/store_man/Plugin_store_tutorial.rst index 9f73d3f02..5959adff4 100644 --- a/rtd/docs/source/store_man/Plugin_store_tutorial.rst +++ b/rtd/docs/source/store_man/Plugin_store_tutorial.rst @@ -43,11 +43,11 @@ STORE_TUTORIAL CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_tutorial. path= - | + | | This option is required; the config line or the options file must supply a default value. The output files will be put into a directory whose root is specified by the path argument. This @@ -68,20 +68,20 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_tutorial. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). schema= - | + | | The container and the schema determine where the output files will be written (see path above). You can have multiples of the same sampler, but with different schema (which means they will diff --git a/rtd/docs/source/store_man/Plugin_stream_csv_store.rst b/rtd/docs/source/store_man/Plugin_stream_csv_store.rst index ef9b246de..0a80eedb3 100644 --- a/rtd/docs/source/store_man/Plugin_stream_csv_store.rst +++ b/rtd/docs/source/store_man/Plugin_stream_csv_store.rst @@ -40,35 +40,35 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be stream_csv_store. path= - | + | | path to the directory of the csv output file container= - | + | | directory of the csv output file stream= - | + | | csv list of streams to which to subscribe. flushtime= - | + | | Flush any file that has not received data on its stream in the last N sec. This is asynchonous to any buffering or rollover that is occuring. Min time if enabled = 120 sec. This will occur again at this interval if there is still no data received. buffer=<0/1> - | + | | Optional buffering of the output. 0 to disable buffering, 1 to enable it with autosize (default) rolltype= - | + | | By default, the store does not rollover and the data is written to a continously open filehandle. Rolltype and rollover are used in conjunction to enable the store to manage rollover, including @@ -76,25 +76,25 @@ CONFIGURATION ATTRIBUTE SYNTAX roll occurs. Valid options are: 1 - | + | | wake approximately every rollover seconds and roll. Rollover is suppressed if no data at all has been written. 2 - | + | | wake daily at rollover seconds after midnight (>=0) and roll. Rollover is suppressed if no data at all has been written. 3 - | + | | roll after approximately rollover records are written. 4 - | + | | roll after approximately rollover bytes are written. 5 - | + | | wake at rollover seconds after midnight (>=0) and roll, then repeat every rollagain (> rollover) seconds during the day. For example "rollagain=3600 rollover=0 rolltype=5" rolls @@ -102,7 +102,7 @@ CONFIGURATION ATTRIBUTE SYNTAX been written. rollover= - | + | | Rollover value controls the frequency of rollover (e.g., number of bytes, number of records, time interval, seconds after midnight). Note that these values are estimates due to the @@ -166,7 +166,7 @@ Options for timing information are driven by #defines in the code source right now. TIMESTAMP_STORE - | + | | Set by #define or #undef TIMESTAMP_STORE. This will write out an absolute timestamp in the file as the last item in the csv and is called 'store_recv_time' in the header. The timestamp is only @@ -176,7 +176,7 @@ TIMESTAMP_STORE and json are timestamped. STREAM_CSV_DIAGNOSTICS - | + | | Set by #define or #undef STREAM_CSV_DIAGNOSTICS. This will write out diagnostic info to the log when stream_cb is called. diff --git a/rtd/docs/source/store_man/index.rst b/rtd/docs/source/store_man/index.rst index fd9179a19..d96eb6013 100644 --- a/rtd/docs/source/store_man/index.rst +++ b/rtd/docs/source/store_man/index.rst @@ -4,5 +4,5 @@ Store Plugin Man Pages .. toctree:: :maxdepth: 1 :glob: - + * diff --git a/rtd/docs/source/ug.rst b/rtd/docs/source/ug.rst index 31e7df01c..c371f1fb2 100644 --- a/rtd/docs/source/ug.rst +++ b/rtd/docs/source/ug.rst @@ -1,7 +1,7 @@ LDMS User's Group ============================== -LDMS User's group will meet every other Monday at Noon (Mountain time). +LDMS User's group will meet every other Monday at Noon (Mountain time). Sign up for meeting announcements using the information below. The LDMS Mailing Lists are hosted by LLNL at listserv.llnl.gov. The current available lists are: diff --git a/rtd/man2rst/Plugin_app_sampler.rst b/rtd/man2rst/Plugin_app_sampler.rst index 530ef4adc..79459414f 100644 --- a/rtd/man2rst/Plugin_app_sampler.rst +++ b/rtd/man2rst/Plugin_app_sampler.rst @@ -8,12 +8,12 @@ Plugin_app_sampler :depth: 3 .. -NAME +NAME ==================== ldmsd_app_sampler - LDMSD app_sampler plugin -SYNOPSIS +SYNOPSIS ======================== **config** **name=app_sampler** **producer=**\ *PRODUCER* @@ -21,7 +21,7 @@ SYNOPSIS **component_id=\ COMPONENT_ID** ] [ **stream=\ STREAM_NAME** ] [ **metrics=\ METRICS** ] [ **cfg_file=\ PATH** ] -DESCRIPTION +DESCRIPTION =========================== **``app_sampler``** collects metrics from **``/proc/``** according @@ -38,34 +38,34 @@ metrics to monitor by specifying **``metrics``** option (comma-separated string) or writing a JSON configuration file and specifying **``cfg_file``** option (see **``EXAMPLES``** section). -CONFIG OPTIONS +CONFIG OPTIONS ============================== -name +name Must be app_sampler. -producer +producer The name of the data producer (e.g. hostname). -instance +instance This is required by sampler_base but is not used by app_sampler. So, this can be any string but must be present. -schema +schema The optional schema name (default: app_sampler). -component_id +component_id An integer identifying the component (default: *0*). -stream +stream The name of the **``ldmsd_stream``** to listen for SLURM job events. (default: slurm). -metrics +metrics The comma-separated list of metrics to monitor. The default is '' (empty), which is equivalent to monitor ALL metrics. -cfg_file +cfg_file The alternative config file in JSON format. The file is expected to have an object that may contain the following attributes: @@ -85,7 +85,7 @@ specified. Attributes other than 'stream' and 'metrics' are ignored. If the **``cfg_file``** is given, **``stream``** and **``metrics``** options are ignored. -LIST OF METRICS +LIST OF METRICS =============================== :: @@ -245,15 +245,15 @@ LIST OF METRICS /* /proc/[pid]/wchan */ wchan, -BUGS +BUGS ==================== No known bugs. -EXAMPLES +EXAMPLES ======================== -Example 1 +Example 1 ---------- Get everyting: @@ -262,7 +262,7 @@ Get everyting: config name=app_sampler -Example 2 +Example 2 ---------- Down-select and with non-default stream name: @@ -271,7 +271,7 @@ Down-select and with non-default stream name: config name=app_sampler metrics=stat_pid,stat_utime stream=mystream -Example 3 +Example 3 ---------- Down-select using config file, using default stream: diff --git a/rtd/man2rst/Plugin_aries_linkstatus.rst b/rtd/man2rst/Plugin_aries_linkstatus.rst index addcbf814..268ab7da8 100644 --- a/rtd/man2rst/Plugin_aries_linkstatus.rst +++ b/rtd/man2rst/Plugin_aries_linkstatus.rst @@ -66,25 +66,25 @@ attributes of the base class. | configuration line name= - | + | | aries_linkstatus file_send= - | + | | Location of the file with the sendlinkstatus metrics, as specified in the gpcdr configuration file. In the Cray-provided default gpcdr configuration, this will be /sys/devices/virtual/gni/gpcdr0/metricsets/linksendstatus/metrics. file_recv= - | + | | Location of the file with the recvlinkstatus metrics, as specified in the gpcdr configuration file. In the Cray-provided default gpcdr configuration, this will be /sys/devices/virtual/gni/gpcdr0/metricsets/linkrecvstatus/metrics. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. This will default to aries_linkstatus, if unspecified. diff --git a/rtd/man2rst/Plugin_aries_mmr.rst b/rtd/man2rst/Plugin_aries_mmr.rst index 65c3afa10..a16bd7d58 100644 --- a/rtd/man2rst/Plugin_aries_mmr.rst +++ b/rtd/man2rst/Plugin_aries_mmr.rst @@ -68,22 +68,22 @@ attributes of the base class. | configuration line name= - | + | | This MUST be aries_mmr, aries_nic_mmr, or aries_rtr_mmr. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. This will default to cray_aries_r or cray_gemini_r as appropriate, if unspecified. aries_rtr_id= - | + | | Optional aries router identifier. Defaults to 0 length string. file= - | + | | Configuration file of aries performance counter names that will be added in exactly as they are specified. At least one file must be specified. @@ -120,7 +120,7 @@ EXAMPLES | #RAW METRICS | AR_NIC_NETMON_ORB_EVENT_CNTR_REQ_FLITS -| +| | AR_NIC_RSPMON_NPT_EVENT_CNTR_NL_FLITS | # this is a test | AR_RTR_1_2_INQ_PRF_INCOMING_FLIT_VC0 diff --git a/rtd/man2rst/Plugin_aries_mmr_configurable.rst b/rtd/man2rst/Plugin_aries_mmr_configurable.rst index 7ed793d51..8a3c53112 100644 --- a/rtd/man2rst/Plugin_aries_mmr_configurable.rst +++ b/rtd/man2rst/Plugin_aries_mmr_configurable.rst @@ -102,11 +102,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be aries_mmr_configurable action= - | + | | Options are initialize, finalize, reset, and ls: **initialize** @@ -115,7 +115,7 @@ attributes of the base class. should be specified at this point. setfile= - | + | | Optional configuration file with the counter value assignment options. | Format: "name,type,default_value" one entry per line. @@ -133,7 +133,7 @@ attributes of the base class. These are printed out in the utility check_mmr_configurable. readfile= - | + | | Configuration file with the names of the counters to read. | Format "name,type" one entry per line. | Type is 'H' for Hex or anything else to default to uint64_t. @@ -149,28 +149,28 @@ attributes of the base class. These are printed out in the utility check_mmr_configurable. rtrid= - | + | | Optional unique rtr string identifier (e.g., c0-0c0s0a0). Defaults to 0 length string. schema= - | + | | Optional schema name. Defaults to 'aries_mmr_configurable'. **finalize** - | + | | Creates the mmr_contexts, sets the set counters to the configured values, and creates the set. Takes no arguments. If finalize fails, all state is cleared and the plugin can be configured again. **ls** - | + | | Prints out the set counter names and their configured values and also the read counter names. Takes no arguments. **reset** - | + | | Resets the set counters to their configured values. Takes no arguments. diff --git a/rtd/man2rst/Plugin_avro_kafka_store.rst b/rtd/man2rst/Plugin_avro_kafka_store.rst index 46232023c..a847bf240 100644 --- a/rtd/man2rst/Plugin_avro_kafka_store.rst +++ b/rtd/man2rst/Plugin_avro_kafka_store.rst @@ -8,12 +8,12 @@ Plugin_avro_kafka_store :depth: 3 .. -NAME +NAME ========================= avro_kafka_store - LDMSD avro_kafka_store plugin -SYNOPSIS +SYNOPSIS ============================= **config** **name=avro_kafka_store** **producer=PRODUCER** @@ -21,7 +21,7 @@ SYNOPSIS [ **encoding=\ AVRO** ] [ **kafka_conf=\ PATH** ] [ **serdes_conf=\ PATH** ] -DESCRIPTION +DESCRIPTION ================================ **``avro_kafka_store``** implements a decomposition capable LDMS metric @@ -37,26 +37,26 @@ When in *AVRO* mode, the plugin manages schema in cooperation with an Avro Schema Registry. The location of this registry is specified in a configuration file or optionally on the **``config``** command line. -CONFIG OPTIONS +CONFIG OPTIONS =================================== -mode +mode A string indicating the encoding mode: "JSON" will encode messages in JSON format, "AVRO" will encode messages using a schema and Avro Serdes. The default is "AVRO". The mode values are not case sensitive. -name +name Must be avro_kafka_store. -kafka_conf +kafka_conf A path to a configuration file in Java property format. This configuration file is parsed and used to configure the Kafka kafka_conf_t configuration object. The format of this file and the supported attributes are available here: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md. -serdes_conf +serdes_conf A path to a configuration file in Java property format. This configuration file is parsed and used to configure the Avro Serdes serdes_conf_t configuration object. The only supported option for @@ -75,37 +75,37 @@ The '%' character introduces a *format specifier* that will be substituted in the topic format string to create the topic name. The format specifiers are as follows: -%F +%F The format in which the message is serialized: "json" or "avro". -%S +%S The set parameter's *schema* name. -%I +%I The instance name of the set, e.g. "orion-01/meminfo". -%P +%P The set parameter's *producer* name, e.g. "orion-01." -%u +%u The user-name string for the owner of the set. If the user-name is not known on the system, the user-id is used. -%U +%U The user-id (uid_t) for the owner of the set. -%g +%g The group-name string for the group of the set. If the group-name is not known on the system, the group-id is used. -%G +%G The group-id (gid_t) for the group of the set. -%a +%a The access/permission bits for the set formatted as a string, e.g. "-rw-rw----". -%A +%A The access/permission bits for the set formatted as an octal number, e.g. 0440. @@ -283,10 +283,10 @@ schema-id to query the Schema registry for a schema. Once found, the client will construct a serdes from the schema definition and use this serdes to decode the message into Avro values. -EXAMPLES +EXAMPLES ============================= -kafka_conf Example File +kafka_conf Example File ------------------------ :: @@ -297,7 +297,7 @@ kafka_conf Example File # Specify the location of the Kafka broker bootstrap.server=localhost:9092 -serdes_conf Example File +serdes_conf Example File ------------------------- :: @@ -307,7 +307,7 @@ serdes_conf Example File # set to anything other than an empty string serdes.schema.url=https://localhost:9092 -Example strg_add command +Example strg_add command ------------------------- :: diff --git a/rtd/man2rst/Plugin_blob_stream_writer.rst b/rtd/man2rst/Plugin_blob_stream_writer.rst index 5308e9030..46f8c7475 100644 --- a/rtd/man2rst/Plugin_blob_stream_writer.rst +++ b/rtd/man2rst/Plugin_blob_stream_writer.rst @@ -38,28 +38,28 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be blob_stream_writer. path= - | + | | path to the directory of the output files container= - | + | | directory of the output file stream= - | + | | stream to which to subscribe. This argument may be repeated. Each stream will be written in a separate file pair. debug=1 - | + | | Enable logging of messages stored to the log file. timing=1 - | + | | Enable writing timestamps to a separate file. OUTPUT FORMAT diff --git a/rtd/man2rst/Plugin_clock.rst b/rtd/man2rst/Plugin_clock.rst index b1cf61767..f5af75541 100644 --- a/rtd/man2rst/Plugin_clock.rst +++ b/rtd/man2rst/Plugin_clock.rst @@ -39,11 +39,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be clock schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`clock\`. diff --git a/rtd/man2rst/Plugin_cray_dvs_sampler.rst b/rtd/man2rst/Plugin_cray_dvs_sampler.rst index 0a4fb56e2..7788bea20 100644 --- a/rtd/man2rst/Plugin_cray_dvs_sampler.rst +++ b/rtd/man2rst/Plugin_cray_dvs_sampler.rst @@ -48,17 +48,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be cray_dvs_sampler schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`cray_dvs_sampler\`. conffile= - | + | | Optional path to the configuration file DATA AND THE CONFIGURATION FILE diff --git a/rtd/man2rst/Plugin_daos_sampler.rst b/rtd/man2rst/Plugin_daos_sampler.rst index 1931048ca..b8d742adb 100644 --- a/rtd/man2rst/Plugin_daos_sampler.rst +++ b/rtd/man2rst/Plugin_daos_sampler.rst @@ -35,20 +35,20 @@ only covers the configuration attributes specific to this plugin; see ldms_sampler_base.man for the attributes of the base class. name= - | + | | This MUST be daos_sampler. producer=$HOSTNAME - | + | | The $HOSTNAME variable provides a good unique producer ID. engine_count=2 - | + | | The default is 2; don't change it unless the number of per-server engines is different. target_count=8 - | + | | The default is 8; don't change it unless the number of targets per engine is different. diff --git a/rtd/man2rst/Plugin_darshan_stream_store.rst b/rtd/man2rst/Plugin_darshan_stream_store.rst index 533135513..6195c1611 100644 --- a/rtd/man2rst/Plugin_darshan_stream_store.rst +++ b/rtd/man2rst/Plugin_darshan_stream_store.rst @@ -38,20 +38,20 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be darshan_stream_store. path= - | + | | The path to the root of the SOS container store (should be created by the user) stream= - | + | | stream to which to subscribe. mode= - | + | | The container permission mode for create, (defaults to 0660). INPUT JSON FORMAT diff --git a/rtd/man2rst/Plugin_dcgm_sampler.rst b/rtd/man2rst/Plugin_dcgm_sampler.rst index 16dbe5e0a..5e2942fb9 100644 --- a/rtd/man2rst/Plugin_dcgm_sampler.rst +++ b/rtd/man2rst/Plugin_dcgm_sampler.rst @@ -37,27 +37,27 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be dcgm_sampler. interval= - | + | | The sampling interval. This MUST be set to the same value that is set on the "start" line, otherwise behavior is undetermined. fields= - | + | | is a comma-separated list of integers representing DCGM field numebers that the plugin should watch. By default the plugin will watch fields 150,155. schema= - | + | | The schema name defaults to "dcgm", but it can be renamed at the user's choice. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) diff --git a/rtd/man2rst/Plugin_dstat.rst b/rtd/man2rst/Plugin_dstat.rst index dc872525d..c09096bcf 100644 --- a/rtd/man2rst/Plugin_dstat.rst +++ b/rtd/man2rst/Plugin_dstat.rst @@ -35,26 +35,26 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be dstat. producer= - | + | | The producer string value. instance= - | + | | The name of the metric set. schema= - | + | | Optional schema name. It is required by most storage backends that the same sampler on different nodes with different metric subsets needs to have a unique schema name. Use auto-schema=1 instead of schema to automatically meet the backend requirement. auto-schema= - | + | | If true, change the schema name to dstat\_$X, where $X will be a unique hex value derived from the data selection options. If both schema and auto-schema are given, for @@ -62,36 +62,36 @@ CONFIGURATION ATTRIBUTE SYNTAX plugin. component_id= - | + | | The component id numerical value. io= - | + | | Include the metrics from /proc/self/io. stat= - | + | | Include the metrics from /proc/self/stat. tick= - | + | | Include the sc_clk_tck from sysconf(3) as a metric. statm= - | + | | Include the metrics from /proc/self/statm. mmalloc= - | + | | Include the mmap memory usage metric from LDMS mmalloc. fd= - | + | | Include the number of open file descriptors found in /proc/self/fd. fdtypes= - | + | | Include the number and types of open file descriptors found in /proc/self/fd. This option may have high overhead on aggregators with many open connections. diff --git a/rtd/man2rst/Plugin_edac.rst b/rtd/man2rst/Plugin_edac.rst index 1e7d41867..e82811fe6 100644 --- a/rtd/man2rst/Plugin_edac.rst +++ b/rtd/man2rst/Plugin_edac.rst @@ -39,22 +39,22 @@ attributes of the base class. | configuration line name= - | + | | This MUST be edac. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to edac. max_mc= - | + | | The number of mc's in /sys/devices/system/edac/mc. Typically this number is 2. max_csrow= - | + | | The number of csrows in a single mc. For example, the value should be 4 for when the largest csrow looks like: /sys/devices/system/edac/mc/mc0/csrow3. Typically this number is diff --git a/rtd/man2rst/Plugin_filesingle.rst b/rtd/man2rst/Plugin_filesingle.rst index 783c394e5..73cd8f547 100644 --- a/rtd/man2rst/Plugin_filesingle.rst +++ b/rtd/man2rst/Plugin_filesingle.rst @@ -35,12 +35,12 @@ See ldms_sampler_base(7) for the common sampler options. | conf= [timing] conf= - | + | | File lines contain the source, type, and default value for each metric. See CONF FILE SYNTAX below. timing - | + | | If keyword 'timing' is included in the options, extra metrics measuring the time to collect every defined metric will be included. This allows for the discovery of slow sensors. Each diff --git a/rtd/man2rst/Plugin_fptrans.rst b/rtd/man2rst/Plugin_fptrans.rst index 1b76a97d7..84802c7ef 100644 --- a/rtd/man2rst/Plugin_fptrans.rst +++ b/rtd/man2rst/Plugin_fptrans.rst @@ -39,11 +39,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be fptrans. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, it will default to \`fptrans\`. diff --git a/rtd/man2rst/Plugin_geopm_sampler.rst b/rtd/man2rst/Plugin_geopm_sampler.rst index acdc35e8e..f9fec8474 100644 --- a/rtd/man2rst/Plugin_geopm_sampler.rst +++ b/rtd/man2rst/Plugin_geopm_sampler.rst @@ -46,11 +46,11 @@ configured with the option - \``geopm_request_path=`\`. | configuration line name= - | + | | This MUST be ldms_geopm_sampler. geopm_request_path= - | + | | This parameter points to the absolute path of the ASCII file containing the list of signals that the user would like to have monitored by the sampler. @@ -69,7 +69,7 @@ configured with the option - \``geopm_request_path=`\`. than the number of available domains. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`ldms_geopm_sampler\`. diff --git a/rtd/man2rst/Plugin_hello_sampler.rst b/rtd/man2rst/Plugin_hello_sampler.rst index 3ea3d8732..24ff1ae8c 100644 --- a/rtd/man2rst/Plugin_hello_sampler.rst +++ b/rtd/man2rst/Plugin_hello_sampler.rst @@ -41,11 +41,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be hello_sampler. stream= - | + | | Name of the stream to which to subscribe. BUGS diff --git a/rtd/man2rst/Plugin_ibmad_records_sampler.rst b/rtd/man2rst/Plugin_ibmad_records_sampler.rst index 83fecc763..959128b2b 100644 --- a/rtd/man2rst/Plugin_ibmad_records_sampler.rst +++ b/rtd/man2rst/Plugin_ibmad_records_sampler.rst @@ -42,37 +42,37 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be ibmad_records_sampler. schema= - | + | | The schema name defaults to "ibmad", but it can be renamed at the user's choice. rate=0 - | + | | Stop the default inclusion of rate values in the set. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) include=PORTLIST - | + | | Ignore any devices and ports discovered that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the exclude option. exclude=PORTLIST - | + | | Collect all devices and ports discovered and active that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the include option. refresh_interval_sec= - | + | | (Optional) The sampler caches the list of infiniband devices, and that cache is refreshed at the beginning of a sample cycle if the refresh interval time has been exceeded. diff --git a/rtd/man2rst/Plugin_ibmad_sampler.rst b/rtd/man2rst/Plugin_ibmad_sampler.rst index 2882de6e7..d4ac045c3 100644 --- a/rtd/man2rst/Plugin_ibmad_sampler.rst +++ b/rtd/man2rst/Plugin_ibmad_sampler.rst @@ -40,31 +40,31 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be ibmad_sampler. schema= - | + | | The schema name defaults to "ibmad_sampler", but it can be renamed at the user's choice. rate=0 - | + | | Stop the default inclusion of rate values in the set. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) include=PORTLIST - | + | | Ignore any devices and ports discovered that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the exclude option. exclude=PORTLIST - | + | | Collect all devices and ports discovered and active that are not matched by PORTLIST. See PORTLIST below. Cannot be combined with the include option. diff --git a/rtd/man2rst/Plugin_ibnet.rst b/rtd/man2rst/Plugin_ibnet.rst index c00d6c901..0bfe0c6c8 100644 --- a/rtd/man2rst/Plugin_ibnet.rst +++ b/rtd/man2rst/Plugin_ibnet.rst @@ -41,64 +41,64 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be ibnet. producer=. - | + | | The producer string value for the timing set. Default is the result of gethostname(). instance= - | + | | The name of the timing metric set. Default is $producer/ibnet_timing. source-list= - | + | | Lidfile is the name of a file of LID/port specifications. See PORT FILE for format details. port-name= [port-number=] - | + | | Hca is the name of the local IB interface to access the network. Num is the number of the port on the interface used to access the network. The default is 1. schema= - | + | | Optional schema base name. The default is ibnet. The name base is suffixed to create uniquely defined schema names based on the plugin options specified. component_id= - | + | | Optional component identifier for the timing set. Defaults to zero. metric-conf= - | + | | The file listing the metric groups to collect. See METRIC GROUPS below. ca_port= - | + | | The port number to use, which must be active. millis= - | + | | The number of milliseconds of the timeout on the MAD calls. Default 0, which will use the mad library timeout of 1 second. timing= - | + | | Disable timing (T=0), enable aggregate timing (T=1), or enable individual port timing(T=2) or enable port offset timing(T=3). The metric set will contain sampling process timing metrics if T > 0. node-name-map= - | + | | The file name nnmap, as used by ibnetdiscover and opensm, of a mapping from IB GUIDs to short names of IB hardware items (switch, node, etc) suitable for use in populating names of @@ -113,7 +113,7 @@ The lid/port file format is lid, hexguid, nports, plist * where hexguid is 0x...., - * nports is int, + * nports is int, * plist is ints nports long or * if range is 1-nports, * if not using a name map, names will be GUID_hex. @@ -139,20 +139,20 @@ INTERNAL METRICS ========================= port_query_time - | + | | Time in seconds spend in the single port MAD call. port_query_offset - | + | | Time in microseconds from start of all MAD calls in the current update to the end of the mad call for the specific port. ib_query_time - | + | | Time in seconds making all MAD calls in the update. ib_data_process_time - | + | | Time in seconds decoding all MAD data in the update BUGS diff --git a/rtd/man2rst/Plugin_ipmireader.rst b/rtd/man2rst/Plugin_ipmireader.rst index 6cf0e9233..a7c4cd271 100644 --- a/rtd/man2rst/Plugin_ipmireader.rst +++ b/rtd/man2rst/Plugin_ipmireader.rst @@ -42,35 +42,35 @@ attributes of the base class. | configuration line name= - | + | | This MUST be ipmireader. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`ipmireader\`. address=
- | + | | address of the host to contact. H flag in the ipmitool command. username= - | + | | username for the query. U flag in the ipmitool command. Defaults to 'admin'. password= - | + | | password for the query. P flag in the ipmitool command. Defaults to 'password'. sdrcache= - | + | | output for the sdr cache file, to improve performance. Optional. retry= - | + | | interval to retry creating set if initially fails (host down). Default 600 sec. diff --git a/rtd/man2rst/Plugin_ipmisensors.rst b/rtd/man2rst/Plugin_ipmisensors.rst index c885695c6..58d7c2a35 100644 --- a/rtd/man2rst/Plugin_ipmisensors.rst +++ b/rtd/man2rst/Plugin_ipmisensors.rst @@ -43,27 +43,27 @@ attributes of the base class. | configuration line name= - | + | | This MUST be ipmisensors. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`ipmisensors\`. address=
- | + | | address of the host to contact. h flag in the ipmi-sensors command. username= - | + | | username for the query. u flag in the ipmi-sensors command. Defaults to 'admin'. password= - | + | | password for the query. p flag in the ipmi-sensors command. Defaults to 'password'. diff --git a/rtd/man2rst/Plugin_jobid.rst b/rtd/man2rst/Plugin_jobid.rst index 8627358d6..73cb07581 100644 --- a/rtd/man2rst/Plugin_jobid.rst +++ b/rtd/man2rst/Plugin_jobid.rst @@ -38,29 +38,29 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be jobid. producer= - | + | | The producer name value. instance= - | + | | The name of the metric set. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`vmstat\`. component_id= - | + | | Optional component identifier. Defaults to zero. with_jobid= - | + | | Option to lookup job_id with set or 0 if not. The job_id column will always appear, but populated witn zero. diff --git a/rtd/man2rst/Plugin_kgnilnd.rst b/rtd/man2rst/Plugin_kgnilnd.rst index 5a1b2fd25..a238a8a00 100644 --- a/rtd/man2rst/Plugin_kgnilnd.rst +++ b/rtd/man2rst/Plugin_kgnilnd.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be kgnilnd. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`kgnilnd\`. diff --git a/rtd/man2rst/Plugin_linux_proc_sampler.rst b/rtd/man2rst/Plugin_linux_proc_sampler.rst index 9185494e9..9e2cbd280 100644 --- a/rtd/man2rst/Plugin_linux_proc_sampler.rst +++ b/rtd/man2rst/Plugin_linux_proc_sampler.rst @@ -48,51 +48,51 @@ attributes of the base class. | configuration line name= - | + | | This MUST be linux_proc_sampler. instance_prefix=PREFIX - | + | | Prepend PREFIX to the set instance names. Typically a cluster name when needed to disambiguate producer names that appear in multiple clusters. (default: no prefix). exe_suffix=1 - | + | | If present, set instance names are appended with the full path of the executable. This is most likely useful for debugging configuration of the notifier up-stream using ldms_ls. (default: no such suffix) sc_clk_tck=1 - | + | | If present, include sc_clk_tck in the metric set. sc_clk_tck is the ticks per second from sysconf(\_SC_CLK_TCK). (default: not included). stream=STREAM - | + | | The name of the \`ldmsd_stream\` to listen for SLURM job events. (default: slurm). argv_sep= - | + | | Replace nul within the cmdline string with char. Special specifiers \\b,\\n,\\t,\\v,\\r,\\f are also supported. syscalls=FILE - | + | | File mapping syscall integers to symbolic names. Not needed unless syscall_name is included in the metrics. See FILES for details. metrics - | + | | The comma-separated list of metrics to monitor. The default is (empty), which is equivalent to monitor ALL metrics. cfg_file=CFILE - | + | | The alternative configuration file in JSON format. The file is expected to have an object that contains the following attributes: { "stream": "STREAM_NAME", "syscalls" : "/file", @@ -101,7 +101,7 @@ attributes of the base class. on the key=value line are ignored. argv_msg=1 - | + | | Publish the argv items to a stream named _argv, where if the schema is not specified, the default schema is linux_proc_sampler. (Default: argv_msg=0; no publication of @@ -110,13 +110,13 @@ attributes of the base class. store them. argv_fmt=<1,2> - | + | | Publish the argv items formatted as (1) a json list of strings ['argv0', 'argv1'] or (2) a json list of key/value tuples, e.g. [ {"k":0, "v":"argv[0]"}, {"k":1, "v":"argv[1]"}]. env_msg=1 - | + | | Publish the environment items to a stream named _env, where if the schema is not specified, the default SCHEMA is linux_proc_sampler. (Default: env_msg=0; no publication of the @@ -126,7 +126,7 @@ attributes of the base class. messages and store them. env_exclude=ELIST - | + | | Exclude the environment items named with regular expressions in ELIST. On the configuration key=value line, ELIST must be a file name of a file containing a list of regular expressions one per @@ -137,7 +137,7 @@ attributes of the base class. shown in EXAMPLES. fd_exclude=ELIST - | + | | Exclude the files named with regular expressions in ELIST. On the configuration key=value line, ELIST must be a file name of a file containing a list of regular expressions one per line. A @@ -147,7 +147,7 @@ attributes of the base class. array of expression strings as shown in EXAMPLES. fd_msg=N - | + | | Publish new /proc/pid/fd scan data to the _files stream every N-th sample, where if the schema is not specified, the default SCHEMA is linux_proc_sampler. (Default: fd_msg=0; no @@ -165,7 +165,7 @@ attributes of the base class. no corresponding events are generated. published_pid_dir= - | + | | Name of the directory where netlink-notifier or other notifier pids of interest may be found. This directory is scanned at sampler startup only, so that pids which were the subject of diff --git a/rtd/man2rst/Plugin_lnet_stats.rst b/rtd/man2rst/Plugin_lnet_stats.rst index 4de54af8e..949907086 100644 --- a/rtd/man2rst/Plugin_lnet_stats.rst +++ b/rtd/man2rst/Plugin_lnet_stats.rst @@ -40,17 +40,17 @@ attributes of the base class. | ldmsctl configuration line. name= - | + | | This MUST be lnet_stats. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`lnet_stats\`. file= - | + | | Optional full path name of stats file to use. If not supplied, the default search path described in NOTES is searched. Typically, this option is only used in test environments which diff --git a/rtd/man2rst/Plugin_loadavg.rst b/rtd/man2rst/Plugin_loadavg.rst index 8129c1cd8..f56d5ce85 100644 --- a/rtd/man2rst/Plugin_loadavg.rst +++ b/rtd/man2rst/Plugin_loadavg.rst @@ -36,16 +36,16 @@ base class. name= [schema=] [metrics=] [force_integer] name= - | + | | This MUST be loadavg. force_integer - | + | | If present, this flag forces load metrics to be stored as integers of 100*value provided in the proc file. schema= - | + | | Optional schema name. If schema is not specified, it will be computed. The default name is loadavg if the metrics option is not supplied. The default name when metrics is specified is @@ -54,7 +54,7 @@ base class. loadavg prefix becomes loadavgi. metrics= - | + | | comma separated list of metrics to include. If not given, all are included. The complete list is load1min, load5min, load15min, runnable, scheduling_entities, newest_pid. diff --git a/rtd/man2rst/Plugin_lustre_client.rst b/rtd/man2rst/Plugin_lustre_client.rst index bc8bfd6c2..341c4ea55 100644 --- a/rtd/man2rst/Plugin_lustre_client.rst +++ b/rtd/man2rst/Plugin_lustre_client.rst @@ -46,27 +46,27 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_client. job_set= - | + | | The name of the metric set that contains the job id information (default=job_id) producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be $producer/$llite_name. component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. perm= - | + | | Set the access permissions for the metric sets. (default 440). NOTES diff --git a/rtd/man2rst/Plugin_lustre_mdc.rst b/rtd/man2rst/Plugin_lustre_mdc.rst index 78242eb12..7a783a443 100644 --- a/rtd/man2rst/Plugin_lustre_mdc.rst +++ b/rtd/man2rst/Plugin_lustre_mdc.rst @@ -42,28 +42,28 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_mdc. producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be / component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. job_set= - | + | | Optional (defaults to "job_info"). Typically should be set to /jobid or /job_info depending on choice of job sampling plugin. mdc_timing=0 - | + | | Optionally exclude timing data from /sys/kernel/debug/lustre/mdc/\*/stats. If given, the sampler may be run by unprivileged users. If /sys/kernel/debug/ cannot be @@ -71,7 +71,7 @@ CONFIGURATION ATTRIBUTE SYNTAX mdc_timing=0 is given. auto_reset=0 - | + | | Turn off the default behavior of resetting the counters when an overflow condition is detected. Reset is implemented by writing 0 to the corresponding /proc or /sys file. diff --git a/rtd/man2rst/Plugin_lustre_mdt.rst b/rtd/man2rst/Plugin_lustre_mdt.rst index 40052185f..6c1355d51 100644 --- a/rtd/man2rst/Plugin_lustre_mdt.rst +++ b/rtd/man2rst/Plugin_lustre_mdt.rst @@ -55,17 +55,17 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_mdt. producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be $producer/$mdt_name. component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. diff --git a/rtd/man2rst/Plugin_lustre_ost.rst b/rtd/man2rst/Plugin_lustre_ost.rst index 48d2f46fe..3cd09b493 100644 --- a/rtd/man2rst/Plugin_lustre_ost.rst +++ b/rtd/man2rst/Plugin_lustre_ost.rst @@ -55,17 +55,17 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be lustre_ost. producer= - | + | | The default used for producer (if not provided) is the result of gethostname(). The set instance names will be $producer/$ost_name. component_id= - | + | | Optional (defaults to 0) number of the host where the sampler is running. All sets on a host will have the same value. diff --git a/rtd/man2rst/Plugin_meminfo.rst b/rtd/man2rst/Plugin_meminfo.rst index 5b881c215..1cf06fd6d 100644 --- a/rtd/man2rst/Plugin_meminfo.rst +++ b/rtd/man2rst/Plugin_meminfo.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be meminfo. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`meminfo\`. diff --git a/rtd/man2rst/Plugin_msr_interlagos.rst b/rtd/man2rst/Plugin_msr_interlagos.rst index d91642795..d596b0171 100644 --- a/rtd/man2rst/Plugin_msr_interlagos.rst +++ b/rtd/man2rst/Plugin_msr_interlagos.rst @@ -288,11 +288,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be msr_interlagos action= - | + | | Options are initialize, add, finalize, halt, continue, reassign, rewrite, and ls: @@ -303,12 +303,12 @@ attributes of the base class. should be specified at this point. corespernuma= - | + | | Cores per numa node. Used to determine which and how many cores are used in setting counters that report per numa node. maxcore= - | + | | Maxcores that will be reported for all core counters and will also be used in counters that report per numa node. Must be >= actual number of cores. Any additional values will be @@ -316,7 +316,7 @@ attributes of the base class. actual number of cores. schema= - | + | | Schema name. Optional. Defaults to msr_interlagos. **add** @@ -325,12 +325,12 @@ attributes of the base class. the order the metrics are added metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. Options are listed in a separate section of this man page. **finalize** - | + | | creates the set after all the adds. No metrics may be added after this point. @@ -340,7 +340,7 @@ attributes of the base class. for all metrics for this counter. metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. metricname=all halts all. @@ -349,7 +349,7 @@ attributes of the base class. | continues collection for this counter after a halt. metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. metricname=all continues all. @@ -359,7 +359,7 @@ attributes of the base class. has been changed for this address external to ldms. metricname= - | + | | The name of counter e.g., L3_CACHE_MISSES. metricname=all rewrites all counters. @@ -370,16 +370,16 @@ attributes of the base class. counter. oldmetricname= - | + | | The name of counter to be replaced e.g., TOT_CYC newmetricname= - | + | | The name of counter that the previous variable will be replaced with e.g., TOT_INS **ls** - | + | | writes info about the intended counters to the log file. BUGS diff --git a/rtd/man2rst/Plugin_opa2.rst b/rtd/man2rst/Plugin_opa2.rst index 9b7210b1f..f8139f351 100644 --- a/rtd/man2rst/Plugin_opa2.rst +++ b/rtd/man2rst/Plugin_opa2.rst @@ -35,29 +35,29 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be opa2. producer= - | + | | The producer string value. instance= - | + | | The set_name supplied is ignored, and the name $producer/$CA/$port is used. schema= - | + | | Optional schema name. Default opa2. The same schema is used for all sets. component_id= - | + | | Optional component identifier. Defaults to zero. ports= - | + | | Port list is a comma separated list of ca_name.portnum or a '\*'. The default is '\*', which collects a set for every host fabric interface port. diff --git a/rtd/man2rst/Plugin_papi.rst b/rtd/man2rst/Plugin_papi.rst index e50e7a8e8..d4e5cfa54 100644 --- a/rtd/man2rst/Plugin_papi.rst +++ b/rtd/man2rst/Plugin_papi.rst @@ -44,37 +44,37 @@ LDMSCTL CONFIGURATION ATTRIBUTE SYNTAX configuration line name= - | + | | This MUST be spapi. producer= - | + | | The producer string value. instance= - | + | | The name of the metric set schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. component_id= - | + | | Optional component identifier. Defaults to zero. with_jobid= - | + | | Option to collect job id with set or 0 if not. events= - | + | | Comma separated list of events. Available events can be determined using papi_avail command if papi is installed on system. pid - The PID for the process being monitored - | + | NOTES ============= @@ -91,7 +91,7 @@ EXAMPLES ================ The following is a short example that measures 4 events. - | + | | Total CPU cycles | Total CPU instructions | Total branch instructions diff --git a/rtd/man2rst/Plugin_perfevent.rst b/rtd/man2rst/Plugin_perfevent.rst index c0d97196c..8670122c9 100644 --- a/rtd/man2rst/Plugin_perfevent.rst +++ b/rtd/man2rst/Plugin_perfevent.rst @@ -49,28 +49,28 @@ the 'init' action arguments. | configuration line name= - | + | | This MUST be perfevent. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. action=init - | + | | Perform initialization action=del metricname= - | + | | Deletes the specified event. action=ls - | + | | List the currently configured events. action=add metricname= pid= cpu= type= id= - | + | | Adds a metric to the list of configured events. | metricname | The metric name for the event @@ -88,7 +88,7 @@ the 'init' action arguments. | The event id. The pid and cpu arguments allow specifying which process and CPU to monitor: - | + | | pid == 0 and cpu == -1 | This measures the calling process/thread on any CPU. | pid == 0 and cpu >= 0 @@ -109,7 +109,7 @@ the 'init' action arguments. For more information visit: http://man7.org/linux/man-pages/man2/perf_event_open.2.html **type** - | + | | This field specifies the overall event type. It has one of the following values: | PERF_TYPE_HARDWARE @@ -133,7 +133,7 @@ the 'init' action arguments. execution of an instruction address. **id** - | + | | This specifies which event you want, in conjunction with the type field. | There are various ways to set the id field that are dependent on @@ -315,7 +315,7 @@ EXAMPLES ===================== The following is a short example that measures 4 events. - | + | | Total CPU cycles | Total CPU instructions | Total branch instructions diff --git a/rtd/man2rst/Plugin_procdiskstats.rst b/rtd/man2rst/Plugin_procdiskstats.rst index 1d30a6d2a..97185aad6 100644 --- a/rtd/man2rst/Plugin_procdiskstats.rst +++ b/rtd/man2rst/Plugin_procdiskstats.rst @@ -41,17 +41,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procdiskstats. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procdiskstats\`. device= - | + | | Comma separated list of devices BUGS diff --git a/rtd/man2rst/Plugin_procinterrupts.rst b/rtd/man2rst/Plugin_procinterrupts.rst index c03eb33ef..d8aa92f2e 100644 --- a/rtd/man2rst/Plugin_procinterrupts.rst +++ b/rtd/man2rst/Plugin_procinterrupts.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procinterrupts. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procinterrupts\`. diff --git a/rtd/man2rst/Plugin_procnet.rst b/rtd/man2rst/Plugin_procnet.rst index 3e70b7a64..eb9b758d0 100644 --- a/rtd/man2rst/Plugin_procnet.rst +++ b/rtd/man2rst/Plugin_procnet.rst @@ -42,15 +42,15 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnet. exclude_ports= - | + | | Comma separated list of ports to exclude. schema= - | + | | Optional schema name. If not specified, will default to \`procnet\`. diff --git a/rtd/man2rst/Plugin_procnetdev.rst b/rtd/man2rst/Plugin_procnetdev.rst index 1fbe74f06..b0a95b81c 100644 --- a/rtd/man2rst/Plugin_procnetdev.rst +++ b/rtd/man2rst/Plugin_procnetdev.rst @@ -40,16 +40,16 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnetdev. ifaces= - | + | | CSV list of ifaces. Order matters. Non-existent ifaces will be included and default to 0-value data. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics or ifaces have a different schema. If not specified, will default to diff --git a/rtd/man2rst/Plugin_procnetdev2.rst b/rtd/man2rst/Plugin_procnetdev2.rst index 0b55a2b8c..37f39f743 100644 --- a/rtd/man2rst/Plugin_procnetdev2.rst +++ b/rtd/man2rst/Plugin_procnetdev2.rst @@ -40,17 +40,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnetdev2. ifaces= - | + | | (Optional) A CSV list of interfaces to sample. If not specified, all available interfaces in /proc/net/dev will be reported. It is OK to specify non-existing interfaces in the ifaces list. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics or ifaces have a different schema. If not specified, will default to diff --git a/rtd/man2rst/Plugin_procnfs.rst b/rtd/man2rst/Plugin_procnfs.rst index 4ae8b96b1..49bc21ea3 100644 --- a/rtd/man2rst/Plugin_procnfs.rst +++ b/rtd/man2rst/Plugin_procnfs.rst @@ -40,11 +40,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be procnfs. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procnfs\`. diff --git a/rtd/man2rst/Plugin_procstat.rst b/rtd/man2rst/Plugin_procstat.rst index 3416980f4..1c49164e7 100644 --- a/rtd/man2rst/Plugin_procstat.rst +++ b/rtd/man2rst/Plugin_procstat.rst @@ -39,7 +39,7 @@ See ldms_sampler_base(7) for the common sampler options. | configuration line maxcpu= - | + | | Values are 0 to N, where 0 logs only totalized data and N reserves slots for N cores. If less than N cores are found, 0-values are reported. If more than N cores are found, they are @@ -50,7 +50,7 @@ See ldms_sampler_base(7) for the common sampler options. anywhere in the cluster. sc_clk_tck=1 - | + | | Enable optional reporting of sysconf(\_SC_CLK_TCK), the scheduler ticks-per-second defined at kernel build time as CONFIG_HZ, collected from sysconf(3). Typically HPC systems use diff --git a/rtd/man2rst/Plugin_procstat2.rst b/rtd/man2rst/Plugin_procstat2.rst index 95cdfbd8f..88f6896a4 100644 --- a/rtd/man2rst/Plugin_procstat2.rst +++ b/rtd/man2rst/Plugin_procstat2.rst @@ -39,17 +39,17 @@ of the base class. | configuration line name= - | + | | This MUST be procstat2. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`procstat2\`. intr_max= - | + | | (Optional). The maximum number of inerrupt numbers supported in intr_list. If not specified, intr_max will be the current number of interrupts in the intr list. diff --git a/rtd/man2rst/Plugin_rapl.rst b/rtd/man2rst/Plugin_rapl.rst index 42a5674d4..8705cc98d 100644 --- a/rtd/man2rst/Plugin_rapl.rst +++ b/rtd/man2rst/Plugin_rapl.rst @@ -42,11 +42,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be rapl. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`rapl\`. diff --git a/rtd/man2rst/Plugin_rdc_sampler.rst b/rtd/man2rst/Plugin_rdc_sampler.rst index 1fe134eed..8a25f553f 100644 --- a/rtd/man2rst/Plugin_rdc_sampler.rst +++ b/rtd/man2rst/Plugin_rdc_sampler.rst @@ -37,48 +37,48 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be rdc_sampler. producer=. - | + | | The producer string value for the timing set. instance= - | + | | The set instance names will be suffixed by device number (gpu%d). schema= - | + | | Optional schema base name. The default is rdc_sampler. The name base is suffixed to create uniquely defined schema names based on the plugin options specified. component_id= - | + | | Optional component identifier for the timing set. Defaults to zero. metrics=LIST - | + | | The list of values to be collected as named in rdc_field_t from rdc/rdc.h. update_freq=MICROSEC - | + | | An argument passed to rdc_field_watch. max_keep_age=SEC - | + | | An argument passed to rdc_field_watch. max_keep_samples=N - | + | | An argument passed to rdc_field_watch. warmup=K - | + | | Delay K cycles update_freq long before attempting to read data from the gpu. diff --git a/rtd/man2rst/Plugin_sampler_atasmart.rst b/rtd/man2rst/Plugin_sampler_atasmart.rst index 75d05e933..d0eb73577 100644 --- a/rtd/man2rst/Plugin_sampler_atasmart.rst +++ b/rtd/man2rst/Plugin_sampler_atasmart.rst @@ -47,17 +47,17 @@ attributes of the base class. | configuration line name= - | + | | This MUST be sampler_atasmart. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`sampler_atasmart\`. disks - | + | | A comma-separated list of disk names (e.g., /dev/sda,/dev/sda1) BUGS diff --git a/rtd/man2rst/Plugin_slingshot_info.rst b/rtd/man2rst/Plugin_slingshot_info.rst index c93f779d8..804b73ae5 100644 --- a/rtd/man2rst/Plugin_slingshot_info.rst +++ b/rtd/man2rst/Plugin_slingshot_info.rst @@ -50,7 +50,7 @@ attributes of the base class. | configuration line name= - | + | | This MUST be slingshot_info. EXAMPLES diff --git a/rtd/man2rst/Plugin_slingshot_metrics.rst b/rtd/man2rst/Plugin_slingshot_metrics.rst index efbdc6f52..9bd5c2577 100644 --- a/rtd/man2rst/Plugin_slingshot_metrics.rst +++ b/rtd/man2rst/Plugin_slingshot_metrics.rst @@ -48,18 +48,18 @@ attributes of the base class. | configuration line name= - | + | | This MUST be slingshot_metrics. counters= - | + | | (Optional) A CSV list of names of slingshot counter names. See Section COUTNER NAMES for details. If neither this option nor counters_file are specified, a default set of counters will be used. counters_files= - | + | | (Optional) A path to a file that contains a list of counter names, one per line. See Section COUNTER NAMES for details. A line will be consider a comment if the character on the line is @@ -67,7 +67,7 @@ attributes of the base class. default set of counters will be used. refresh_interval_sec= - | + | | (Optional) The sampler caches the list of slinghost devices, and that cache is refreshed at the beginning of a sample cycle if the refresh interval time has been exceeded. diff --git a/rtd/man2rst/Plugin_store_app.rst b/rtd/man2rst/Plugin_store_app.rst index 5b77ec790..237233afa 100644 --- a/rtd/man2rst/Plugin_store_app.rst +++ b/rtd/man2rst/Plugin_store_app.rst @@ -8,12 +8,12 @@ Plugin_store_app :depth: 3 .. -NAME +NAME ================== ldmsd_store_app - LDMSD store_app storage plugin -SYNOPSIS +SYNOPSIS ====================== **load** **name**\ =\ **store_app** @@ -27,7 +27,7 @@ SYNOPSIS **strgp_prdcr_add** **name**\ =\ *STRGP_NAME* **regex**\ =\ *PRDCR_REGEX* -DESCRIPTION +DESCRIPTION ========================= **``store_app``** is an LDMSD storage plugin for storing data from the @@ -66,21 +66,21 @@ attributes: - **``job_time_rank``**: (indexed) the join of **``job_id``**, **``timestamp``**, and **``rank``**. -CONFIG OPTIONS +CONFIG OPTIONS ============================ -name +name The name of the plugin instance to configure. -path +path The path to the directory that contains SOS containers (one container per strgp). -perm +perm The octal mode (e.g. 0777) that is used in SOS container creation. The default is **0660**. -EXAMPLES +EXAMPLES ====================== :: diff --git a/rtd/man2rst/Plugin_store_csv.rst b/rtd/man2rst/Plugin_store_csv.rst index c0fea3629..a0aad9e78 100644 --- a/rtd/man2rst/Plugin_store_csv.rst +++ b/rtd/man2rst/Plugin_store_csv.rst @@ -72,16 +72,16 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_csv. opt_file= - | + | | The options for the plugin and specific instances will be read from the named file. See OPTIONS FILE. path= - | + | | This option is required; the config line or the options file must supply a default value. The output files will be put into a directory whose root is specified by the path argument. This @@ -93,7 +93,7 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX time e.g., XXX/meminfo_ctr/meminfo-123456789. altheader=<0/!0> - | + | | Distinguishes whether or not to write the header to a separate file than the data file. 0 = same file. Any non-zero is a separate file. Default is the same file. If a separate file is @@ -104,7 +104,7 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX written at each rollover. Default is altheader=0. typeheader= - | + | | Controls the presence and format of a .KIND file. The kind CSV file gives type information on each metric (or metric array). For example, if the metric file is named meminfo, the kind file @@ -132,18 +132,18 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX contrast to the repetition employed by format 0. ietfcsv=<0/1> - | + | | Turns on (1) or off (0) use of IETF 4180 quoting for header column names. userdata=<0/!0> - | + | | Distinguishes whether or not to write each metrics' user data along with each data value. 0 = no write. Any non-zero means to write the values. Default is to not write. buffer=<0/1/N> - | + | | Distinguishes whether or not to buffer the data for the writeout. 0 = does not buffer. 1 enables buffering with the system determining the flush. N will flush after approximately N @@ -151,13 +151,13 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX these it is. Default is system controlled buffering (1). buffertype=<3/4> - | + | | If buffer=N then buffertype determines if the buffer parameter refers to kB of writeout or number of lines. The values are the same as in rolltype, so only 3 and 4 are applicable. rolltype= - | + | | By default, the store does not rollover and the data is written to a continously open filehandle. Rolltype and rollover are used in conjunction to enable the store to manage rollover, including @@ -165,26 +165,26 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX roll occurs. Valid options are: 1 - | + | | wake approximately every rollover seconds and roll. Rollover is suppressed if no data at all has been written and rollempty=0. 2 - | + | | wake daily at rollover seconds after midnight (>=0) and roll. Rollover is suppressed if no data at all has been written and rollempty=0. 3 - | + | | roll after approximately rollover records are written. 4 roll after approximately rollover bytes are written. 5 - | + | | wake at rollover seconds after midnight (>=0) and roll, then repeat every rollagain (> rollover) seconds during the day. For example "rollagain=3600 rollover=0 rolltype=5" rolls @@ -192,18 +192,18 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX been written and rollempty=0. rollover= - | + | | Rollover value controls the frequency of rollover (e.g., number of bytes, number of records, time interval, seconds after midnight). Note that these values are estimates. rollempty=0 - | + | | Turn off rollover of empty files. Default value is 1 (create extra empty files). create_perm= - | + | | Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The default permission is 0600 for data files. The mode specified can include execute bits @@ -212,18 +212,18 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX and 0644 for data files. create_uid= - | + | | Specify a new user id for data files. If unspecified, no change in user ownership is made. Changes in ownership of the files do not affect intermediate directories. create_gid= - | + | | Specify a new group id for data files. If unspecified, no change in group ownership is made. rename_template= - | + | | This option relocates closed CSV files, typically to a subdirectory, for processing by other tools that watch directories. The metapath template is applied to define a new @@ -240,14 +240,14 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX output directory and handle completed files. rename_perm= - | + | | Only octal (e.g.0744) specifications are allowed. If unspecified or 0 is given, then no change is made. The permissions are changed before the rename and even if the rename fails. This option is applied only if rename_template is applied. rename_uid= - | + | | Specify a new user id for the file. If unspecified, no change in user ownership is made. Changes in ownership of the files do not affect intermediate directories that might be created following @@ -255,28 +255,28 @@ STORE_CSV CONFIGURATION ATTRIBUTE SYNTAX applied. rename_gid= - | + | | Specify a new group id for the file. If unspecified, no change in group ownership is made. This option is applied only if rename_template is applied. expand_array= - | + | | The default is false. Each array element is stored in a column. True means that all elements are stored in a single column. array_sep= - | + | | Specify a character to separate array elements. If exand_array is true, the value is ignored. array_lquote= - | + | | Specify the left-quote character if expand_array is true. If expand_array is false, the value is ignored. array_rquote= - | + | | Specify the right-quote character if expand_array is true. If expand_array is false, the value is ignored. @@ -312,21 +312,21 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_csv. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). They also are used to match any specific config lines. schema= - | + | | The container and the schema determine where the output files will be written (see path above). You can have multiples of the same sampler, but with different schema (which means they will @@ -334,7 +334,7 @@ output files via identification of the container and schema. containers (and therefore files). decomposition= - | + | | Optionally use set-to-row decomposition with the specified configuration file in JSON format. See more about decomposition in ldmsd_decomposition(7). @@ -373,38 +373,38 @@ The following % escape sequence replacements are performed on the rename_template value for file renamings: %P - | + | | plugin name %C - | + | | container name %S - | + | | schema name %T - | + | | file type (DATA, HEADER, KIND, UNITS, CNAMES, PYNAMES) %B - | + | | basename(closed-file-name) %D - | + | | dirname(closed-file-name) %{ENV_VAR_NAME} - | + | | getenv(ENV_VAR_NAME). The use of undefined or empty environment vars yields an empty substitution, not an error. Characters in the environment variable are restricted to: 'A-Za-z0-9%@()+-\_./:='; other characters present will prevent the rename. %s - | + | | timestamp suffix, if it exists. NOTES diff --git a/rtd/man2rst/Plugin_store_flatfile.rst b/rtd/man2rst/Plugin_store_flatfile.rst index 082c8376a..71b96fe0b 100644 --- a/rtd/man2rst/Plugin_store_flatfile.rst +++ b/rtd/man2rst/Plugin_store_flatfile.rst @@ -40,21 +40,21 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_flatfile. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). They also are used to match any action=custom configuration.node/meminfo. schema= - | + | | The container and schema determines where the output files will be written (see path above). diff --git a/rtd/man2rst/Plugin_store_kafka.rst b/rtd/man2rst/Plugin_store_kafka.rst index ff10f685c..08c58605c 100644 --- a/rtd/man2rst/Plugin_store_kafka.rst +++ b/rtd/man2rst/Plugin_store_kafka.rst @@ -40,7 +40,7 @@ PLUGIN CONFIGURATION Configuration Options: **name=**\ *store_kafka* - | + | | The name of the plugin. This must be **store_kafka**. **path=**\ *KAFKA_CONFIG_JSON_FILE* @@ -63,20 +63,20 @@ STRGP CONFIGURATION strgp options: **name=**\ *NAME* - | + | | The name of the strgp. **plugin=**\ store_kafka - | + | | The plugin must be store_kafka. **container=**\ *KAFKA_SERVER_LIST* - | + | | A comma-separated list of Kafka servers (host[:port]). For example: container=localhost,br1.kf:9898. **decomposition=**\ *DECOMP_CONFIG_JSON_FILE* - | + | | Set-to-row decomposition configuration file (JSON format). See more about decomposition in **ldmsd_decomposition**\ (7). diff --git a/rtd/man2rst/Plugin_store_rabbitkw.rst b/rtd/man2rst/Plugin_store_rabbitkw.rst index ac632cb66..b14499155 100644 --- a/rtd/man2rst/Plugin_store_rabbitkw.rst +++ b/rtd/man2rst/Plugin_store_rabbitkw.rst @@ -41,69 +41,69 @@ user, and pwfile are shared across all metric sets. | These parameters are: name= - | + | | This MUST be store_rabbitkw. routing_key - | + | | The routing key shared by all metric sets is . host= - | + | | The rabbitmq server host. The default is localhost. port= - | + | | The server port on the nearest rabbitmq host. The default is 5672. exchange= - | + | | The amqp exchange to publish with is . The default is amq.topic. This must preexist; the plugin will no cause its creation. vhost= - | + | | The virtual host to be used is . The default is "/". user= - | + | | The amqp username is . The default is "guest". pwfile= - | + | | The file contains the amqp user password in the format 'secretword=password. The default password "guest" is assumed if no file is specified. retry= - | + | | If amqp connection fails due to network or server issue, retry every seconds. Default is 60. heartbeat= - | + | | Heartbeat interval used to detect failed connections. timeout= - | + | | Timeout to use for connections, in milliseconds. Default is 1000. extraprops= - | + | | Turn on (y) or off (n) the use of extra properties with all messages. If AMQP-based filtering is not planned, 'n' will reduce message sizes slightly. logmsg= - | + | | Enable (y) or disable (n, the default) logging all message metric content at the DEBUG level. This is a debugging option. useserver= - | + | | Enable (y, the default) or disable (n) calls to the amqp server; this is a debugging option. @@ -114,16 +114,16 @@ STORE ATTRIBUTE SYNTAX | name= schema= container= name= - | + | | This MUST be store_rabbitkw. schema= - | + | | The name of the metric group, independent of the host name. The schema will be used as a header in messages if extraprops is y. container= - | + | | The container will be used as a header in messages if extraprops is y. @@ -137,12 +137,12 @@ The properties follow the AMQP standard, with LDMS specific interpretations: timestamp - | + | | The sample collection time in MICROSECONDS UTC. Divide by 1,000,000 to get seconds UTC. app_id - | + | | The app_id is LDMS. Optional AMQ event contents @@ -151,19 +151,19 @@ Optional AMQ event contents These fields and headers are present if extraprops=y is configured. content_type - | + | | <"text/plain"> for all. reply_to - | + | | The metric set instance name. container - | + | | The container configuration name. schema - | + | | The schema configuration name. PAYLOAD FORMAT diff --git a/rtd/man2rst/Plugin_store_rabbitv3.rst b/rtd/man2rst/Plugin_store_rabbitv3.rst index e8600a282..e1c59cc35 100644 --- a/rtd/man2rst/Plugin_store_rabbitv3.rst +++ b/rtd/man2rst/Plugin_store_rabbitv3.rst @@ -40,48 +40,48 @@ and pwfile are shared across all metric sets. | These parameters are: name= - | + | | This MUST be store_rabbitv3. root= - | + | | The routing key prefix shared by all metric sets will be . host= - | + | | The rabbitmq server host. The default is localhost. port= - | + | | The server port on the nearest rabbitmq host. The default is 5672. exchange= - | + | | The amqp exchange to publish with is . The default is amq.topic. vhost= - | + | | The virtual host to be used is . The default is "/". user= - | + | | The amqp username is . The default is "guest". pwfile= - | + | | The file contains the amqp user password in the format 'secretword=password. The default password "guest" is assumed if no file is specified. extraprops= - | + | | Turn on (y) or off (n) the use of extra properties with all messages. mint - | + | | The number of seconds between emission of time and host invariant (meta) metrics. @@ -92,15 +92,15 @@ STORE ATTRIBUTE SYNTAX | name= schema= container= name= - | + | | This MUST be store_rabbitv3. schema= - | + | | The name of the metric group, independent of the host name. container= - | + | | The container will be used in the routing key. The current routing key patterns is: .... @@ -121,16 +121,16 @@ The properties follow the AMQP standard, with LDMS specific interpretations: timestamp - | + | | The sample collection time in MICROSECONDS UTC. Divide by 1,000,000 to get seconds UTC. type - | + | | The ldms metric data type. app_id - | + | | The app_id is the integer component_id, if it has been defined by the sampler. @@ -140,36 +140,36 @@ Optional AMQ event contents These fields and headers are present if extraprops=y is configured. content_type - | + | | <"text/plain"> for all. reply_to - | + | | The producer name. metric - | + | | The label registered by the sampler plugin, which might be anything. metric_name_amqp - | + | | The label modified to work as a routing key, not necessarily easily read. metric_name_least - | + | | The label modified to work as a programming variable name, possibly shortened and including a hash suffix. Not expected to be fully human-readable in all cases. It will be the same across runs for metric sets whose content labels do not vary across runs. container - | + | | The container configuration name. schema - | + | | The schema configuration name. PAYLOAD FORMAT diff --git a/rtd/man2rst/Plugin_store_sos.rst b/rtd/man2rst/Plugin_store_sos.rst index 41fc197a6..f09e6e2d7 100644 --- a/rtd/man2rst/Plugin_store_sos.rst +++ b/rtd/man2rst/Plugin_store_sos.rst @@ -38,11 +38,11 @@ STORE_SOS INIT CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_sos. path= - | + | | The store will be put into a directory whose root is specified by the path argument. This directory must exist; the store will be created. The full path to the store will be @@ -61,27 +61,27 @@ container and schema for a store. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_sos. name= - | + | | The policy name for this strgp. container= - | + | | The container and schema define the store as described above (see path). schema= - | + | | The container and schema define the store as described above (see path). You can have multiples of the same path and container, but with different schema (which means they will have different metrics) and they will be stored in the same store. decomposition= - | + | | Optionally use set-to-row decomposition with the specified configuration file in JSON format. See more about decomposition in ldmsd_decomposition(7). @@ -101,11 +101,11 @@ into XXX. Any commands given with no argument, will return usage info. | Create a partition. **-C** ** - | + | | Path to the container **-s** *state* - | + | | State of the new partition (case insensitive). Default is OFFLINE. Optional parameter. Valid options are: @@ -117,7 +117,7 @@ into XXX. Any commands given with no argument, will return usage info. or deleted. **part_name** - | + | | Name of the partition **sos_part_delete** @@ -126,11 +126,11 @@ into XXX. Any commands given with no argument, will return usage info. OFFLINE state to be deleted. **-C** ** - | + | | Path to the container **name** - | + | | Name of the parition **sos_part_modify** @@ -138,11 +138,11 @@ into XXX. Any commands given with no argument, will return usage info. | Modify the state of a partition. **-C** ** - | + | | Path to the container **-s** *state* - | + | | State of the new partition (case insensitive). Default is OFFLINE. Optional parameter. Valid options are: @@ -154,24 +154,24 @@ into XXX. Any commands given with no argument, will return usage info. or deleted. **part_name** - | + | | Name of the partition **sos_part_move** - | + | | Move a partition to another storage location. -C -p part_name **-C** ** - | + | | Path to the container **-p** ** - | + | | The new path. **part_name** - | + | | Name of the partition USING SOS COMMANDS TO LOOK AT DATA IN A PARTITION @@ -185,7 +185,7 @@ command options are below. Example usage is in the example section. | Print a directory of the schemas. **-C** ** - | + | | Path to the container **sos_cmd** @@ -193,7 +193,7 @@ command options are below. Example usage is in the example section. | Show debug information for the container **-C** ** - | + | | Path to the container **sos_cmd** @@ -201,22 +201,22 @@ command options are below. Example usage is in the example section. | Print data from a container **-C** ** - | + | | Path to the container **-q** Used to query **-S** ** - | + | | Schema querying against **-X** ** - | + | | Variable that is indexed to use in the query. **-V** ** - | + | | One or more vars to output. NOTES diff --git a/rtd/man2rst/Plugin_store_timescale.rst b/rtd/man2rst/Plugin_store_timescale.rst index d020281b5..06a846870 100644 --- a/rtd/man2rst/Plugin_store_timescale.rst +++ b/rtd/man2rst/Plugin_store_timescale.rst @@ -44,37 +44,37 @@ STORE_TIMESCALE CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_timescale. user= - | + | | This option is required; It will be used as the user name to connect to timescaledb. pwfile= - | + | | This option is required; The file must have content secretword=, the password will be used as the password to connect to timescaledb. hostaddr= - | + | | This option is required; It will be used as the ip addr of timescaledb to connect to. port= - | + | | This option is required; It will be used as the port number of timescaledb to connect to. dbname= - | + | | This option is required; It will be used as the timescaledb database name to connect to. measurement_limit= - | + | | This is optional; It specifies the maximum length of the sql statement to create table or insert data into timescaledb; default 8192. @@ -91,20 +91,20 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_timescale. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). schema= - | + | | The container and the schema determine where the output files will be written (see path above). You can have multiples of the same sampler, but with different schema (which means they will diff --git a/rtd/man2rst/Plugin_store_tutorial.rst b/rtd/man2rst/Plugin_store_tutorial.rst index 9f73d3f02..5959adff4 100644 --- a/rtd/man2rst/Plugin_store_tutorial.rst +++ b/rtd/man2rst/Plugin_store_tutorial.rst @@ -43,11 +43,11 @@ STORE_TUTORIAL CONFIGURATION ATTRIBUTE SYNTAX | ldmsd_controller configuration line name= - | + | | This MUST be store_tutorial. path= - | + | | This option is required; the config line or the options file must supply a default value. The output files will be put into a directory whose root is specified by the path argument. This @@ -68,20 +68,20 @@ output files via identification of the container and schema. | ldmsd_controller strgp_add line plugin= - | + | | This MUST be store_tutorial. name= - | + | | The policy name for this strgp. container= - | + | | The container and the schema determine where the output files will be written (see path above). schema= - | + | | The container and the schema determine where the output files will be written (see path above). You can have multiples of the same sampler, but with different schema (which means they will diff --git a/rtd/man2rst/Plugin_stream_csv_store.rst b/rtd/man2rst/Plugin_stream_csv_store.rst index ef9b246de..0a80eedb3 100644 --- a/rtd/man2rst/Plugin_stream_csv_store.rst +++ b/rtd/man2rst/Plugin_stream_csv_store.rst @@ -40,35 +40,35 @@ CONFIGURATION ATTRIBUTE SYNTAX | configuration line name= - | + | | This MUST be stream_csv_store. path= - | + | | path to the directory of the csv output file container= - | + | | directory of the csv output file stream= - | + | | csv list of streams to which to subscribe. flushtime= - | + | | Flush any file that has not received data on its stream in the last N sec. This is asynchonous to any buffering or rollover that is occuring. Min time if enabled = 120 sec. This will occur again at this interval if there is still no data received. buffer=<0/1> - | + | | Optional buffering of the output. 0 to disable buffering, 1 to enable it with autosize (default) rolltype= - | + | | By default, the store does not rollover and the data is written to a continously open filehandle. Rolltype and rollover are used in conjunction to enable the store to manage rollover, including @@ -76,25 +76,25 @@ CONFIGURATION ATTRIBUTE SYNTAX roll occurs. Valid options are: 1 - | + | | wake approximately every rollover seconds and roll. Rollover is suppressed if no data at all has been written. 2 - | + | | wake daily at rollover seconds after midnight (>=0) and roll. Rollover is suppressed if no data at all has been written. 3 - | + | | roll after approximately rollover records are written. 4 - | + | | roll after approximately rollover bytes are written. 5 - | + | | wake at rollover seconds after midnight (>=0) and roll, then repeat every rollagain (> rollover) seconds during the day. For example "rollagain=3600 rollover=0 rolltype=5" rolls @@ -102,7 +102,7 @@ CONFIGURATION ATTRIBUTE SYNTAX been written. rollover= - | + | | Rollover value controls the frequency of rollover (e.g., number of bytes, number of records, time interval, seconds after midnight). Note that these values are estimates due to the @@ -166,7 +166,7 @@ Options for timing information are driven by #defines in the code source right now. TIMESTAMP_STORE - | + | | Set by #define or #undef TIMESTAMP_STORE. This will write out an absolute timestamp in the file as the last item in the csv and is called 'store_recv_time' in the header. The timestamp is only @@ -176,7 +176,7 @@ TIMESTAMP_STORE and json are timestamped. STREAM_CSV_DIAGNOSTICS - | + | | Set by #define or #undef STREAM_CSV_DIAGNOSTICS. This will write out diagnostic info to the log when stream_cb is called. diff --git a/rtd/man2rst/Plugin_sysclassib.rst b/rtd/man2rst/Plugin_sysclassib.rst index 31725181a..d93c04b0b 100644 --- a/rtd/man2rst/Plugin_sysclassib.rst +++ b/rtd/man2rst/Plugin_sysclassib.rst @@ -34,20 +34,20 @@ specific to the this plugin; see ldms_sampler_base.man for the attributes of the base class. **config**\ name=\ **[schema=]**\ ports=\ **[metrics_type=]** - | + | | configuration line name= - | + | | This MUST be sysclassib. metrics_type= - | + | | Values are 0 or 1. 0 = counter data only. 1 = include rate data (per second) in addition. Default is 0. ports= - | + | | CSV list of the form CARD1.PORT1,CARD2.PORT2. Default is all discovered values. diff --git a/rtd/man2rst/Plugin_tutorial_sampler.rst b/rtd/man2rst/Plugin_tutorial_sampler.rst index c549bb570..2a0188d7a 100644 --- a/rtd/man2rst/Plugin_tutorial_sampler.rst +++ b/rtd/man2rst/Plugin_tutorial_sampler.rst @@ -45,18 +45,18 @@ attributes of the base class. | configuration line name= - | + | | This MUST be tutorial_sampler. num_metrics= - | + | | Optional number of metrics for this set. Metrics will be U64. Metric names will be 'metric\_%d'. If not specified, default number of metrics is determined by DEFAULTNUMMETRICS in the source. schema= - | + | | Optional schema name. It is intended that any sets with different metrics have a different schema. If not specified, will default to \`tutorial_sampler\`. Therefore, if you are @@ -76,10 +76,10 @@ Within ldmsd_controller or a configuration file: :: config name=tutorial_sampler producer=localhost1 instance=localhost1/test1 schema=test1 component_id=1 - config name=tutorial_sampler producer=localhost1 instance=localhost1/test2 schema=test2 component_id=2 num_metrics=5 - config name=tutorial_sampler producer=localhost1 instance=localhost1/test3 schema=test3 component_id=1 num_metrics=2 - job_set=localhost1/jobid - start name=tutorial_sampler interval=1000000 + config name=tutorial_sampler producer=localhost1 instance=localhost1/test2 schema=test2 component_id=2 num_metrics=5 + config name=tutorial_sampler producer=localhost1 instance=localhost1/test3 schema=test3 component_id=1 num_metrics=2 + job_set=localhost1/jobid + start name=tutorial_sampler interval=1000000 > ldms_ls localhost1/test1: consistent, last update: Thu Oct 24 10:55:14 2019 -0600 [223680us] M u64 component_id 1 D u64 job_id 0 D u64 app_id 0 diff --git a/rtd/man2rst/Plugin_tx2mon.rst b/rtd/man2rst/Plugin_tx2mon.rst index 4acdcdd4f..0f41c1a73 100644 --- a/rtd/man2rst/Plugin_tx2mon.rst +++ b/rtd/man2rst/Plugin_tx2mon.rst @@ -37,7 +37,7 @@ tx2mon are listed here [auto-schema=] schema= - | + | | Optional schema name. It is required by most storage backends that the same sampler on different nodes with different metric subsets needs to have a unique schema name. Use auto-schema=1 @@ -45,20 +45,20 @@ tx2mon are listed here backend requirement. auto-schema= - | + | | If true, change the schema name to tx2mon\_$X, where $X will be a unique value derived from the data selection options. If both schema and auto-schema=1 are given, the schema name given is used as the base instead of "tx2mon". array= - | + | | For per-core data, report all array value elements if true. Report only maximum and minimum values if false. The default is false. extra= - | + | | For per-core data, report additional information of the internal block frequencies and the set system metrics. These additional values are static. If false, additional information will not be diff --git a/rtd/man2rst/Plugin_variable.rst b/rtd/man2rst/Plugin_variable.rst index 5144c9ad1..c0514fbeb 100644 --- a/rtd/man2rst/Plugin_variable.rst +++ b/rtd/man2rst/Plugin_variable.rst @@ -39,17 +39,17 @@ instance name. | configuration line name= - | + | | This MUST be variable. schema= - | + | | Optional schema name prefix. The string given will be suffixed with an integer N in the range 1-9 to create the schema name. The schema will also contain N integer metrics. instance= - | + | | Optional instance name prefix. The string given will be suffixed with an integer in the range 1-9 to create the instance name. If not specified, will default prefix is \`$HOST/variable\`. diff --git a/rtd/man2rst/Plugin_variorum_sampler.rst b/rtd/man2rst/Plugin_variorum_sampler.rst index 58ab0d0ed..43f01b5a8 100644 --- a/rtd/man2rst/Plugin_variorum_sampler.rst +++ b/rtd/man2rst/Plugin_variorum_sampler.rst @@ -54,11 +54,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be variorum_sampler. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`variorum_sampler\`. diff --git a/rtd/man2rst/Plugin_vmstat.rst b/rtd/man2rst/Plugin_vmstat.rst index 6e0f5864c..bbfd2f0ee 100644 --- a/rtd/man2rst/Plugin_vmstat.rst +++ b/rtd/man2rst/Plugin_vmstat.rst @@ -39,11 +39,11 @@ attributes of the base class. | configuration line name= - | + | | This MUST be vmstat. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. If not specified, will default to \`vmstat\`. diff --git a/rtd/man2rst/ldms-csv-anonymize.rst b/rtd/man2rst/ldms-csv-anonymize.rst index da3d0f443..fe14aebe4 100644 --- a/rtd/man2rst/ldms-csv-anonymize.rst +++ b/rtd/man2rst/ldms-csv-anonymize.rst @@ -36,12 +36,12 @@ OPTIONS ====================== --input= - | + | | Args is a file name or space-separated list of file names to be processed. Filenames cannot contain whitespace. --out-dir= - | + | | Path is a directory (must pre-exist and should not be the same as any directory containing the input) which will be filled with the changed files. The original files will are not changed. If an @@ -49,30 +49,30 @@ OPTIONS may be lost or corrupted. --col-sep= - | + | | Split columns at this character. The default is comma. ---save-maps= - | +--save-maps= + | | The path prefix for the generated map files. If the resulting map filenames coincide with an existing file, the existing file is overwritten. --imap= - | + | | An integer mapping file to preload. It must contain two columns of integers and magic. Normally it is the output of a prior run. See MAPS below. --nmap= - | + | | A name mapping file to preload. It must contain two columns of names and magic. Normally it is the output of a prior run. Each real name is replaced with 'n' and a sequential number. See MAPS below. --pmap= - | + | | A path element mapping file to preload. It must contain two columns of path elements and magic. Normally it is the output of a prior run. Path elements are unqualified subdirectory names. Each unique @@ -81,7 +81,7 @@ OPTIONS application identities. See MAPS below. --hmap= - | + | | A host name mapping file to preload. It must contain columns of host elements and magic. It may be host name fragment information or the output of a prior run. Any hostname found in the input data @@ -90,7 +90,7 @@ OPTIONS See MAPS below. --gen-args=[,M:H]\*, - | + | | Creating the M:C specification needed in a data transformation run can be done by first using the argument generation mode. Given a file starting with a header line of column names and the list of @@ -98,11 +98,11 @@ OPTIONS M:C arguments needed for the data transformation. --debug - | + | | Echo some details of the transformation as it runs. --seed - | + | | Supply a seed to the random number generator. No random values are used at this time in the processing, however. diff --git a/rtd/man2rst/ldms-csv-export-sos.rst b/rtd/man2rst/ldms-csv-export-sos.rst index 36d5b5bd2..28864c4bc 100644 --- a/rtd/man2rst/ldms-csv-export-sos.rst +++ b/rtd/man2rst/ldms-csv-export-sos.rst @@ -35,7 +35,7 @@ OPTIONS ======================= --data= - | + | | DATA is a file name of a LDMS .HEADER, .KIND, or data file. The file name and at least the first line of the file are digested to determine the content and the column types. LDMS CSV file name @@ -45,71 +45,71 @@ OPTIONS must also be gzipped. --blacklist= - | + | | BLACKLIST is the name of a file with column names to exclude from the schema, one per line. leading # comments allowed in the file. --whitelist= - | + | | WHITELIST is the name of a file with column names to include in the schema, one per line. leading # comments allowed in the file. Any other columns found are excluded. --exclude= - | + | | LIST is a string of metric names separated by commas. Columns named are excluded from the generated schema. --include= - | + | | LIST is a string of metric names separated by commas. Columns named are included in the generated schema and all other columns found are excluded. --schema-name= - | + | | NAME overrides the default schema name determined from the data file name. --schema-file= - | + | | Use an existing schema file FILE instead of generating a schema. When not specified, a schema file is always generated. Schema files may not be gzipped. ---map-file= - | +--map-file= + | | Override the output map file name derived from the data file name. ---alias-file= - | +--alias-file= + | | Provide the list of metrics to rename when creating or matching a schema discovered from a header line. ---strip-udata - | +--strip-udata + | | Suppress output of .userdata fields and remove .value suffix from schema element names. --guess - | + | | Guess the ldms data column types. (can be slow on large files) --maxlines= - | + | | Parse no more than MAXLINES to guess data types with the --guess option. The default if unspecified is 100000 lines. --assume= - | + | | Assume all unknown data columns are type ASSUME. --verbose - | + | | Show process debugging details. --widen - | + | | Widen numeric types discovered to 64 bits. METRIC FILTERING @@ -194,11 +194,11 @@ Other examples --schema-file=fptrans.SCHEMASOS # test input guess when x.14 does not exist - ldms-csv-export-sos --data=x.HEADER.14 --guess + ldms-csv-export-sos --data=x.HEADER.14 --guess # test input guess when y.KIND.14 does not exist but y.14 does ldms-csv-export-sos --data=y.HEADER.14 \ - --guess --maxlines=4000 + --guess --maxlines=4000 # test input guess and widen ldms-csv-export-sos --data=y.HEADER.14 \ diff --git a/rtd/man2rst/ldms-ibnet-sampler-gen.rst b/rtd/man2rst/ldms-ibnet-sampler-gen.rst index a6b770516..349134bae 100644 --- a/rtd/man2rst/ldms-ibnet-sampler-gen.rst +++ b/rtd/man2rst/ldms-ibnet-sampler-gen.rst @@ -79,7 +79,7 @@ EXAMPLES ldms-ibnet-sampler-gen --lidnames --net cluster-p-netdiscover --samplers x --out x | sort -k2 -t, > lid.host.txt - ldms-ibnet-sampler-gen --net cluster-p-netdiscover --samplers clustre-samplers --sharp 37 --annotate --out sbx + ldms-ibnet-sampler-gen --net cluster-p-netdiscover --samplers clustre-samplers --sharp 37 --annotate --out sbx :: diff --git a/rtd/man2rst/ldms-plugins.rst b/rtd/man2rst/ldms-plugins.rst index 47448cbca..ef9d57b0e 100644 --- a/rtd/man2rst/ldms-plugins.rst +++ b/rtd/man2rst/ldms-plugins.rst @@ -32,11 +32,11 @@ The names all, store, and sampler are interpreted as described in ldmsd(8). -b - | + | | Produce brief output, omitting usages. -n - | + | | Produce names only. EXAMPLES diff --git a/rtd/man2rst/ldms-run-static-tests.rst b/rtd/man2rst/ldms-run-static-tests.rst index 78c6e8452..774b543f7 100644 --- a/rtd/man2rst/ldms-run-static-tests.rst +++ b/rtd/man2rst/ldms-run-static-tests.rst @@ -40,11 +40,11 @@ OPTIONS ========================= -l - | + | | List the enabled plugins. -h - | + | | List help message. LANGUAGE @@ -60,19 +60,19 @@ Uses the current set environment to run. Environment may need to be configured before excuting this test script. input - | + | | The name of the input file as specified when ldms-static-test.sh is invoked for each enabled plugin. testname - | + | | The base name (directories stripped) of the input file name. This variable makes it possible to use similar input across many test files when the name of the input file is the same as the plugin tested. strict - | + | | If the variable "strict" is used for KILL_LDMSD (ldms-static-test(8)) the script will output "FAIL $testname" and return an XFAIL to indicate an expected failure only if the test @@ -81,7 +81,7 @@ strict test.$testname.log under the default output location of test_dir. file - | + | | The file "static-test-list" located in ldms/scripts/ defines a list of samplers that are expected to fail. If there is a failed test and the sampler is listed in this file, then run-static-test.sh @@ -89,7 +89,7 @@ file list to meet their needs. bypass <1,0> - | + | | This variable assignment is used to determine an expected failure (1) or normal failure (0) of a sampler plugin. This variable is set to (1) if the sampler is listed in $file and set to (0) otherwise. @@ -106,13 +106,13 @@ FILES ======================= *$input_file.$i* - | + | | For each value of i specifed to start an ldmsd, a configuration file named $input_file.$i must also exist. This configuration file is used when starting the daemon. *test_dir* - | + | | Test output directory of ldms-static-test.sh. The default output location is \`pwd`/ldmstest/$testname. @@ -120,11 +120,11 @@ GENERATED FILES ================================= *$test_dir/test.$testname.log* - | + | | The log file containing stderr and stdout of ldms-static-test.sh. *$test_dir/test.$testname.tgz* - | + | | Location of the compressed file logfile. SEE ALSO diff --git a/rtd/man2rst/ldms-sensors-config.rst b/rtd/man2rst/ldms-sensors-config.rst index f38bd8802..f6ac04a51 100644 --- a/rtd/man2rst/ldms-sensors-config.rst +++ b/rtd/man2rst/ldms-sensors-config.rst @@ -36,29 +36,29 @@ OPTIONS ======================= --sensors= - | + | | specify an alternate location of the sensors program. The default is /usr/bin/sensors, and the PATH variable is not used to search for alternatives. --nodash - | + | | Replace all - characters in metric names with \_ characters. --lscpu= - | + | | specify an alternate location of the lscpu program. The default is /usr/bin/lscpu and the PATH variable is not used to search for alternatives. --test-lscpu= - | + | | Specify the location of a pre-collected strace log of lscpu to use instead of lscpu run on the local system. Used for testing or remote configuration. --test-sensors= - | + | | Specify the location of a pre-collected strace log of sensors to use instead of sensors run on the local system. Used for testing or remote configuration. diff --git a/rtd/man2rst/ldms-static-test.rst b/rtd/man2rst/ldms-static-test.rst index 34ae91ec9..486a91f55 100644 --- a/rtd/man2rst/ldms-static-test.rst +++ b/rtd/man2rst/ldms-static-test.rst @@ -37,11 +37,11 @@ OPTIONS ==================== -l - | + | | List the canned tests available. -h - | + | | List help message. LANGUAGE @@ -51,7 +51,7 @@ The following macro language is provided as extensions on bash. Other bash use is also possible, but not recommended. DAEMONS - | + | | Give all the numbers that will be used in the LDMSD invocations anywhere in the test. This causes port variables to be defined so that any daemon can connect to any other by referencing $portN as @@ -59,7 +59,7 @@ DAEMONS aggregation relationships of LDMSD calls may be infeasible. LDMSD [conf-options] - | + | | This starts a number of daemons described by daemon-numbers. The numbers can be a given list, such as "1 2 3". The environment of each daemon (and its config script) will contain the variable i set @@ -70,25 +70,25 @@ LDMSD [conf-options] See CONFIGURATION OPTIONS below for the explanation of [conf-options]. MESSAGE [arguments] - | + | | The expanded arguments are logged. LDMS_LS [ldms_ls_args] - | + | | This invokes ldms_ls on the k-th ldmsd. KILL_LDMSD [strict] - | + | | Kills the listed daemons. If optional keyword strict is supplied, then missing daemons will cause the bypass variable to be set and the script to include an error code when it exits. SLEEP - | + | | Sleeps n seconds and logs a message about it. JOBDATA jobfile [daemon-numbers] - | + | | Creates jobfile with data for the jobid plugin to parse. If daemon numbers are specified, creates a jobfile.$k for each value of k listed in daemon-numbers. Each file will have unique numeric @@ -96,32 +96,32 @@ JOBDATA jobfile [daemon-numbers] slurm-plugin sampler binary format. vgon - | + | | Turns on use of valgrind for any ldmsd or ldms_ls subsequently started. vgoff - | + | | Turns off use of valgrind for any ldmsd or ldms_ls subsequently started. file_created - | + | | Verifies the existence and readability of filename. rollover_created - | + | | Verifies the existence and readability of rollover files matching pattern filename.[0-9]\*. bypass=<0,1> - | + | | This variable assignment disables (1) or enables (0) all the macros described above. Typical use is to skip one or more operations while debugging a test script. KILL_LDMSD_STRICT=<0,1> - | + | | This variable allows the script author to control whether KILL_LDMSD is strict by default or not. If enabled (1), the script will exit with error code 1 following a failed KILL_LDMSD. If @@ -131,7 +131,7 @@ KILL_LDMSD_STRICT=<0,1> the numeric arguments to KILL_LDMSD also sets KILL_LDMSD_STRICT=1. portbase= - | + | | The listening port numbers assigned to the daemons will be K+i, where i is as described for macro LDMSD. It is a good idea (to support automated testing) if portbase is set in so @@ -145,13 +145,13 @@ The LDMSD command supports the following options. Note that all -P options are processed before all -p options in a single LDMSD call. -p - | + | | The prolog file is included before the usually expected input file. The location of prolog files is handled as are the test input files. See FILES below. Multiple -p options are allowed. -P - | + | | The looped-prolog-file is included before the usually expected input file, once for each value in daemon-csl. Daemon-csl is a comma separated list of daemon numbers, e.g. a complete argument @@ -162,14 +162,14 @@ The location of looped prolog files is handled as are the test input files. See FILES below. Multiple -P options are allowed. -c - | + | | Where multiple daemon numbers are specified, the input generated for the first number is cloned to all subsequent daemons. See FILES. This allows a single file to serve many similar daemon instances in scale testing. -s - | + | | After an ldmsd is started, wait wait_microseconds before checking for the daemon PID file to exist. The appropriate wait time is variable depending on the complexity of the configuration. If not @@ -182,31 +182,31 @@ The following variables can be set in the script to affect the launch of ldmsd: LDMSD_EXTRA - | + | | If set, these arguments are are appended to the ldmsd launch. Typical use is to specify "-m MEMSIZE" or other unusual arguments. The following flags are always determined for the user and must not be present in LDMSD_EXTRA: -x -c -l -v -r. VG - | + | | If valgrind is used (see vgon, vgoff), then $VG is the name of the debugging tool wrapped around the launch of ldmsd. The default is 'valgrind'. VGARGS - | + | | If valgrind is used (see vgon, vgoff), then $VGARGS is appended to the default valgrind arguments. VGTAG - | + | | If valgrind is used (see vgon, vgoff), then $VGTAG is inserted in the valgrind output file name when defined. A good practice is for VGTAG to start with ".". KILL_NO_TEARDOWN - | + | | Set KILL_NO_TEARDOWN=1 to suppress attempting configuration cleanup during KILL_LDMSD. If set, ldmsd internal cleanup() function will attempt partial cleanup, but possibly leave active data structures @@ -214,52 +214,52 @@ KILL_NO_TEARDOWN the input file and the configuration file. i - | + | | Daemon configuration files and commands can refer to ${i} where i is the integer daemon number supplied via LDMSD for the specific daemon using the script. portN - | + | | Daemon configuration files and commands can refer to ${portN} where N is any value of 'i' described above. portN is the data port number of the N-th daemon. input - | + | | The name of the input file as specified when invoking this command. testname - | + | | The base name (directories stripped) of the input file name. This variable makes it possible to use similar input across many test files when the name of the input file is the same as the plugin tested. TESTDIR - | + | | Root directory of the testing setup. STOREDIR - | + | | A directory that should be used for store output configuration. LOGDIR - | + | | A directory that should be used for log outputs. LDMS_AUTH_FILE - | + | | Secret file used for daemon communication. XPRT - | + | | The transport used. It may be specified in the environment to override the default 'sock', and it is exported to the executed daemon environment. HOST - | + | | The host name used for a specific interface. It may be specified in the environment to override the default 'localhost', and it is exported to the executed daemon environment. @@ -281,7 +281,7 @@ FILES ================== *$input_file.$i* - | + | | For each value of i specifed to start an ldmsd, a configuration file named $input_file.$i must also exist. This configuration file is used when starting the daemon. @@ -293,12 +293,12 @@ the first are ignored. Where prologs are also specified, the regular prolog inclusion process is applied to the first file. *[test_dir]* - | + | | If test_dir is supplied, it is used as the test output directory. The default output location is \`pwd`/ldmstest/$testname. *$docdir/examples/static-test/$input_file* - | + | | If input_file is not found in the current directory, it is checked for in $docdir/examples/static-test/$input_file. @@ -311,35 +311,35 @@ GENERATED FILES for ldms_ls of the kth daemon with PID %p, if valgrind is active. *$test_dir/logs/$k.txt* - | + | | The log for the kth daemon. *$test_dir/logs/teardown.$k.txt* - | + | | The teardown log for the kth daemon. *$test_dir/run/conf.$k* - | + | | The input for the kth daemon. *$test_dir/run/revconf.$k* - | + | | The input for the kth daemon teardown. *$test_dir/run/env.$k* - | + | | The environment present for the kth daemon. *$test_dir/run/start.$k* - | + | | The start command of the kth daemon. *$test_dir/store/* - | + | | The root of store output locations. *$test_dir/run/ldmsd/secret* - | + | | The secret file for authentication. SEE ALSO diff --git a/rtd/man2rst/ldms_build_install.rst b/rtd/man2rst/ldms_build_install.rst index 0ede02ebd..e212f2ca7 100644 --- a/rtd/man2rst/ldms_build_install.rst +++ b/rtd/man2rst/ldms_build_install.rst @@ -94,20 +94,20 @@ A number of top level "enable|disable-feature" options exist. The defaults are chosen for a generic linux build to work by default. **--enable|disable-rpath** - | + | | Disable this. Do not hardcode runtime library paths. **--enable|disable-ldms** - | + | | Enable this. Default enabled. **--enable|disable-sos** - | + | | Used to enable or disable sos. Enable only if you are going to use the store_sos plugin. Default disable. **--enable|disable-ocm|baler|me|komondor** - | + | | Disable all of these. All default disabled. OVIS_LIB LEVEL OPTIONS @@ -117,19 +117,19 @@ A number of top level "enable|disable-feature" options exist. The defaults are chosen for a generic linux build to work by default. **--enable|disable-auth** - | + | | Enables or disables authentication. Default enabled. **--enable|disable-sock** - | + | | Enables or disables the sock transport. Default enabled. **--enable|disable-rdma** - | + | | Enables or disables the rdma transport. Default disabled **--enable|disable-ugni** - | + | | Enables or disables the ugni transport. The is cray-specific for rdma over gemini or aries. Default disabled. @@ -144,11 +144,11 @@ General Options --------------- **--enable|disable-ovis_auth** - | + | | If --enable, then disable/enable authentication. Default enabled. **--enable|disable-python** - | + | | Enable the ldms python api and the configuration tools that depend on the API. Default: enabled if python and cython detected. **--enable|disable-readline** @@ -158,41 +158,41 @@ General Options method), then this can be disabled. **--with-libevent**\ *[=path]* - | + | | Specify libevent path [default=/usr] Generic Sampler Options ----------------------- **--enable|disable-meminfo|procinterrupts|procnfs|procnetdev|vmstat** - | + | | Enable or disable generic linux samplers for data in /proc. Default enabled. **--enable|disable-lustre** - | + | | Enable or disable the lustre module. Default enabled. Cray-specific Sampler Options ----------------------------- **--enable|disable-kgnilnd** - | + | | Enable the kgnilnd sampler. Default disabled. **--enable|disable-cray_system_sampler** - | + | | Enable or disable the cray_system_sampler module. Default disabled. If you enable this, then consider the following options: **--enable-gemini-gpcdr** - | + | | Enable the gemini-gpcdr version of the cray_system_sampler. Default disabled. Both the gemini and aries versions can be built simultaneously. **--enable-aries-gpcdr** - | + | | Enable the aries-gpcdr version of the cray_system_sampler. Default disabled. For the Aries, we recommended getting the HSN metrics via aries-mmr, instead of the aries-gpcdr sampler. Still @@ -201,39 +201,39 @@ Cray-specific Sampler Options be built simultaneously. **--enable-cray-nvidia**\ OR\ **--with-cray-nvidia-inc**\ [=path] - | + | | For gemini systems with gpus, Enable the cray-nvidia metric sampling in the cray_gemini_r_sampler. You need not specify --enable-cray-nvidia if you are instead specifying the path to the include file via --with-cray-nvidia-inc. **--enable|disable-lustre** - | + | | Enable or disable the lustre module for use in the cray_system_sampler. Default enabled. **--with-rca**\ *[=path]* - | + | | Specify the path to the rca includes via --with-rca [default=/usr]. **--with-krca**\ *[=path]* - | + | | Specify the path to the krca includes via --with-krca [default=/usr]. **--with-cray-hss-devel**\ *[=path]* - | + | | Specify the path to the hss-devel includes via --with-cray-hss-devel [default=/usr]. **--enable|disable-aries-mmr** - | + | | Enable or disable the aries-mmr module. Default disabled. If you enable this, then consider the following options: **--with-aries-libgpcd**\ *LIBDIR,INCDIR* - | + | | Locations of gpcd library and headers for aries_mmr sampler. E.g. --with-aries-libgpcd=/special/libs,/private/headerdir @@ -241,7 +241,7 @@ Store Options ------------- **--enable|disable-csv** - | + | | Enable the csv stores (store_csv and store_function_csv). Default enable. **--enable|disable-sos** | Enable or disable the sos stores. Enable this only if you are going diff --git a/rtd/man2rst/ldms_ibnet_schema_name.rst b/rtd/man2rst/ldms_ibnet_schema_name.rst index 43b75847b..b340837da 100644 --- a/rtd/man2rst/ldms_ibnet_schema_name.rst +++ b/rtd/man2rst/ldms_ibnet_schema_name.rst @@ -50,15 +50,15 @@ EXAMPLES flowctlcounters vloppackets vlopdata - vlxmitflowctlerrors - vlxmitcounters + vlxmitflowctlerrors/t + vlxmitcounters/t swportvlcong - rcvcc + rcvcc/t slrcvfecn slrcvbecn - xmitcc + xmitcc/t vlxmittimecc - smplctl + smplctl/t yields diff --git a/rtd/man2rst/ldms_ls.rst b/rtd/man2rst/ldms_ls.rst index e04c63011..39dfc5024 100644 --- a/rtd/man2rst/ldms_ls.rst +++ b/rtd/man2rst/ldms_ls.rst @@ -58,19 +58,19 @@ OPTIONS If the NAME is specified on the command line without -E/-S/-I, only information for that instance = NAME is displayed. **-E** *NAME* - | + | | Indicates that the NAME is a regular expression. **-S** *NAME* - | + | | Indicates that the NAME is a schema name. **-I** *NAME* - | + | | Indicates that the NAME is an instance name. This is the default. **-h** *HOST* - | + | | HOST to query. Default is localhost. **-x** *TRANSPORT* @@ -95,7 +95,7 @@ If the NAME is specified on the command line without -E/-S/-I, only information plugin manual for the option details. **-m** *MEMORY_SIZE* - | + | | MEMORY_SIZE is the size of memory reserved for metric sets. This value has precedence over the value of the LDMS_LS_MEM_SZ environment variable. The given size must be less than 1 petabytes. diff --git a/rtd/man2rst/ldms_sampler_base.rst b/rtd/man2rst/ldms_sampler_base.rst index aefcf2c1e..8af324bb7 100644 --- a/rtd/man2rst/ldms_sampler_base.rst +++ b/rtd/man2rst/ldms_sampler_base.rst @@ -41,54 +41,54 @@ CONFIGURATION ATTRIBUTE SYNTAX [component_id=] [schema=] [job_set= job_id= app_id= job_start= job_end=] -| +| | configuration line name= - | + | | This will be the name of the plugin being loaded. producer= - | + | | A unique name for the host providing the data. instance= - | + | | A unique name for the metric set. schema= - | + | | Optional schema name. It is intended that the same sampler on different nodes with different metrics have a different schema. Defaults to the sampler name. component_id= - | + | | Optional unique number for the component being monitored, Defaults to zero. job_set= - | + | | The instance name of the set containing the job data, default is 'job_info'. job_id= - | + | | The name of the metric containing the Job Id, default is 'job_id'. app_id= - | + | | The name of the metric containing the Application Id, default is 'app_id'. job_start= - | + | | The name of the metric containing the Job start time, default is 'job_start'. job_end= - | + | | The name of the metric containing the Job end time, default is 'job_end'. diff --git a/rtd/man2rst/ldmsctl.rst b/rtd/man2rst/ldmsctl.rst index 5263501ca..fcb2d25d1 100644 --- a/rtd/man2rst/ldmsctl.rst +++ b/rtd/man2rst/ldmsctl.rst @@ -103,7 +103,7 @@ Load a plugin | **load** attr= **name** *name* - | + | | The plugin name List the usage of the loaded plugins @@ -117,7 +117,7 @@ unload a plugin | **term** attr= **name** *name* - | + | | The plugin name Send a configuration command to the specified plugin. @@ -126,51 +126,51 @@ Send a configuration command to the specified plugin. **config** attr= **name** *name* - | + | | The plugin name **attr=value** - | + | | Plugin specific attr=value tuples **Attributes specific for sampler plugins (Some sampler plugins may have additional** attributes) **producer** *producer* - | + | | A unique name for the host providing the data **instance** *instance* - | + | | The set instance name. The name must be unique among all metric sets in all LDMS daemons. **[component_id** *component_id*\ **]** - | + | | A unique number for the comopnent being monitored. The default is zero. **[schema** *schema*\ **]** - | + | | The name of the metric set schema. **[job_set** *job_set*\ **]** - | + | | The set instance name of the set containing the job data. The default is 'job_info'. **[uid** *uid*\ **]** - | + | | The user id of the set's owner. The default is the returned value of geteuid(). **[gid** *gid*\ **]** - | + | | The group id of the set's owner. The default is the returned value of getegid(). **[perm** *perm*\ **]** - | + | | The sampler plugin instance access permission. The default is 0440. @@ -180,15 +180,15 @@ Start a sampler plugin **start** attr= **name** *name* - | + | | The plugin name. **interval** *interval* - | + | | The sample interval in microseconds. **[offset** *offset*\ **]** - | + | | Offset (shift) from the sample mark in microseconds. Offset can be positive or negative with magnitude up to 1/2 the sample interval. If this offset is specified, including 0, collection @@ -201,7 +201,7 @@ Stop a sampler plugin **stop** attr= **name** *name* - | + | | The plugin name. PRODUCER COMMAND SYNTAX @@ -213,29 +213,29 @@ Add a producer to the aggregator | **prdcr_add** attr= **name** *name* - | + | | The producer name. The producer name must be unique in an aggregator. It is independent of any attributes specified for the metric sets or hosts. **xprt** *xprt* - | + | | The transport name [sock, rdma, ugni] **host** *host* - | + | | The hostname of the host **type** *conn_type* - | + | | The connection type [active, passive] **interval** *interval* - | + | | The connection retry interval **[perm** *permission*\ **]** - | + | | The permission to modify the producer in the future Delete a producer from the aggregator @@ -245,7 +245,7 @@ Delete a producer from the aggregator | **prdcr_del** attr= **name** *name* - | + | | The producer name Start a producer @@ -254,11 +254,11 @@ Start a producer **prdcr_start** attr= **name** *name* - | + | | The producer name **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -268,11 +268,11 @@ Start all producers matching a regular expression **prdcr_start_regex** attr= **regex** *regex* - | + | | A regular expression **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -282,7 +282,7 @@ Stop a producer **prdcr_stop** attr= **name** *name* - | + | | The producer name Stop all producers matching a regular expression @@ -291,7 +291,7 @@ Stop all producers matching a regular expression **prdcr_stop_regex** attr= **regex** *regex* - | + | | A regular expression Query producer status @@ -300,7 +300,7 @@ Query producer status **prdcr_status** attr= **[name** *name*\ **]** - | + | | The producer name. If none is given, the statuses of all producers are reported. @@ -310,11 +310,11 @@ Subscribe for stream data from all matching producers **prdcr_subsribe** **regex** *regex* - | + | | The regular expression matching producer name **stream** *stream* - | + | | The stream name UPDATER COMMAND SYNTAX @@ -326,21 +326,21 @@ Add an updater process that will periodically sample producer metric sets **updtr_add** attr= **name** *name* - | + | | The update policy name. The policy name should be unique. It is independent of any attributes specified for the metric sets or hosts. **interval** *interval* - | + | | The update/collect interval **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. **[push** *onchange|true*\ **]** - | + | | Push mode: 'onchange' and 'true'. 'onchange' means the Updater will get an update whenever the set source ends a transaction or pushes the update. 'true' means the Updater will receive an @@ -355,7 +355,7 @@ Add an updater process that will periodically sample producer metric sets *false*. **[perm** *permission*\ **]** - | + | | The permission to modify the updater in the future Remove an updater from the configuration @@ -364,7 +364,7 @@ Remove an updater from the configuration **updtr_del** attr= **name** *name* - | + | | The update policy name Add a match condition that specifies the sets to update. @@ -373,15 +373,15 @@ Add a match condition that specifies the sets to update. **updtr_match_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -392,15 +392,15 @@ Remove a match condition from the Updater. **updtr_match_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -413,11 +413,11 @@ This is required before starting the updater. **updtr_prdcr_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Remove matching producers to an updater policy @@ -426,11 +426,11 @@ Remove matching producers to an updater policy **updtr_prdcr_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Start updaters. @@ -439,16 +439,16 @@ Start updaters. **updtr_start** attr= **name** *name* - | + | | The update policy name **[interval** *interval*\ **]** - | + | | The update interval in micro-seconds. If this is not specified, the previously configured value will be used. Optional. **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. Stop an updater. @@ -459,7 +459,7 @@ The Updater must be stopped in order to change it's configuration. **updtr_stop** attr= **name** *name* - | + | | The update policy name Query the updater status @@ -468,7 +468,7 @@ Query the updater status **updtr_status** attr= **[name** *name*\ **]** - | + | | The updater name. If none is given, the statuses of all updaters are reported. @@ -481,23 +481,23 @@ Create a Storage Policy and open/create the storage instance. **strgp_add** attr= **name** *name* - | + | | The unique storage policy name. **plugin** *plugin* - | + | | The name of the storage backend. **container** *container* - | + | | The storage backend container name. **schema** *schema* - | + | | The schema name of the metric set to store. **[perm** *permission*\ **]** - | + | | The permission to modify the storage in the future Remove a Storage Policy @@ -508,7 +508,7 @@ Remove a Storage Policy | **strgp_del** attr= **name** *name* - | + | | The storage policy name Add a regular expression used to identify the producers this storage policy will apply to. @@ -519,11 +519,11 @@ Add a regular expression used to identify the producers this storage policy will | **strgp_prdcr_add** attr= **name** *name* - | + | | The storage policy name **regex** *name* - | + | | A regular expression matching metric set producers. Remove a regular expression from the producer match list @@ -535,7 +535,7 @@ Remove a regular expression from the producer match list | The storage policy name **regex** *regex* - | + | | The regex of the producer to remove. Add the name of a metric to store @@ -547,7 +547,7 @@ Add the name of a metric to store | The storage policy name **metric** *metric* - | + | | The metric name. If the metric list is NULL, all metrics in the metric set will be stored. @@ -560,7 +560,7 @@ Remove a metric from the set of stored metrics. | The storage policy name **metric** *metric* - | + | | The metric to remove Start a storage policy. @@ -587,7 +587,7 @@ Query the storage policy status **strgp_status** attr= **[name** *name*\ **]** - | + | | The storage policy name. If none is given, the statuses of all storage policies are reported. @@ -610,11 +610,11 @@ Publish data to the named stream **plublish** attr= **name** *name* - | + | | The stream name **data** *data* - | + | | The data to publish Subscribe to a stream @@ -623,7 +623,7 @@ Subscribe to a stream **subscribe** attr= **name** *name* - | + | | The stream name LDMS DAEMON COMMAND SYNTAX @@ -669,7 +669,7 @@ MISC COMMAND SYNTAX Display the list of available commands -------------------------------------- -| +| | **help** | [*command]* @@ -679,19 +679,19 @@ Display the list of available commands Set the user data value for a metric in a metric set. ----------------------------------------------------- -| +| | **udata** attr= **set** *set* - | + | | The sampler plugin name **metric** *metric* - | + | | The metric name **udata** *udata* - | + | | The desired user-data. This is a 64b unsigned integer. Set the user data of multiple metrics using regular expression. @@ -703,19 +703,19 @@ Set the user data of multiple metrics using regular expression. | **udata_regex** attr= **set** *set* - | + | | The metric set name. **regex** *regex* - | + | | A regular expression to match metric names to be set **base** *base* - | + | | The base value of user data (uint64) **[incr** *incr*\ **]** - | + | | Increment value (int). The default is 0. If incr is 0, the user data of all matched metrics are set to the base value. Optional. diff --git a/rtd/man2rst/ldmsd.rst b/rtd/man2rst/ldmsd.rst index b848acb5f..567b354aa 100644 --- a/rtd/man2rst/ldmsd.rst +++ b/rtd/man2rst/ldmsd.rst @@ -148,7 +148,7 @@ General/Configuration Options: configuration. See also REORDERED COMMANDS below. **-m, --set_memory** *MEMORY_SIZE* - | + | | MEMORY_SIZE is the maximum size of pre-allocated memory for metric sets. The given size must be less than 1 petabytes. For example, 20M or 20mb are 20 megabytes. The default is adequate for most @@ -160,7 +160,7 @@ General/Configuration Options: value at ldmsd exit. **-n, --daemon_name** *NAME* - | + | | The name of the daemon. By default, it is "*HOSTNAME:PORT*". The failover feature uses the daemon name to verify the buddy name, and the producer name of kernel metric sets is the daemon name. @@ -180,7 +180,7 @@ Communication Options: ---------------------- **-x** *XPRT:PORT:HOST* - | + | | Specifies the transport type to listen on. May be specified more than once for multiple transports. The XPRT string is one of 'rdma', 'sock', or 'ugni' (CRAY XE/XK/XC). A transport specific @@ -212,7 +212,7 @@ Log Verbosity Options: ---------------------- **-l, --log_file** *LOGFILE* - | + | | LOGFILE is the path to the log file for status messages. Default is stdout unless given. The syslog facility is used if LOGFILE is exactly "syslog". Silence can be obtained by specifying /dev/null @@ -220,7 +220,7 @@ Log Verbosity Options: below. **-v, --log_level** *LOG_LEVEL* - | + | | LOG_LEVEL can be one of DEBUG, INFO, ERROR, CRITICAL or QUIET. The default level is ERROR. QUIET produces only user-requested output. (Note: this has changed from the previous release where q @@ -230,7 +230,7 @@ Log Verbosity Options: Truncate the log file if it already exists. **-L,**\ *--log_config* ** \| \| ** - | + | | Append configuration replay messages or configuration debugging messages to the log indicated by -l (when PATH is omitted) or to the file named PATH. Bit values of CINT correspond to: @@ -265,7 +265,7 @@ Thread Options: --------------- **-P, --worker_threads** *THR_COUNT* - | + | | THR_COUNT is the number of event threads to start. SPECIFYING COMMAND-LINE OPTIONS IN CONFIGURATION FILES diff --git a/rtd/man2rst/ldmsd_controller.rst b/rtd/man2rst/ldmsd_controller.rst index e34e16015..8451c0775 100644 --- a/rtd/man2rst/ldmsd_controller.rst +++ b/rtd/man2rst/ldmsd_controller.rst @@ -59,7 +59,7 @@ LDMSD_CONTROLLER OPTIONS The LDMS authentication plugin. Please see **ldms_authentication**\ (7) for more information. -**-A,--auth-arg** *NAME=VALUE* +**-A,--auth-arg** *NAME=VALUE* Options *NAME*\ =\ *VALUE* Passing the *NAME*\ =\ *VALUE* option to the LDMS Authentication plugin. This command line option can be given multiple times. Please see **ldms_authentication**\ (7) for more @@ -67,11 +67,11 @@ LDMSD_CONTROLLER OPTIONS options. **--source** *SOURCE* - | + | | Path to the config file **--script** *SCRIPT* - | + | | Execute the script and send the output commands to the connected ldmsd @@ -98,7 +98,7 @@ Load a plugin | **load** attr= **name** *name* - | + | | The plugin name List the usage of the loaded plugins @@ -112,7 +112,7 @@ unload a plugin | **term** attr= **name** *name* - | + | | The plugin name Send a configuration command to the specified plugin. @@ -121,51 +121,51 @@ Send a configuration command to the specified plugin. **config** attr= **name** *name* - | + | | The plugin name **attr=value** - | + | | Plugin specific attr=value tuples **Attributes specific for sampler plugins (Some sampler plugins may have additional** attributes) **producer** *producer* - | + | | A unique name for the host providing the data **instance** *instance* - | + | | The set instance name. The name must be unique among all metric sets in all LDMS daemons. **[component_id** *component_id*\ **]** - | + | | A unique number for the comopnent being monitored. The default is zero. **[schema** *schema*\ **]** - | + | | The name of the metric set schema. **[job_set** *job_set*\ **]** - | + | | The set instance name of the set containing the job data. The default is 'job_info'. **[uid** *uid*\ **]** - | + | | The user id of the set's owner. The default is the returned value of geteuid(). **[gid** *gid*\ **]** - | + | | The group id of the set's owner. The default is the returned value of getegid(). **[perm** *perm*\ **]** - | + | | The sampler plugin instance access permission. The default is 0440. @@ -175,15 +175,15 @@ Start a sampler plugin **start** attr= **name** *name* - | + | | The plugin name. **interval** *interval* - | + | | The sample interval in microseconds. **[offset** *offset*\ **]** - | + | | Offset (shift) from the sample mark in microseconds. Offset can be positive or negative with magnitude up to 1/2 the sample interval. If this offset is specified, including 0, collection @@ -196,7 +196,7 @@ Stop a sampler plugin **stop** attr= **name** *name* - | + | | The plugin name. AUTHENTICATION COMMAND SYNTAX @@ -209,16 +209,16 @@ Add an authentication domain ATTRIBUTES* ... ] **name**\ =\ *NAME* - | + | | The name of the authentication domain. This is the name referred to by **listen** and **prdcr_add** commands. **plugin**\ =\ *none*\ \|\ *ovis*\ \|\ *munge* - | + | | The LDMS Authentication Plugin for this domain. [ ... *PLUGIN ATTRIBUTES* ... ] - | + | | Arbitrary plugin attributes. Please consult the manual of the authentication plugin for more information. @@ -233,22 +233,22 @@ Instruct ldmsd to listen to a port [**host**\ =\ *HOST*] [**auth**\ =\ *AUTH_REF*] **port**\ =\ *PORT* - | + | | The port to listen to. Also, please be sure not to use ephemeral port (ports in the range of **/proc/sys/net/ip4/ip_local_port_range**). **xprt**\ =\ *sock*\ \|\ *rdma*\ \|\ *ugni*\ \|\ *fabric* - | + | | The type of the transport. **host**\ =\ *HOST* - | + | | An optional hostname or IP address to bind. If not given, listen to all addresses (0.0.0.0 or PORT). **auth**\ =\ *AUTH_REF* - | + | | Instruct **ldmsd** to use *AUTH_REF* (a name reference to **auth** object created by **auth_add** command) to authenticate connections on this port. If not given, the port uses the @@ -264,33 +264,33 @@ Add a producer to the aggregator | **prdcr_add** attr= **name** *name* - | + | | The producer name. The producer name must be unique in an aggregator. It is independent of any attributes specified for the metric sets or hosts. **xprt** *xprt* - | + | | The transport name [sock, rdma, ugni] **host** *host* - | + | | The hostname of the host **type** *conn_type* - | + | | The connection type [active, passive] **interval** *interval* - | + | | The connection retry interval **[perm** *permission*\ **]** - | + | | The permission to modify the producer in the future **[auth** *AUTH_REF*\ **]** - | + | | Instruct **ldmsd** to use *AUTH_REF* (a name reference to **auth** object created by **auth_add** command) with the connections to this producer. If not given, the default @@ -304,7 +304,7 @@ Delete a producer from the aggregator | **prdcr_del** attr= **name** *name* - | + | | The producer name Start a producer @@ -313,11 +313,11 @@ Start a producer **prdcr_start** attr= **name** *name* - | + | | The producer name **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -327,11 +327,11 @@ Start all producers matching a regular expression **prdcr_start_regex** attr= **regex** *regex* - | + | | A regular expression **[interval** *interval*\ **]** - | + | | The connection retry interval in microsec. If unspecified, the previously configured value will be used. Optional. @@ -341,7 +341,7 @@ Stop a producer **prdcr_stop** attr= **name** *name* - | + | | The producer name Stop all producers matching a regular expression @@ -350,7 +350,7 @@ Stop all producers matching a regular expression **prdcr_stop_regex** attr= **regex** *regex* - | + | | A regular expression Query producer status @@ -359,7 +359,7 @@ Query producer status **prdcr_status** attr= **[name** *name*\ **]** - | + | | The producer name. If none is given, the statuses of all producers are reported. @@ -369,11 +369,11 @@ Subscribe for stream data from all matching producers **prdcr_subsribe** **regex** *regex* - | + | | The regular expression matching producer name **stream** *stream* - | + | | The stream name UPDATER COMMAND SYNTAX @@ -385,21 +385,21 @@ Add an updater process that will periodically sample producer metric sets **updtr_add** attr= **name** *name* - | + | | The update policy name. The policy name should be unique. It is independent of any attributes specified for the metric sets or hosts. **interval** *interval* - | + | | The update/collect interval **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. **[push** *onchange|true*\ **]** - | + | | Push mode: 'onchange' and 'true'. 'onchange' means the Updater will get an update whenever the set source ends a transaction or pushes the update. 'true' means the Updater will receive an @@ -414,7 +414,7 @@ Add an updater process that will periodically sample producer metric sets *false*. **[perm** *permission*\ **]** - | + | | The permission to modify the updater in the future Remove an updater from the configuration @@ -423,7 +423,7 @@ Remove an updater from the configuration **updtr_del** attr= **name** *name* - | + | | The update policy name Add a match condition that specifies the sets to update. @@ -432,15 +432,15 @@ Add a match condition that specifies the sets to update. **updtr_match_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -451,15 +451,15 @@ Remove a match condition from the Updater. **updtr_match_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | The regular expression **match** *match (inst|schema)* - | + | | The value with which to compare; if match=inst, the expression will match the set's instance name, if match=schema, the expression will match the set's schema name. @@ -472,11 +472,11 @@ This is required before starting the updater. **updtr_prdcr_add** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Remove matching producers to an updater policy @@ -485,11 +485,11 @@ Remove matching producers to an updater policy **updtr_prdcr_del** attr= **name** *name* - | + | | The update policy name **regex** *regex* - | + | | A regular expression matching zero or more producers Start updaters. @@ -498,16 +498,16 @@ Start updaters. **updtr_start** attr= **name** *name* - | + | | The update policy name **[interval** *interval*\ **]** - | + | | The update interval in micro-seconds. If this is not specified, the previously configured value will be used. Optional. **[offset** *offset*\ **]** - | + | | Offset for synchronized aggregation. Optional. Stop an updater. @@ -518,7 +518,7 @@ The Updater must be stopped in order to change it's configuration. **updtr_stop** attr= **name** *name* - | + | | The update policy name Query the updater status @@ -527,7 +527,7 @@ Query the updater status **updtr_status** attr= **[name** *name*\ **]** - | + | | The updater name. If none is given, the statuses of all updaters are reported. @@ -540,23 +540,23 @@ Create a Storage Policy and open/create the storage instance. **strgp_add** attr= **name** *name* - | + | | The unique storage policy name. **plugin** *plugin* - | + | | The name of the storage backend. **container** *container* - | + | | The storage backend container name. **schema** *schema* - | + | | The schema name of the metric set to store. **[perm** *permission*\ **]** - | + | | The permission to modify the storage in the future Remove a Storage Policy @@ -567,7 +567,7 @@ Remove a Storage Policy | **strgp_del** attr= **name** *name* - | + | | The storage policy name Add a regular expression used to identify the producers this storage policy will apply to. @@ -578,11 +578,11 @@ Add a regular expression used to identify the producers this storage policy will | **strgp_prdcr_add** attr= **name** *name* - | + | | The storage policy name **regex** *name* - | + | | A regular expression matching metric set producers. Remove a regular expression from the producer match list @@ -594,7 +594,7 @@ Remove a regular expression from the producer match list | The storage policy name **regex** *regex* - | + | | The regex of the producer to remove. Add the name of a metric to store @@ -606,7 +606,7 @@ Add the name of a metric to store | The storage policy name **metric** *metric* - | + | | The metric name. If the metric list is NULL, all metrics in the metric set will be stored. @@ -619,7 +619,7 @@ Remove a metric from the set of stored metrics. | The storage policy name **metric** *metric* - | + | | The metric to remove Start a storage policy. @@ -646,7 +646,7 @@ Query the storage policy status **strgp_status** attr= **[name** *name*\ **]** - | + | | The storage policy name. If none is given, the statuses of all storage policies are reported. @@ -669,11 +669,11 @@ Publish data to the named stream **plublish** attr= **name** *name* - | + | | The stream name **data** *data* - | + | | The data to publish Subscribe to a stream @@ -682,7 +682,7 @@ Subscribe to a stream **subscribe** attr= **name** *name* - | + | | The stream name LDMS DAEMON COMMAND SYNTAX @@ -728,7 +728,7 @@ MISC COMMAND SYNTAX Display the list of available commands -------------------------------------- -| +| | **help** | [*command]* @@ -738,19 +738,19 @@ Display the list of available commands Set the user data value for a metric in a metric set. ----------------------------------------------------- -| +| | **udata** attr= **set** *set* - | + | | The sampler plugin name **metric** *metric* - | + | | The metric name **udata** *udata* - | + | | The desired user-data. This is a 64b unsigned integer. Set the user data of multiple metrics using regular expression. @@ -762,19 +762,19 @@ Set the user data of multiple metrics using regular expression. | **udata_regex** attr= **set** *set* - | + | | The metric set name. **regex** *regex* - | + | | A regular expression to match metric names to be set **base** *base* - | + | | The base value of user data (uint64) **[incr** *incr*\ **]** - | + | | Increment value (int). The default is 0. If incr is 0, the user data of all matched metrics are set to the base value. Optional. diff --git a/rtd/man2rst/ldmsd_stream_publish.rst b/rtd/man2rst/ldmsd_stream_publish.rst index 9412e39a6..71055ad6a 100644 --- a/rtd/man2rst/ldmsd_stream_publish.rst +++ b/rtd/man2rst/ldmsd_stream_publish.rst @@ -31,45 +31,45 @@ COMMAND LINE SYNTAX ==================================== ldmsd_sstream_publish -x -h -p -s -a -A -t -f [-l] - | + | -x - | + | | transport of the ldmsd to which to connect. -p - | + | | port of the ldmsd to which to connect. -a - | + | | auth to connect to the ldmsd -A - | + | | auth-opts to connect to the ldmsd -s - | + | | Name of the stream (this will be used for subscribing) -t - | + | | Optional data-format. Either 'string' or 'json'. Default is string. -l - | + | | Optional line mode. Publishes file one line at a time as separate publish calls -f - | + | | File that is published. If not specified, input is copied from STDIN. -r N - | + | | Repeat the publication of the file N times, with a delay interval specifed by -i. Repeating is not supported unless the input is a file. If the -l option is given, the file and diff --git a/rtd/man2rst/ldmsd_stream_subscribe.rst b/rtd/man2rst/ldmsd_stream_subscribe.rst index 9d063fdef..5829c2c99 100644 --- a/rtd/man2rst/ldmsd_stream_subscribe.rst +++ b/rtd/man2rst/ldmsd_stream_subscribe.rst @@ -29,57 +29,57 @@ COMMAND LINE SYNTAX ====================================== ldmsd_stream_subscribe -x -h -p -s -a -A -f -D -i -R -q -E - | + | -x,--xprt - | + | | transport type on which to listen. -p,--port - | + | | port on which to listen. -h,--host - | + | | hostname or IP address of interface on which to listen. -a,--auth - | + | | authentication to expect from publishers. -A,--auth_arg - | + | | auth options if needed (for e.g. ovis auth or munge on unusual port) -s,--stream - | + | | Name of the stream to subscribe. -f,--file - | + | | File where messages delivered are written. If not specified, STDOUT. -E,--events-raw - | + | | Suppress delivery envelope information in message output. -q,--quiet - | + | | Suppress message output to file or stdout entirely. -D,--daemonize - | + | | Put the process in the background as a daemon. -R,--daemon-noroot - | + | | Prevent file system root (/) change-directory when starting the daemon. (Does nothing if -D is not present). -i,--daemon-io - | + | | Keep the input and output file descriptors attached to the daemon instead of closing them. (Does nothing if -D is not present). diff --git a/rtd/pandoc_man_2_rst.py b/rtd/pandoc_man_2_rst.py index 21b277a0c..99f84df3c 100644 --- a/rtd/pandoc_man_2_rst.py +++ b/rtd/pandoc_man_2_rst.py @@ -16,7 +16,7 @@ os.system(f'/usr/local/bin/pandoc -f man -s -t rst --toc {i} -o man2rst/{fname}') plugin = fname.replace('.rst','') plugin_title = '='*len(plugin) - os.system('sed -i -e "0,/man/{s/man/'+plugin+'/}" man2rst/'+fname) - os.system('sed -i -e "s/===/'+plugin_title+'/" man2rst/'+fname) - os.system(f'cp man2rst/{fname} docs/source/{dest[c]}') + os.system('sed -i -e "0,/man/{s/man/'+plugin+'/}" man2rst/'+fname) + os.system('sed -i -e "s/===/'+plugin_title+'/" man2rst/'+fname) + os.system(f'cp man2rst/{fname} docs/source/{dest[c]}') diff --git a/rtd/update-branches.py b/rtd/update-branches.py deleted file mode 100644 index a29ce03b8..000000000 --- a/rtd/update-branches.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -""" -Updates all other branches to the latest master - -Since this repository is structured such that different branches build with -different themes, the result is a lot of branches that can fall behind master. -This script simply updates all branches to master. It is meant to be run by -somebody with push access to "origin". - - ./update-branches.py -""" - -import os -import subprocess -import sys - - -path_to_conf = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docs') -sys.path.append(path_to_conf) - -from conf import branch_to_theme_mapping # noqa - - -branches = branch_to_theme_mapping.keys() - -for branch in branches: - if branch == 'master': - continue - - print(u'*' * 77) - print(u'Syncing branch {} to master...'.format(branch)) - subprocess.check_output(['git', 'checkout', branch]) - subprocess.check_output(['git', 'merge', '--ff-only', 'master']) - subprocess.check_output(['git', 'push', 'origin', branch]) - print(u'*' * 77) - print(u'\n') - -print(u'Returning to master branch...') -subprocess.check_output(['git', 'checkout', 'master'])