Merge pull request #183 from desihub/render-columns

Render standard column definitions in ReadTheDocs

.github/workflows/python-package.yml

@@ -33,7 +33,7 @@ jobs:
             - name: Install Python dependencies
               run: |
                 python -m pip install --upgrade pip wheel
-                python -m pip install pytest
+                python -m pip install pytest jinja2
                 python -m pip install numpy\<1.23
                 python -m pip install "git+https://github.com/desihub/desiutil.git@${{ matrix.desiutil-version }}#egg=desiutil"
                 python -m pip install "astropy${{ matrix.astropy-version }}"
@@ -98,7 +98,10 @@ jobs:
                 python -m pip install --upgrade pip wheel
                 python -m pip install Sphinx sphinx-toolbox
             - name: Test the documentation
+              env:
+                PYTHONPATH: ${{ github.workspace }}/py
               run: |
+                python -m desidatamodel.columns > doc/column_descriptions.rst
                 sphinx-build -W --keep-going -b html doc doc/_build/html
 
     api:

.gitignore

@@ -38,3 +38,6 @@ MANIFEST
 
 # Mac OSX
 .DS_Store
+
+# Generated files
+doc/column_descriptions.rst

.readthedocs.yml

@@ -5,16 +5,24 @@
 # Required
 version: 2
 
+build:
+    os: "ubuntu-22.04"
+    tools:
+        python: "3.10"
+    jobs:
+        pre_build:
+            - env PYTHONPATH=${PWD}/py python -m desidatamodel.columns > doc/column_descriptions.rst
+
 # Build documentation in the doc/ directory with Sphinx
 sphinx:
     configuration: doc/conf.py
 
 # Optionally build your docs in additional formats such as PDF and ePub
-formats: all
+# formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-    version: 3
+    # version: 3
     # system_packages: true
     install:
         - requirements: doc/rtd-requirements.txt

doc/Makefile

@@ -8,10 +8,11 @@ SPHINXBUILD   = sphinx-build
 # SPHINXBUILD   = sphinx-build-2.7
 PAPER         =
 BUILDDIR      = _build
+PYTHON        = python
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/.)
 endif
 
 # Internal variables.
@@ -49,9 +50,13 @@ help:
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 
 clean:
+	$(RM) column_descriptions.rst
 	rm -rf $(BUILDDIR)/*
 
-html:
+column_descriptions.rst:
+	env PYTHONPATH=$(abspath ../py) $(PYTHON) -m desidatamodel.columns > column_descriptions.rst
+
+html: column_descriptions.rst
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."

doc/_static/desidatamodel.css

@@ -20,3 +20,7 @@ section details.summary-required-header-keywords-table div.wy-table-responsive t
 section details.summary-required-header-keywords-table div.wy-table-responsive table.keywords tbody tr.row-odd td { background-color: rgb(232, 232, 252); }
 section section div.wy-table-responsive table.columns tbody tr.row-even td { background-color: rgb(223, 246, 226); }
 section section div.wy-table-responsive table.columns tbody tr.row-odd td { background-color: rgb(232, 252, 232); }
+/*
+ * Experimental: use a wide "viewport". This makes tables look better.
+ */
+.wy-nav-content { max-width: none; }

doc/api.rst

@@ -8,6 +8,9 @@ desidatamodel API
 .. automodule:: desidatamodel.check
     :members:
 
+.. automodule:: desidatamodel.columns
+    :members:
+
 .. automodule:: desidatamodel.scan
     :members:
 

doc/changes.rst

@@ -5,9 +5,11 @@ desidatamodel Change Log
 23.6 (unreleased)
 -----------------
 
+* Add note about equivalent width values in ``fuji`` and ``guadalupe`` (PR `#181`_).
 * Add note about units in FITS files (PR `#178`_).
 
 .. _`#178`: https://github.com/desihub/desidatamodel/pull/178
+.. _`#181`: https://github.com/desihub/desidatamodel/pull/181
 
 23.1 (2023-06-12)
 -----------------

doc/conf.py

@@ -121,7 +121,7 @@
 # some external dependencies are not met at build time and break the
 # building process.
 autodoc_mock_imports = []
-for missing in ('astropy', 'desiutil', 'numpy'):
+for missing in ('astropy', 'desiutil', 'jinja2', 'numpy'):
     try:
         foo = import_module(missing)
     except ImportError:

doc/index.rst

@@ -62,6 +62,7 @@ Bitmask definitions and environment variables used by the DESI data pipelines:
    :maxdepth: 1
 
    bitmasks
+   column_descriptions
    Environment variables <envvar>
    Units in data files <units>
 

doc/rtd-requirements.txt

@@ -1 +1,3 @@
+jinja2>3.1
 sphinx-toolbox
+sphinx-rtd-theme>1.2

py/desidatamodel/check.py

@@ -10,6 +10,7 @@
 import os
 import re
 import itertools
+from pathlib import Path
 from sys import argv
 from argparse import ArgumentParser
 
@@ -25,10 +26,15 @@ class DataModel(DataModelUnit):
 
     Parameters
     ----------
-    filename : :class:`str`
+    filename : :class:`str` or :class:`pathlib.Path`
         The full path of the data model file.
-    section : :class:`str`
+    section : :class:`str` or :class:`pathlib.Path`
         The full path to the section of the data model containing the file.
+
+    Raises
+    ------
+    TypeError
+        If `filename` or `section` have an unexpected type.
     """
     # Marker for optional keywords and columns.
     _o = '[1]_'
@@ -82,10 +88,23 @@ class DataModel(DataModelUnit):
     _expectedtypes = ('ascii', 'csv', 'ecsv', 'fits', 'json', 'yaml')
 
     def __init__(self, filename, section):
-        shortname = filename.replace(f'{section}/', '')
+        if isinstance(filename, str):
+            self.filename = filename
+            self.section = section
+            shortname = filename.replace(f'{section}/', '')
+        elif isinstance(filename, Path):
+            self.filename = str(filename)
+            self.section = str(section)
+            shortname = str(filename).replace(f'{section}/', '')
+        else:
+            raise TypeError('Unexpected type for filename!')
+        if isinstance(section, str):
+            self.section = section
+        elif isinstance(section, Path):
+            self.section = str(section)
+        else:
+            raise TypeError('Unexpected type for section!')
         log.debug('Creating DataModel for %s.', shortname)
-        self.filename = filename
-        self.section = section
         self.title = None
         self.ref = None
         self.regexp = None

py/desidatamodel/columns.py

@@ -0,0 +1,63 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+# -*- coding: utf-8 -*-
+"""
+=====================
+desidatamodel.columns
+=====================
+
+Render the standard column descriptions file.
+"""
+import csv
+import sys
+import importlib.resources as ir
+import jinja2
+
+
+def format_columns(rows):
+    """Does something.
+
+    Parameters
+    ----------
+    rows : iterable
+        An iterable containing rows with any number of columns.
+
+    Returns
+    -------
+    :class:`tuple`
+        A tuple containing a format string, and an RST-style table separator.
+    """
+    lengths = list()
+    for row in rows:
+        for k, col in enumerate(row):
+            try:
+                if len(col) > lengths[k]:
+                    lengths[k] = len(col)
+            except IndexError:
+                lengths.append(len(col))
+    format_string = " ".join(["{{{0}:{1}}}".format(k, c) for k, c in enumerate(lengths)])
+    separator = ' '.join(['='*k for k in lengths])
+    return (format_string, separator)
+
+
+def main():
+    """Entry point for command-line scripts.
+
+    Returns
+    -------
+    :class:`int`
+        An integer suitable for passing to :func:`sys.exit`.
+    """
+    env = jinja2.Environment(loader=jinja2.PackageLoader('desidatamodel', package_path='data'),
+                             trim_blocks=True)
+    template = env.get_template('column_descriptions.rst')
+    columns = ir.files('desidatamodel') / 'data' / 'column_descriptions.csv'
+    with open(columns, newline='') as cf:
+        reader = csv.reader(cf)
+        format_string, separator = format_columns(reader)
+        cf.seek(0)
+        print(template.render(reader=reader, format_string=format_string, separator=separator))
+    return 0
+
+
+if __name__ == '__main__':  # pragma: no cover
+    sys.exit(main())

py/desidatamodel/data/column_descriptions.rst

@@ -0,0 +1,22 @@
+============================
+Standard Column Descriptions
+============================
+
+This file is an auto-generated version of `column_descriptions.csv`_,
+which contains the descriptions of table column names regardless of
+what file(s) they appear in.
+
+.. _`column_descriptions.csv`: https://github.com/desihub/desidatamodel/blob/main/py/desidatamodel/data/column_descriptions.csv
+
+{% for row in reader %}
+{% if loop.first %}
+{{ separator }}
+{% endif %}
+{{ format_string.format(*row) }}
+{% if loop.first %}
+{{ separator }}
+{% endif %}
+{% if loop.last %}
+{{ separator }}
+{% endif %}
+{% endfor %}

py/desidatamodel/stub.py

@@ -10,7 +10,9 @@
 import os
 import re
 from html import escape
-from pkg_resources import resource_filename
+from pathlib import Path
+import importlib.resources as ir
+# from pkg_resources import resource_filename
 from astropy.io import fits
 from astropy.io.fits.card import Undefined
 from astropy.table import Table
@@ -102,6 +104,8 @@ def __init__(self, filename, description_file=None, error=False):
                     self.headers.append(fx[k].header)
             if isinstance(filename, (str,)):
                 self.filename = filename
+            elif isinstance(filename, (Path,)):
+                self.filename = str(filename)
         self._basef = None
         self._modelname = None
         self._filesize = None
@@ -752,7 +756,7 @@ def main():
     parser.add_argument("--column_descriptions",
                         help="CSV file with column info Name,Type,Units,Description; "
                              "default=%(default)s",
-                        default=resource_filename('desidatamodel', 'data/column_descriptions.csv'))
+                        default=(ir.files('desidatamodel') / 'data' / 'column_descriptions.csv'))
 
     options = parser.parse_args()
     if options.verbose:

py/desidatamodel/test/datamodeltestcase.py

@@ -7,6 +7,7 @@
 import unittest
 import logging
 import shutil
+import importlib.resources as ir
 from packaging import version
 
 from astropy import __version__ as astropyVersion
@@ -24,6 +25,7 @@ def setUpClass(cls):
         cls.astropyVersion = version.parse(astropyVersion)
         cls.maxDiff = None
         cls.data_dir = tempfile.mkdtemp()
+        cls.test_files = ir.files('desidatamodel.test') / 't'
         if DM in os.environ:
             cls.old_env = os.environ[DM]
         else: