diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c7554f43..956a33d7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,7 +33,7 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip wheel - python -m pip install pytest + python -m pip install pytest jinja2 python -m pip install numpy\<1.23 python -m pip install "git+https://github.com/desihub/desiutil.git@${{ matrix.desiutil-version }}#egg=desiutil" python -m pip install "astropy${{ matrix.astropy-version }}" @@ -98,7 +98,10 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install Sphinx sphinx-toolbox - name: Test the documentation + env: + PYTHONPATH: ${{ github.workspace }}/py run: | + python -m desidatamodel.columns > doc/column_descriptions.rst sphinx-build -W --keep-going -b html doc doc/_build/html api: diff --git a/.gitignore b/.gitignore index b1bf8746..ec36eaef 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ MANIFEST # Mac OSX .DS_Store + +# Generated files +doc/column_descriptions.rst diff --git a/.readthedocs.yml b/.readthedocs.yml index 4374d5a5..f8a86a28 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,16 +5,24 @@ # Required version: 2 +build: + os: "ubuntu-22.04" + tools: + python: "3.10" + jobs: + pre_build: + - env PYTHONPATH=${PWD}/py python -m desidatamodel.columns > doc/column_descriptions.rst + # Build documentation in the doc/ directory with Sphinx sphinx: configuration: doc/conf.py # Optionally build your docs in additional formats such as PDF and ePub -formats: all +# formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3 + # version: 3 # system_packages: true install: - requirements: doc/rtd-requirements.txt diff --git a/doc/Makefile b/doc/Makefile index 17186b25..0bd6714b 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -8,10 +8,11 @@ SPHINXBUILD = sphinx-build # SPHINXBUILD = sphinx-build-2.7 PAPER = BUILDDIR = _build +PYTHON = python # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/.) endif # Internal variables. @@ -49,9 +50,13 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: + $(RM) column_descriptions.rst rm -rf $(BUILDDIR)/* -html: +column_descriptions.rst: + env PYTHONPATH=$(abspath ../py) $(PYTHON) -m desidatamodel.columns > column_descriptions.rst + +html: column_descriptions.rst $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." diff --git a/doc/_static/desidatamodel.css b/doc/_static/desidatamodel.css index a64cd741..15010745 100644 --- a/doc/_static/desidatamodel.css +++ b/doc/_static/desidatamodel.css @@ -20,3 +20,7 @@ section details.summary-required-header-keywords-table div.wy-table-responsive t section details.summary-required-header-keywords-table div.wy-table-responsive table.keywords tbody tr.row-odd td { background-color: rgb(232, 232, 252); } section section div.wy-table-responsive table.columns tbody tr.row-even td { background-color: rgb(223, 246, 226); } section section div.wy-table-responsive table.columns tbody tr.row-odd td { background-color: rgb(232, 252, 232); } +/* + * Experimental: use a wide "viewport". This makes tables look better. + */ +.wy-nav-content { max-width: none; } diff --git a/doc/api.rst b/doc/api.rst index 7b62daab..16cad9a6 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -8,6 +8,9 @@ desidatamodel API .. automodule:: desidatamodel.check :members: +.. automodule:: desidatamodel.columns + :members: + .. automodule:: desidatamodel.scan :members: diff --git a/doc/changes.rst b/doc/changes.rst index 6b1da387..94da7ab0 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,9 +5,11 @@ desidatamodel Change Log 23.6 (unreleased) ----------------- +* Add note about equivalent width values in ``fuji`` and ``guadalupe`` (PR `#181`_). * Add note about units in FITS files (PR `#178`_). .. _`#178`: https://github.com/desihub/desidatamodel/pull/178 +.. _`#181`: https://github.com/desihub/desidatamodel/pull/181 23.1 (2023-06-12) ----------------- diff --git a/doc/conf.py b/doc/conf.py index ea4b3b17..b37db22f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -121,7 +121,7 @@ # some external dependencies are not met at build time and break the # building process. autodoc_mock_imports = [] -for missing in ('astropy', 'desiutil', 'numpy'): +for missing in ('astropy', 'desiutil', 'jinja2', 'numpy'): try: foo = import_module(missing) except ImportError: diff --git a/doc/index.rst b/doc/index.rst index 5b29f857..739f51cd 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -62,6 +62,7 @@ Bitmask definitions and environment variables used by the DESI data pipelines: :maxdepth: 1 bitmasks + column_descriptions Environment variables Units in data files diff --git a/doc/rtd-requirements.txt b/doc/rtd-requirements.txt index 99f2a583..abd4c674 100644 --- a/doc/rtd-requirements.txt +++ b/doc/rtd-requirements.txt @@ -1 +1,3 @@ +jinja2>3.1 sphinx-toolbox +sphinx-rtd-theme>1.2 diff --git a/py/desidatamodel/check.py b/py/desidatamodel/check.py index d57c6bd6..946349d3 100644 --- a/py/desidatamodel/check.py +++ b/py/desidatamodel/check.py @@ -10,6 +10,7 @@ import os import re import itertools +from pathlib import Path from sys import argv from argparse import ArgumentParser @@ -25,10 +26,15 @@ class DataModel(DataModelUnit): Parameters ---------- - filename : :class:`str` + filename : :class:`str` or :class:`pathlib.Path` The full path of the data model file. - section : :class:`str` + section : :class:`str` or :class:`pathlib.Path` The full path to the section of the data model containing the file. + + Raises + ------ + TypeError + If `filename` or `section` have an unexpected type. """ # Marker for optional keywords and columns. _o = '[1]_' @@ -82,10 +88,23 @@ class DataModel(DataModelUnit): _expectedtypes = ('ascii', 'csv', 'ecsv', 'fits', 'json', 'yaml') def __init__(self, filename, section): - shortname = filename.replace(f'{section}/', '') + if isinstance(filename, str): + self.filename = filename + self.section = section + shortname = filename.replace(f'{section}/', '') + elif isinstance(filename, Path): + self.filename = str(filename) + self.section = str(section) + shortname = str(filename).replace(f'{section}/', '') + else: + raise TypeError('Unexpected type for filename!') + if isinstance(section, str): + self.section = section + elif isinstance(section, Path): + self.section = str(section) + else: + raise TypeError('Unexpected type for section!') log.debug('Creating DataModel for %s.', shortname) - self.filename = filename - self.section = section self.title = None self.ref = None self.regexp = None diff --git a/py/desidatamodel/columns.py b/py/desidatamodel/columns.py new file mode 100644 index 00000000..56e9e2d3 --- /dev/null +++ b/py/desidatamodel/columns.py @@ -0,0 +1,63 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- +""" +===================== +desidatamodel.columns +===================== + +Render the standard column descriptions file. +""" +import csv +import sys +import importlib.resources as ir +import jinja2 + + +def format_columns(rows): + """Does something. + + Parameters + ---------- + rows : iterable + An iterable containing rows with any number of columns. + + Returns + ------- + :class:`tuple` + A tuple containing a format string, and an RST-style table separator. + """ + lengths = list() + for row in rows: + for k, col in enumerate(row): + try: + if len(col) > lengths[k]: + lengths[k] = len(col) + except IndexError: + lengths.append(len(col)) + format_string = " ".join(["{{{0}:{1}}}".format(k, c) for k, c in enumerate(lengths)]) + separator = ' '.join(['='*k for k in lengths]) + return (format_string, separator) + + +def main(): + """Entry point for command-line scripts. + + Returns + ------- + :class:`int` + An integer suitable for passing to :func:`sys.exit`. + """ + env = jinja2.Environment(loader=jinja2.PackageLoader('desidatamodel', package_path='data'), + trim_blocks=True) + template = env.get_template('column_descriptions.rst') + columns = ir.files('desidatamodel') / 'data' / 'column_descriptions.csv' + with open(columns, newline='') as cf: + reader = csv.reader(cf) + format_string, separator = format_columns(reader) + cf.seek(0) + print(template.render(reader=reader, format_string=format_string, separator=separator)) + return 0 + + +if __name__ == '__main__': # pragma: no cover + sys.exit(main()) diff --git a/py/desidatamodel/data/column_descriptions.rst b/py/desidatamodel/data/column_descriptions.rst new file mode 100644 index 00000000..46565cef --- /dev/null +++ b/py/desidatamodel/data/column_descriptions.rst @@ -0,0 +1,22 @@ +============================ +Standard Column Descriptions +============================ + +This file is an auto-generated version of `column_descriptions.csv`_, +which contains the descriptions of table column names regardless of +what file(s) they appear in. + +.. _`column_descriptions.csv`: https://github.com/desihub/desidatamodel/blob/main/py/desidatamodel/data/column_descriptions.csv + +{% for row in reader %} +{% if loop.first %} +{{ separator }} +{% endif %} +{{ format_string.format(*row) }} +{% if loop.first %} +{{ separator }} +{% endif %} +{% if loop.last %} +{{ separator }} +{% endif %} +{% endfor %} diff --git a/py/desidatamodel/stub.py b/py/desidatamodel/stub.py index 1270f657..95604488 100644 --- a/py/desidatamodel/stub.py +++ b/py/desidatamodel/stub.py @@ -10,7 +10,9 @@ import os import re from html import escape -from pkg_resources import resource_filename +from pathlib import Path +import importlib.resources as ir +# from pkg_resources import resource_filename from astropy.io import fits from astropy.io.fits.card import Undefined from astropy.table import Table @@ -102,6 +104,8 @@ def __init__(self, filename, description_file=None, error=False): self.headers.append(fx[k].header) if isinstance(filename, (str,)): self.filename = filename + elif isinstance(filename, (Path,)): + self.filename = str(filename) self._basef = None self._modelname = None self._filesize = None @@ -752,7 +756,7 @@ def main(): parser.add_argument("--column_descriptions", help="CSV file with column info Name,Type,Units,Description; " "default=%(default)s", - default=resource_filename('desidatamodel', 'data/column_descriptions.csv')) + default=(ir.files('desidatamodel') / 'data' / 'column_descriptions.csv')) options = parser.parse_args() if options.verbose: diff --git a/py/desidatamodel/test/datamodeltestcase.py b/py/desidatamodel/test/datamodeltestcase.py index 27389515..e06160f8 100644 --- a/py/desidatamodel/test/datamodeltestcase.py +++ b/py/desidatamodel/test/datamodeltestcase.py @@ -7,6 +7,7 @@ import unittest import logging import shutil +import importlib.resources as ir from packaging import version from astropy import __version__ as astropyVersion @@ -24,6 +25,7 @@ def setUpClass(cls): cls.astropyVersion = version.parse(astropyVersion) cls.maxDiff = None cls.data_dir = tempfile.mkdtemp() + cls.test_files = ir.files('desidatamodel.test') / 't' if DM in os.environ: cls.old_env = os.environ[DM] else: diff --git a/py/desidatamodel/test/test_check.py b/py/desidatamodel/test/test_check.py index 2a34786d..08396150 100644 --- a/py/desidatamodel/test/test_check.py +++ b/py/desidatamodel/test/test_check.py @@ -7,7 +7,6 @@ from packaging import version import unittest from unittest.mock import patch -from pkg_resources import resource_filename from .datamodeltestcase import DataModelTestCase, DM @@ -18,6 +17,23 @@ class TestCheck(DataModelTestCase): + def test_DataModel_init(self): + """Test initialization of the DataModel object. + """ + modelfile = self.test_files / 'fits_file.rst' + model = DataModel(modelfile, self.test_files) + self.assertEqual(model.filename, str(modelfile)) + self.assertEqual(model.section, str(self.test_files)) + + def test_DataModel_init_bad_type(self): + """Test initialization of the DataModel object. + """ + modelfile = self.test_files / 'fits_file.rst' + with self.assertRaises(TypeError): + model = DataModel(None, self.test_files) + with self.assertRaises(TypeError): + model = DataModel(modelfile, None) + def test_scan_model(self): """Test identification of data model files. """ @@ -112,7 +128,7 @@ def test_files_to_regexp_with_bad_filetype(self): def test_get_regexp_filesize(self): """Test extraction of file size from data model documents. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) foo = model.get_regexp('/desi/spectro/data') self.assertEqual(model.filetype, 'fits') @@ -121,7 +137,7 @@ def test_get_regexp_filesize(self): def test_get_regexp_missing_filesize(self): """Test extraction of file size from data model documents, missing size. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_no_size.rst') + modelfile = self.test_files / 'fits_file_no_size.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) foo = model.get_regexp('/desi/spectro/data') self.assertEqual(model.filetype, 'fits') @@ -200,7 +216,7 @@ def test_extract_metadata(self): 'length of dimension 1'), ('NAXIS2', '3', 'int', 'length of dimension 2')]}} - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() self.assertEqual(self.title, 'fits_file') @@ -245,7 +261,7 @@ def test_extract_metadata(self): 'length of dimension 1'), ('NAXIS2', '3', 'int', 'length of dimension 2')]}} - modelfile = resource_filename('desidatamodel.test', 't/fits_file_collapse.rst') + modelfile = self.test_files / 'fits_file_collapse.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() self.assertEqual(len(meta.keys()), len(ex_meta.keys())) @@ -265,7 +281,7 @@ def test_extract_metadata(self): def test_extract_metadata_missing_extname(self): """Test reading metadata with missing EXTNAME. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() lines = model._metafile_data.split('\n') @@ -283,7 +299,7 @@ def test_extract_metadata_bad_keyword_unit(self): """Test reading metadata with bad FITS BUNIT values. """ erg_msg = self.badUnitMessage('ergs') - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() lines = model._metafile_data.split('\n') @@ -301,7 +317,7 @@ def test_extract_metadata_bad_keyword_unit(self): def test_extract_metadata_missing_keyword_unit(self): """Test reading metadata with missing units for header keywords. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() lines = model._metafile_data.split('\n') @@ -320,7 +336,7 @@ def test_extract_metadata_bad_column_unit(self): """Test reading metadata with bad FITS column units. """ erg_msg = self.badUnitMessage('ergs') - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() lines = model._metafile_data.split('\n') @@ -337,7 +353,7 @@ def test_extract_metadata_bad_column_unit(self): def test_extract_metadata_missing_column_type(self): """Test reading metadata with missing FITS column types. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() lines = model._metafile_data.split('\n') @@ -354,7 +370,7 @@ def test_extract_metadata_missing_column_type(self): def test_extract_metadata_with_hdu_span(self): """Test reading metadata with a HDU span. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_hduspan.rst') + modelfile = self.test_files / 'fits_file_hduspan.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() self.assertEqual(meta['TWO']['number'], 2) @@ -362,7 +378,7 @@ def test_extract_metadata_with_hdu_span(self): def test_extract_metadata_with_hdu_span_no_spanext(self): """Test reading metadata with a HDU span, but with no reference HDU. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_hduspan_no_spanext.rst') + modelfile = self.test_files / 'fits_file_hduspan_no_spanext.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) with self.assertRaises(DataModelError) as e: meta = model.extract_metadata() @@ -372,7 +388,7 @@ def test_extract_metadata_with_hdu_span_no_spanext(self): def test_extract_metadata_with_hdu_span_bad_extname(self): """Test reading metadata with a HDU span, but with bad EXTNAME specification. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_hduspan_bad_extname.rst') + modelfile = self.test_files / 'fits_file_hduspan_bad_extname.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) meta = model.extract_metadata() self.assertLog(log, -1, "Range specification from HDU 2 to HDU 5 does not have a matching EXTNAME specification!") @@ -380,7 +396,7 @@ def test_extract_metadata_with_hdu_span_bad_extname(self): def test_extract_metadata_bad_format(self): """Test reading metadata with a bad HDU format specification. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_bad_format.rst') + modelfile = self.test_files / 'fits_file_bad_format.rst' model = DataModel(modelfile, os.path.dirname(modelfile)) with self.assertRaises(DataModelError) as e: meta = model.extract_metadata() @@ -390,7 +406,7 @@ def test_extract_metadata_bad_format(self): def test_validate_prototypes(self): """Test the data model validation function. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -399,7 +415,7 @@ def test_validate_prototypes(self): def test_validate_prototype_no_prototype(self): """Test the data model validation method with no prototype. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -410,7 +426,7 @@ def test_validate_prototype_not_verifiable_prototype(self): """Test the data model validation method with prototypes that are not currently verifiable. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -421,7 +437,7 @@ def test_validate_prototype_not_verifiable_prototype(self): def test_validate_prototype_oserror(self): """Test the data model validation method with a file that throws an error. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -443,32 +459,32 @@ def test_validate_prototype_oserror(self): def test_validate_prototype_hdu_mismatch(self): """Test the data model validation method with wrong number of HDUs. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) foo = f.extract_metadata() f.hdumeta['foobar'] = 'baz' f.validate_prototype(error=True) - self.assertLog(log, -2, "{0} has the wrong number of sections (HDUs) according to {1}, skipping to next candidate.".format(modelfile.replace('.rst', '.fits'), modelfile)) + self.assertLog(log, -2, "{0} has the wrong number of sections (HDUs) according to {1}, skipping to next candidate.".format(str(modelfile).replace('.rst', '.fits'), modelfile)) self.assertLog(log, -1, "No useful prototype files found for {0}!".format(modelfile)) def test_validate_prototype_hdu_keyword_mismatch(self): """Test the data model validation method with wrong number of HDU keywords. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) f.validate_prototype() f._stub_meta[0]['keywords'].append(('BUNIT', 'erg', 'str', 'This is a test.')) f.validate_prototype(error=True) - self.assertLog(log, -1, "Prototype file {0} has these keywords in HDU0 missing from model: {{'BUNIT'}}".format(modelfile.replace('.rst', '.fits'))) + self.assertLog(log, -1, "Prototype file {0} has these keywords in HDU0 missing from model: {{'BUNIT'}}".format(str(modelfile).replace('.rst', '.fits'))) def test_validate_prototype_hdu_keyword_type_mismatch(self): """Test the data model validation method with a keyword type mismatch. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -480,51 +496,51 @@ def test_validate_prototype_hdu_keyword_type_mismatch(self): def test_validate_prototype_hdu_wrong_keyword(self): """Test the data model validation method with wrong HDU keyword names. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) f.validate_prototype() f._stub_meta[0]['keywords'][-1] = ('BUNIT', 'erg', 'str', 'This is a test.') f.validate_prototype(error=True) - self.assertLog(log, -2, "Prototype file {0} has these keywords in HDU0 missing from model: {{'BUNIT'}}".format(modelfile.replace('.rst', '.fits'))) + self.assertLog(log, -2, "Prototype file {0} has these keywords in HDU0 missing from model: {{'BUNIT'}}".format(str(modelfile).replace('.rst', '.fits'))) self.assertLog(log, -1, "Model file {0} has these keywords in HDU0 missing from data: {{'BZERO'}}".format(modelfile)) def test_validate_prototype_hdu_extension_type(self): """Test the data model validation method with wrong HDU extension type. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) f.validate_prototype() f._stub_meta[1]['extension'] = 'IMAGE' f.validate_prototype(error=True) - self.assertLog(log, -1, "Prototype file {0} has an extension type mismatch in HDU1 (IMAGE != BINTABLE) according to {1}.".format(modelfile.replace('.rst', '.fits'), modelfile)) + self.assertLog(log, -1, "Prototype file {0} has an extension type mismatch in HDU1 (IMAGE != BINTABLE) according to {1}.".format(str(modelfile).replace('.rst', '.fits'), modelfile)) # f._stub_meta[1]['extname'] = '' # f.validate_prototype(error=True) - # self.assertLog(log, -1, "Prototype file {0} has no EXTNAME in HDU1.".format(modelfile.replace('.rst', '.fits'))) + # self.assertLog(log, -1, "Prototype file {0} has no EXTNAME in HDU1.".format(str(modelfile).replace('.rst', '.fits'))) def test_validate_prototype_hdu_extension_name(self): """Test the data model validation method with wrong HDU extension name. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) f.validate_prototype() f._stub_meta[1]['extname'] = 'GALAXY' f.validate_prototype(error=True) - self.assertLog(log, -1, "Prototype file {0} has an EXTNAME mismatch in HDU1 (GALAXY != Galaxies) according to {1}.".format(modelfile.replace('.rst', '.fits'), modelfile)) + self.assertLog(log, -1, "Prototype file {0} has an EXTNAME mismatch in HDU1 (GALAXY != Galaxies) according to {1}.".format(str(modelfile).replace('.rst', '.fits'), modelfile)) f._stub_meta[1]['extname'] = '' f.validate_prototype(error=True) - self.assertLog(log, -2, "Prototype file {0} has no EXTNAME in HDU1.".format(modelfile.replace('.rst', '.fits'))) + self.assertLog(log, -2, "Prototype file {0} has no EXTNAME in HDU1.".format(str(modelfile).replace('.rst', '.fits'))) self.assertLog(log, -1, "Could not find EXTNAME = '' in {0}; trying by HDU number.".format(modelfile)) def test_validate_prototype_hdu_bad_format(self): """Test the data model validation method with a bad HDU format in the model. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -538,7 +554,7 @@ def test_validate_prototype_hdu_bad_format(self): def test_validate_prototype_hdu_alternate_format(self): """Test the data model validation method with an alternate HDU format in the model. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -552,7 +568,7 @@ def test_validate_prototype_hdu_alternate_format(self): def test_validate_prototype_hdu_bad_extension(self): """Test the data model validation method with a bad HDU extension in the model. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -566,7 +582,7 @@ def test_validate_prototype_hdu_bad_extension(self): def test_validate_prototype_hdu_missing_column(self): """Test the data model validation method with missing column in the model. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -578,7 +594,7 @@ def test_validate_prototype_hdu_missing_column(self): def test_validate_prototype_hdu_missing_data_column(self): """Test the data model validation method with missing column in the data. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -590,7 +606,7 @@ def test_validate_prototype_hdu_missing_data_column(self): def test_validate_prototype_hdu_bad_column_type(self): """Test the data model validation method with a bad column type. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -602,7 +618,7 @@ def test_validate_prototype_hdu_bad_column_type(self): def test_validate_prototype_hdu_bad_column_unit(self): """Test the data model validation method with a bad column unit. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -614,7 +630,7 @@ def test_validate_prototype_hdu_bad_column_unit(self): def test_validate_prototype_optional_keywords(self): """Test the data model validation method with optional keywords. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_optional_columns.rst') + modelfile = self.test_files / 'fits_file_optional_columns.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -627,7 +643,7 @@ def test_validate_prototype_optional_keywords(self): def test_validate_prototype_optional_columns(self): """Test the data model validation method with optional columns. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_optional_columns.rst') + modelfile = self.test_files / 'fits_file_optional_columns.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -639,7 +655,7 @@ def test_validate_prototype_optional_columns(self): def test_validate_prototype_variable_columns(self): """Test the data model validation method with variable-size columns. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file_variable_columns.rst') + modelfile = self.test_files / 'fits_file_variable_columns.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) f.get_regexp(os.path.dirname(modelfile)) collect_files(os.path.dirname(modelfile), [f]) @@ -651,7 +667,7 @@ def test_validate_prototype_variable_columns(self): def test_extract_columns(self): """Test extraction of columns from a row of data. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) foo = '======= ============= ==== ============' columns = list(map(len, foo.split())) @@ -663,12 +679,11 @@ def test_extract_columns(self): def test_cross_reference(self): """Test parsing of cross-references. """ - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + modelfile = self.test_files / 'fits_file.rst' f = DataModel(modelfile, os.path.dirname(modelfile)) line = "See :doc:`Other file `" ref = f._cross_reference(line) - self.assertEqual(ref, resource_filename('desidatamodel.test', - 't/fits_file.rst')) + self.assertEqual(ref, str(modelfile)) @patch('sys.argv', ['check_model', '--verbose', '--compare-files', 'DESI_SPECTRO_DATA', '/desi/spectro/data/desi-00000000.fits.fz']) def test_options(self): @@ -679,11 +694,3 @@ def test_options(self): self.assertTrue(options.files) self.assertEqual(options.section, 'DESI_SPECTRO_DATA') self.assertEqual(options.directory, '/desi/spectro/data/desi-00000000.fits.fz') - - -def test_suite(): - """Allows testing of only this module with the command:: - - python setup.py test -m - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/py/desidatamodel/test/test_columns.py b/py/desidatamodel/test/test_columns.py new file mode 100644 index 00000000..b691fb70 --- /dev/null +++ b/py/desidatamodel/test/test_columns.py @@ -0,0 +1,21 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- +"""Test desidatamodel.columns functions +""" +import unittest + +from ..columns import format_columns + + +class TestColumns(unittest.TestCase): + + def test_format_columns(self): + """Test column formatting operations. + """ + data = [('one', 'two', 'three'), + ('four', 'five', 'six'), + ('seven', 'eight', 'nine'), + ('ten', 'eleven', 'twelve')] + format_string, separator = format_columns(data) + self.assertEqual(format_string, "{0:5} {1:6} {2:6}") + self.assertEqual(separator, '===== ====== ======') diff --git a/py/desidatamodel/test/test_model.py b/py/desidatamodel/test/test_model.py index 49996755..18e14874 100644 --- a/py/desidatamodel/test/test_model.py +++ b/py/desidatamodel/test/test_model.py @@ -30,11 +30,3 @@ def test_model(self): error=True) for f in files: meta = f.extract_metadata(error=True) - - -def test_suite(): - """Allows testing of only this module with the command:: - - python setup.py test -m - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/py/desidatamodel/test/test_scan.py b/py/desidatamodel/test/test_scan.py index 50c8431a..0a7f8b2a 100644 --- a/py/desidatamodel/test/test_scan.py +++ b/py/desidatamodel/test/test_scan.py @@ -23,7 +23,7 @@ def test_UnionStub(self): """Test initialization of UnionStub. """ - model = DataModel(resource_filename('desidatamodel.test', 't/fits_file_optional_columns.rst'), + model = DataModel(self.test_files / 'fits_file_optional_columns.rst', os.path.join(os.environ[DM], 'doc', 'examples')) foo = model.get_regexp('/desi/spectro/data') union = UnionStub(model, 10, error=False) @@ -36,7 +36,7 @@ def test_UnionStub(self): def test_UnionStub_mark_optional(self): """Test final output of UnionStub. """ - model = DataModel(resource_filename('desidatamodel.test', 't/fits_file_optional_columns.rst'), + model = DataModel(self.test_files / 'fits_file_optional_columns.rst', os.path.join(os.environ[DM], 'doc', 'examples')) foo = model.get_regexp('/desi/spectro/data') union = UnionStub(model, 10, error=False) @@ -60,9 +60,9 @@ def test_UnionStub_mark_optional(self): def test_UnionStub_update(self): """Test updates to the union model. """ - stubs = [Stub(resource_filename('desidatamodel.test', 't/fits_file.fits')), - Stub(resource_filename('desidatamodel.test', 't/fits_file.fits')), - Stub(resource_filename('desidatamodel.test', 't/fits_file.fits'))] + stubs = [Stub(self.test_files / 'fits_file.fits'), + Stub(self.test_files / 'fits_file.fits'), + Stub(self.test_files / 'fits_file.fits')] stubs[0].hdumeta[0]['keywords'][2] = ('BSCALE', '1', 'int', 'No scaling.') stubs[0].hdumeta[1]['format'].append(('OPT1', 'int32', '', 'Comment')) stubs[0].hdumeta[1]['format'].append(('OPT2', 'float64', 'yr', 'Comment')) @@ -71,7 +71,7 @@ def test_UnionStub_update(self): stubs[1].hdumeta[0]['keywords'][2] = ('BSCALE', '1', 'int', 'No scaling.') stubs[1].hdumeta[1]['format'].append(('OPT2', 'float64', 'yr', 'Comment')) stubs[2].hdumeta[0]['keywords'].append(('KEYTEST', 'example', 'str', 'Comment')) - model = DataModel(resource_filename('desidatamodel.test', 't/fits_file_optional_columns.rst'), + model = DataModel(self.test_files / 'fits_file_optional_columns.rst', os.path.join(os.environ[DM], 'doc', 'examples')) foo = model.get_regexp('/desi/spectro/data') modelmeta = model.extract_metadata() @@ -121,13 +121,13 @@ def test_collect_files(self): def test_union_metadata(self, mock_update, mock_mark): """Test collection of stub data. """ - stubs = [Stub(resource_filename('desidatamodel.test', 't/fits_file.fits')), - Stub(resource_filename('desidatamodel.test', 't/fits_file.fits')), - Stub(resource_filename('desidatamodel.test', 't/fits_file.fits'))] + stubs = [Stub(self.test_files / 'fits_file.fits'), + Stub(self.test_files / 'fits_file.fits'), + Stub(self.test_files / 'fits_file.fits')] stubs[0].hdumeta[0]['extname'] = 'primary' stubs[1].hdumeta[1]['extname'] = '' stubs[2].nhdr = 1 - model = DataModel(resource_filename('desidatamodel.test', 't/fits_file_optional_columns.rst'), + model = DataModel(self.test_files / 'fits_file_optional_columns.rst', os.path.join(os.environ[DM], 'doc', 'examples')) foo = model.get_regexp('/desi/spectro/data') union_metadata(model, stubs) @@ -149,11 +149,3 @@ def test_options(self): self.assertEqual(options.number, 1000) self.assertEqual(options.model, 'DESI_SPECTRO_DATA/NIGHT/EXPID/desi-EXPID.rst') self.assertListEqual(options.directory, ['/desi/spectro/data']) - - -def test_suite(): - """Allows testing of only this module with the command:: - - python setup.py test -m - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/py/desidatamodel/test/test_stub.py b/py/desidatamodel/test/test_stub.py index 937f56a5..aedab83c 100644 --- a/py/desidatamodel/test/test_stub.py +++ b/py/desidatamodel/test/test_stub.py @@ -3,9 +3,9 @@ """Test desidatamodel.stub functions """ import os +import importlib.resources as ir import unittest from unittest.mock import patch, call -from pkg_resources import resource_filename from astropy.io import fits from astropy.io.fits.card import Undefined from collections import OrderedDict @@ -45,8 +45,7 @@ def test_Stub(self): # # Use a real file, and make sure no exceptions are raised. # - with fits.open(resource_filename('desidatamodel.test', - 't/fits_file.fits')) as hdulist: + with fits.open(self.test_files / 'fits_file.fits') as hdulist: stub = Stub(hdulist) self.assertEqual(stub.nhdr, 2) # @@ -626,8 +625,8 @@ def test_extract_keywords(self): def test_process_file(self): """Full test of parsing a FITS file. """ - filename = resource_filename('desidatamodel.test', 't/fits_file.fits') - modelfile = resource_filename('desidatamodel.test', 't/fits_file.rst') + filename = self.test_files / 'fits_file.fits' + modelfile = self.test_files / 'fits_file.rst' with open(modelfile) as m: modeldata = m.read() stub = Stub(filename) @@ -643,7 +642,7 @@ def test_read_column_descriptions(self): # this test mainly verifies that future edits of # data/column_descriptions.csv didn't break the format, # e.g. descriptions with spaces and commas are properly quoted - filename = resource_filename('desidatamodel', 'data/column_descriptions.csv') + filename = ir.files('desidatamodel') / 'data' / 'column_descriptions.csv' coldesc = read_column_descriptions(filename) colname = 'FLUX_R' self.assertIn(colname, coldesc.keys()) @@ -653,9 +652,9 @@ def test_read_column_descriptions(self): @patch('desidatamodel.stub.log') def test_Stub_with_descriptions(self, mock_log): - descfile = resource_filename('desidatamodel.test', 't/column_descriptions.csv') - filename = resource_filename('desidatamodel.test', 't/fits_file.fits') - filename_desc = resource_filename('desidatamodel.test', 't/fits_file_desc.fits') + descfile = self.test_files / 'column_descriptions.csv' + filename = self.test_files / 'fits_file.fits' + filename_desc = self.test_files / 'fits_file_desc.fits' # no descriptions stub = Stub(filename) @@ -682,15 +681,7 @@ def test_Stub_with_descriptions(self, mock_log): ]) # incorrect format column description file - baddescfile = resource_filename('desidatamodel.test', 't/bad_column_descriptions.csv') + baddescfile = self.test_files / 'bad_column_descriptions.csv' with self.assertRaises(ValueError): stub = Stub(filename, description_file=baddescfile) lines = str(stub) - - -def test_suite(): - """Allows testing of only this module with the command:: - - python setup.py test -m - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/py/desidatamodel/test/test_top_level.py b/py/desidatamodel/test/test_top_level.py index d073fcf5..da195864 100644 --- a/py/desidatamodel/test/test_top_level.py +++ b/py/desidatamodel/test/test_top_level.py @@ -22,11 +22,3 @@ def test_version(self): """Ensure the version conforms to PEP386/PEP440. """ self.assertRegex(theVersion, self.versionre) - - -def test_suite(): - """Allows testing of only this module with the command:: - - python setup.py test -m - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/py/desidatamodel/test/test_unit.py b/py/desidatamodel/test/test_unit.py index a147fa93..5da52169 100644 --- a/py/desidatamodel/test/test_unit.py +++ b/py/desidatamodel/test/test_unit.py @@ -4,8 +4,6 @@ """ # import os import unittest -# from pkg_resources import resource_filename -# from .. import DataModelError from .datamodeltestcase import DataModelTestCase from ..unit import DataModelUnit, log @@ -30,11 +28,3 @@ def test_check_model(self): c = u.check_unit('ergs', error=True) self.assertEqual(str(e.exception), erg_msg) self.assertLog(log, -1, erg_msg) - - -def test_suite(): - """Allows testing of only this module with the command:: - - python setup.py test -m - """ - return unittest.defaultTestLoader.loadTestsFromName(__name__) diff --git a/py/desidatamodel/test/test_update.py b/py/desidatamodel/test/test_update.py index 0850061b..e14d961f 100644 --- a/py/desidatamodel/test/test_update.py +++ b/py/desidatamodel/test/test_update.py @@ -4,7 +4,7 @@ """ import unittest import csv -from pkg_resources import resource_filename +import importlib.resources as ir from ..update import update @@ -15,7 +15,7 @@ def test_column_descriptions(self): """Ensure that every column described in the CSV file at least has a non-empty type and description. """ - coldef_file = resource_filename('desidatamodel', 'data/column_descriptions.csv') + coldef_file = ir.files('desidatamodel') / 'data' / 'column_descriptions.csv' with open(coldef_file, newline='') as csv_columns: reader = csv.reader(csv_columns) # diff --git a/py/desidatamodel/update.py b/py/desidatamodel/update.py index 1b8ab76b..d2b7242a 100755 --- a/py/desidatamodel/update.py +++ b/py/desidatamodel/update.py @@ -9,13 +9,13 @@ """ import re -import sys +import importlib.resources as ir from html import escape -from pkg_resources import resource_filename +# from pkg_resources import resource_filename import argparse import numpy as np from astropy.table import Table -from astropy.io.ascii import RST +# from astropy.io.ascii import RST from desiutil.log import get_logger @@ -128,7 +128,8 @@ def update(lines, force=False): """ log = get_logger() - coldef_file = resource_filename('desidatamodel', 'data/column_descriptions.csv') + # coldef_file = resource_filename('desidatamodel', 'data/column_descriptions.csv') + coldef_file = ir.files('desidatamodel') / 'data' / 'column_descriptions.csv' coldefs = read_column_descriptions(coldef_file) output_lines = list() @@ -216,8 +217,6 @@ def main(): :class:`int` An integer suitable for passing to :func:`sys.exit`. """ - - import argparse parser = argparse.ArgumentParser() parser.add_argument('-i', '--infile', required=True, help='Input model filename') diff --git a/requirements.txt b/requirements.txt index c39194c9..ebe7d3bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ astropy sphinx-toolbox +jinja2 git+https://github.com/desihub/desiutil.git@3.3.0#egg=desiutil diff --git a/setup.py b/setup.py index 1302561e..a686072b 100755 --- a/setup.py +++ b/setup.py @@ -64,7 +64,10 @@ # Autogenerate command-line scripts. # # setup_keywords['entry_points'] = {'console_scripts': ['check_model = desidatamodel.check:main', -# 'generate_model = desidatamodel.stub:main']} +# 'deep_scan_metadata = desidatamodel.scan:main', +# 'generate_model = desidatamodel.stub:main', +# 'update_bitmasks = desidatamodel.bitmasks:main', +# 'update_column_descriptions = desidatamodel.update:main']} # # Add internal data directories. #