Include user documents (#90)

* Clarify directory naming

* Refactor to write directly into wrapped directory

* Add basic c++ test with custom conf.py

* Create wrapped_sphinx_directory with an absolute path

* Limit navigation_depth to 4 (#89)

* Include user documents

* Copy all contents of /doc

* Fixups to match with refactor-sphinx-directories

* Fix flake errors, use dir/ instead of doc/dir titles

* Add tests for documentation

rosdoc2/verbs/build/builders/sphinx_builder.py

@@ -24,6 +24,7 @@
 from ..collect_inventory_files import collect_inventory_files
 from ..create_format_map_from_package import create_format_map_from_package
 from ..generate_interface_docs import generate_interface_docs
+from ..include_user_docs import include_user_docs
 
 logger = logging.getLogger('rosdoc2')
 
@@ -33,7 +34,7 @@ def esc_backslash(path):
     return path.replace('\\', '\\\\') if path else path
 
 
-def generate_package_toc_entry(*, build_context, interface_counts) -> str:
+def generate_package_toc_entry(*, build_context, interface_counts, doc_directories) -> str:
     """Construct a table of content (toc) entry for the package being processed."""
     build_type = build_context.build_type
     always_run_doxygen = build_context.always_run_doxygen
@@ -43,9 +44,16 @@ def generate_package_toc_entry(*, build_context, interface_counts) -> str:
     # inside the string to fall under the `:toctree:` directive
     toc_entry_cpp = '   C++ API <generated/index>\n'
     toc_entry_py = '   Python API <modules>\n'
-    toc_entry_msg = '   Message Definitions <generated/message_definitions>\n'
-    toc_entry_srv = '   Service Definitions <generated/service_definitions>\n'
-    toc_entry_action = '   Action Definitions <generated/action_definitions>\n'
+    toc_entry_msg = '   Message Definitions <interfaces/message_definitions>\n'
+    toc_entry_srv = '   Service Definitions <interfaces/service_definitions>\n'
+    toc_entry_action = '   Action Definitions <interfaces/action_definitions>\n'
+    toc_doc_entry = """\
+.. toctree::
+   :titlesonly:
+   :maxdepth: 2
+
+   Documentation <user_docs>
+"""
 
     toc_entry = '\n'
 
@@ -59,6 +67,11 @@ def generate_package_toc_entry(*, build_context, interface_counts) -> str:
         toc_entry += toc_entry_srv
     if interface_counts['action'] > 0:
         toc_entry += toc_entry_action
+
+    # User documentation
+    if doc_directories:
+        toc_entry += toc_doc_entry
+
     return toc_entry
 
 
@@ -450,10 +463,14 @@ def build(self, *, doc_build_folder, output_staging_directory):
         interface_counts = generate_interface_docs(
             package_xml_directory,
             self.build_context.package.name,
-            os.path.join(wrapped_sphinx_directory, 'generated')
+            os.path.join(wrapped_sphinx_directory, 'interfaces')
         )
         logger.info(f'interface_counts: {interface_counts}')
 
+        # include user documentation
+        doc_directories = include_user_docs(package_xml_directory, wrapped_sphinx_directory)
+        logger.info(f'doc_directories: {doc_directories}')
+
         # Check if the user provided a sphinx directory.
         sphinx_project_directory = self.sphinx_sourcedir
         if sphinx_project_directory is not None:
@@ -501,7 +518,8 @@ def build(self, *, doc_build_folder, output_staging_directory):
             sphinx_project_directory,
             python_src_directory,
             intersphinx_mapping_extensions,
-            interface_counts)
+            interface_counts,
+            doc_directories)
 
         # If the package has python code, then invoke `sphinx-apidoc` before building
         has_python = self.build_context.build_type == 'ament_python' or \
@@ -623,6 +641,7 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory(
         python_src_directory,
         intersphinx_mapping_extensions,
         interface_counts,
+        doc_directories,
     ):
         """Generate the rosdoc2 sphinx project configuration files."""
         # Generate a default index.rst
@@ -633,7 +652,8 @@ def generate_wrapping_rosdoc2_sphinx_project_into_directory(
             'root_title_underline': '=' * len(root_title),
             'package_toc_entry': generate_package_toc_entry(
                 build_context=self.build_context,
-                interface_counts=interface_counts)
+                interface_counts=interface_counts,
+                doc_directories=doc_directories)
         })
 
         with open(os.path.join(wrapped_sphinx_directory, 'index.rst'), 'w+') as f:

rosdoc2/verbs/build/include_user_docs.py

@@ -0,0 +1,98 @@
+# Copyright 2024 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import shutil
+
+from rosdoc2.slugify import slugify
+
+logger = logging.getLogger('rosdoc2')
+
+documentation_rst_template = """\
+Documentation
+=============
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   user_docs/*
+"""
+
+subdirectory_rst_template = """\
+{name}/
+{name_underline}=
+
+.. toctree::
+   :caption: Documentation in this subdirectory
+   :maxdepth: 2
+   :glob:
+
+   user_docs/{name}/*
+"""
+
+
+def include_user_docs(package_dir: str,
+                      output_dir: str,
+                      ):
+    """Generate rst files for user documents."""
+    logger.info(f'include_user_docs: package_dir {package_dir} output_dir {output_dir}')
+    # Search the ./doc directory
+    doc_dir = os.path.join(package_dir, 'doc')
+    # Search /doc to insure there is at least one item of renderable documentation
+    doc_directories = []
+    for root, _, files in os.walk(doc_dir):
+        for file in files:
+            # ensure a valid documentation file exists, directories might only contain resources.
+            (_, ext) = os.path.splitext(file)
+            if ext in ['.rst', '.md', '.markdown']:
+                logger.debug(f'Found renderable documentation file in {root} named {file}')
+                relpath = os.path.relpath(root, doc_dir)
+                relpath = relpath.replace('\\', '/')
+                doc_directories.append(relpath)
+                break
+
+    if not doc_directories:
+        logger.debug('no documentation found in /doc')
+        return doc_directories
+
+    logger.info(f'Documentation found in /doc in directories {doc_directories}')
+    # At this point we know that there are some directories that have documentation in them under
+    # /doc, but we do not know which ones might also be needed for images or includes. So we copy
+    # everything to the output directory.
+    shutil.copytree(
+        os.path.abspath(doc_dir),
+        os.path.abspath(os.path.join(output_dir, 'user_docs')),
+        dirs_exist_ok=True)
+
+    toc_content = documentation_rst_template
+    # generate a glob rst entry for each directory with documents
+    for relpath in doc_directories:
+        # directories that will be explicitly listed in index.rst
+        if relpath == '.':
+            continue
+        docname = 'user_docs_' + slugify(relpath)  # This is the name that sphinx uses
+        content = subdirectory_rst_template.format_map(
+            {'name': relpath, 'name_underline': '=' * len(relpath)})
+        sub_path = os.path.join(output_dir, docname + '.rst')
+        with open(sub_path, 'w+') as f:
+            f.write(content)
+        toc_content += f'   {relpath}/ <{docname}>\n'
+
+    sub_path = os.path.join(output_dir, 'user_docs.rst')
+    with open(sub_path, 'w+') as f:
+        f.write(toc_content)
+
+    return doc_directories

test/test_builder.py

@@ -197,14 +197,16 @@ def test_full_package(session_dir):
         'c++ api',
         'message definitions',
         'service definitions',
-        'action definitions'
+        'action definitions',
+        'instructions',  # has documentation
     ]
     file_includes = [
         'generated/index.html'
     ]
     links_exist = [
         'full_package.dummy.html',
         'modules.html',
+        'user_docs/morestuff/more_of_more/subsub.html'  # a deep documentation file
     ]
     excludes = [
         'dontshowme'
@@ -240,7 +242,7 @@ def test_only_messages(session_dir):
         PKG_NAME,
         'message definitions',
     ]
-    links_exist = ['generated/msg/NumPwrResult.html']
+    links_exist = ['interfaces/msg/NumPwrResult.html']
 
     do_test_package(PKG_NAME, session_dir, includes=includes, links_exist=links_exist)