From 9e6297fadf8d20dac7a5503da07a8db4bfaeabd6 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Thu, 25 Apr 2024 07:25:39 -0700 Subject: [PATCH 1/8] Add CodeBase object to store code base information Represents a code base as a combination of: - A list of (source) directories; and - A list of exclude patterns. The intent of this object is to replace usages of untyped dictionaries to store code base information, to improve documentation and usability. Signed-off-by: John Pennycook --- codebasin/__init__.py | 113 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/codebasin/__init__.py b/codebasin/__init__.py index 9530e88..d215d52 100644 --- a/codebasin/__init__.py +++ b/codebasin/__init__.py @@ -1,7 +1,11 @@ # Copyright (C) 2019-2024 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause +import os import shlex import warnings +from pathlib import Path + +import pathspec import codebasin.source import codebasin.walkers @@ -123,3 +127,112 @@ def from_json(cls, instance: dict): command=command, output=output, ) + + +class CodeBase: + """ + A representation of all source files in the code base. + + Attributes + ---------- + directories: list[str | os.PathLike[str]] + The set of source directories that make up the code base. + + exclude_patterns: list[str] + A set of patterns describing source files excluded from the code base. + """ + + def __init__( + self, + *directories: str | os.PathLike[str], + exclude_patterns: list[str] = [], + ): + """ + Raises + ------ + TypeError + If any directory in `directories` is not a path. + If `exclude_patterns` is not a list of strings. + """ + if not isinstance(exclude_patterns, list): + raise TypeError("'exclude_patterns' must be a list.") + if not all([isinstance(d, (str, os.PathLike)) for d in directories]): + raise TypeError( + "Each directory in 'directories' must be PathLike.", + ) + if not all([isinstance(p, str) for p in exclude_patterns]): + raise TypeError( + "Each pattern in 'exclude_patterns' must be a string.", + ) + self._directories = [Path(d).resolve() for d in directories] + self._excludes = exclude_patterns + + def __repr__(self): + return ( + f"CodeBase(directories={self.directories}, " + + f"exclude_patterns={self.exclude_patterns})" + ) + + @property + def directories(self): + return [str(d) for d in self._directories] + + @property + def exclude_patterns(self): + return self._excludes + + def __contains__(self, path: os.PathLike) -> bool: + """ + Returns + _______ + bool + True if `path` is a recognized source file in one of the code + base's listed directories and does not match any exclude + pattern(s). + """ + path = Path(path).resolve() + + # Files that don't exist aren't part of the code base. + if not path.exists(): + return False + + # Directories cannot be source files. + if path.is_dir(): + return False + + # Files with unrecognized extensions are not source files. + if not codebasin.source.is_source_file(path): + return False + + # Files outside of any directory are not in the code base. + # Store the root for evaluation of relative exclude paths later. + root = None + for directory in self.directories: + if path.is_relative_to(directory): + root = directory + break + if root is None: + return False + + # Files matching an exclude pattern are not in the code base. + # + # Use GitIgnoreSpec to match git behavior in weird corner cases. + # Convert relative paths to match .gitignore subdirectory behavior. + spec = pathspec.GitIgnoreSpec.from_lines(self.exclude_patterns) + try: + relative_path = path.relative_to(root) + if spec.match_file(relative_path): + return False + except ValueError: + pass + + return True + + def __iter__(self): + """ + Iterate over all files in the code base by walking each directory. + """ + for directory in self.directories: + for path in Path(directory).rglob("*"): + if self.__contains__(path): + yield str(path) From 5fc4f8f3c701797d5672675bce400dd581d3c95e Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Thu, 25 Apr 2024 07:28:28 -0700 Subject: [PATCH 2/8] Replace uses of code base dictionary with CodeBase Signed-off-by: John Pennycook --- bin/codebasin | 25 ++++++++----------- codebasin/finder.py | 4 ++-- codebasin/walkers/exporter.py | 6 +---- codebasin/walkers/platform_mapper.py | 36 +--------------------------- 4 files changed, 14 insertions(+), 57 deletions(-) diff --git a/bin/codebasin b/bin/codebasin index 96a3874..3b0c8a5 100755 --- a/bin/codebasin +++ b/bin/codebasin @@ -10,7 +10,7 @@ import logging import os import sys -from codebasin import config, finder, report, util +from codebasin import CodeBase, config, finder, report, util from codebasin.walkers.platform_mapper import PlatformMapper version = "1.2.0" @@ -107,14 +107,7 @@ def main(): # Determine the root directory based on where codebasin is run. rootdir = os.path.realpath(os.getcwd()) - # Set up a default codebase and configuration object. - codebase = { - "files": [], - "platforms": [], - "exclude_files": set(), - "exclude_patterns": args.excludes, - "rootdir": rootdir, - } + # Set up a default configuration object. configuration = {} # Load the analysis file if it exists. @@ -132,8 +125,7 @@ def main(): if "codebase" in analysis_toml: if "exclude" in analysis_toml["codebase"]: - excludes = analysis_toml["codebase"]["exclude"] - codebase["exclude_patterns"] += excludes + args.excludes += analysis_toml["codebase"]["exclude"] for name in args.platforms: if name not in analysis_toml["platform"].keys(): @@ -142,16 +134,20 @@ def main(): + "does not exist in the configuration file.", ) + cmd_platforms = args.platforms.copy() for name in analysis_toml["platform"].keys(): - if args.platforms and name not in args.platforms: + if cmd_platforms and name not in cmd_platforms: continue if "commands" not in analysis_toml["platform"][name]: raise ValueError(f"Missing 'commands' for platform {name}") p = analysis_toml["platform"][name]["commands"] db = config.load_database(p, rootdir) - codebase["platforms"].append(name) + args.platforms.append(name) configuration.update({name: db}) + # Construct a codebase object associated with the root directory. + codebase = CodeBase(rootdir, exclude_patterns=args.excludes) + # Parse the source tree, and determine source line associations. # The trees and associations are housed in state. state = finder.find( @@ -180,8 +176,7 @@ def main(): if report_enabled("clustering"): basename = os.path.basename(args.analysis_file) filename = os.path.splitext(basename)[0] - platform_names = [p for p in codebase["platforms"]] - output_prefix = "-".join([filename] + platform_names) + output_prefix = "-".join([filename] + args.platforms) clustering_output_name = output_prefix + "-dendrogram.png" clustering = report.clustering(clustering_output_name, setmap) diff --git a/codebasin/finder.py b/codebasin/finder.py index c64afce..9db03af 100644 --- a/codebasin/finder.py +++ b/codebasin/finder.py @@ -142,11 +142,11 @@ def find( # Build a tree for each unique file for all platforms. state = ParserState(summarize_only) - for f in codebase["files"]: + for f in codebase: state.insert_file(f) for p in configuration: for e in configuration[p]: - if e["file"] not in codebase["files"]: + if e["file"] not in codebase: filename = e["file"] if legacy_warnings: log.warning( diff --git a/codebasin/walkers/exporter.py b/codebasin/walkers/exporter.py index 23b86d7..d5db7b8 100644 --- a/codebasin/walkers/exporter.py +++ b/codebasin/walkers/exporter.py @@ -6,7 +6,6 @@ from codebasin import util from codebasin.preprocessor import CodeNode, FileNode -from codebasin.walkers.platform_mapper import exclude from codebasin.walkers.tree_walker import TreeWalker log = logging.getLogger("codebasin") @@ -38,10 +37,7 @@ def walk(self, state): def _export_node(self, _filename, _node, _map): # Do not export files that the user does not consider to be part of # the codebase - if isinstance(_node, FileNode) and exclude( - _node.filename, - self.codebase, - ): + if isinstance(_node, FileNode) and _node.filename not in self.codebase: return if isinstance(_node, CodeNode): diff --git a/codebasin/walkers/platform_mapper.py b/codebasin/walkers/platform_mapper.py index f616874..753a1ae 100644 --- a/codebasin/walkers/platform_mapper.py +++ b/codebasin/walkers/platform_mapper.py @@ -2,9 +2,6 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os - -import pathspec from codebasin.preprocessor import CodeNode, FileNode from codebasin.walkers.tree_mapper import TreeMapper @@ -12,34 +9,6 @@ log = logging.getLogger("codebasin") -def exclude(filename, cb): - # Always exclude files that were explicitly listed as excluded. - if filename in cb["exclude_files"]: - log.info(f"Excluding {filename}; matches 'exclude_files'.") - return True - - # Only exclude files outside of the root directory if they weren't - # explicitly listed as part of the codebase. - path = os.path.realpath(filename) - if not path.startswith(cb["rootdir"]): - if filename in cb["files"]: - return False - log.info(f"Excluding {filename}; outside of root directory.") - return True - - # Exclude files matching an exclude pattern. - # - # Use GitIgnoreSpec to match git behavior in weird corner cases. - # Convert relative paths to match .gitignore subdirectory behavior. - spec = pathspec.GitIgnoreSpec.from_lines(cb["exclude_patterns"]) - rel = os.path.relpath(path, cb["rootdir"]) - if spec.match_file(rel): - log.info(f"Excluding {filename}; matches exclude pattern.") - return True - - return False - - class PlatformMapper(TreeMapper): """ Specific TreeMapper that builds a mapping of nodes to platforms. @@ -57,10 +26,7 @@ def _map_node(self, _node, _map): """ # Do not map files that the user does not consider to be part of # the codebase - if isinstance(_node, FileNode) and exclude( - _node.filename, - self.codebase, - ): + if isinstance(_node, FileNode) and _node.filename not in self.codebase: return if isinstance(_node, CodeNode): From ab645f8144f9f14bf10484e6cf1e8ca0aff18e90 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Thu, 25 Apr 2024 07:29:25 -0700 Subject: [PATCH 3/8] Update tests to use CodeBase Since a lot of the functionality driven by the tests has not yet been updated to use Path, a lot of casts between Path and string are introduced here. These casts will eventually be removed, but require changes to other functionality. Signed-off-by: John Pennycook --- tests/basic_asm/test_basic_asm.py | 35 ++---- tests/basic_fortran/test_basic_fortran.py | 22 ++-- tests/build-dir/test_build_dir.py | 24 ++-- tests/code-base/__init__.py | 2 + tests/code-base/test_code_base.py | 106 ++++++++++++++++++ .../test_commented_directive.py | 22 ++-- tests/define/test_define.py | 22 ++-- tests/disjoint/test_disjoint.py | 44 ++------ tests/duplicates/test_duplicates.py | 18 +-- tests/exclude/test_exclude.py | 23 ++-- tests/include/test_include.py | 28 ++--- tests/literals/test_literals.py | 22 ++-- tests/macro_expansion/test_macro_expansion.py | 30 ++--- tests/multi_line/test_multi_line.py | 22 ++-- tests/nesting/test_nesting.py | 22 ++-- tests/once/test_once.py | 23 ++-- tests/operators/test_operators.py | 22 ++-- 17 files changed, 225 insertions(+), 262 deletions(-) create mode 100644 tests/code-base/__init__.py create mode 100644 tests/code-base/test_code_base.py diff --git a/tests/basic_asm/test_basic_asm.py b/tests/basic_asm/test_basic_asm.py index 63d1c88..32265f6 100644 --- a/tests/basic_asm/test_basic_asm.py +++ b/tests/basic_asm/test_basic_asm.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,25 +15,16 @@ class TestBasicAsm(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/basic_asm/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = {frozenset(["CPU"]): 24} def test_yaml(self): """basic_asm/basic_asm.yaml""" - files = ["test.s", "test.S", "test.asm"] - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, f)) for f in files - ], - "platforms": ["CPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) entries = [] - for f in codebase["files"]: + for f in codebase: entries.append( { "file": f, @@ -43,7 +34,7 @@ def test_yaml(self): }, ) configuration = {"CPU": entries} - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( @@ -54,17 +45,9 @@ def test_yaml(self): def test_ptx(self): """basic_asm/basic_asm_ptx.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "test.ptx")), - ], - "platforms": ["GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) entry = { - "file": codebase["files"][0], + "file": str(self.rootdir / "test.ptx"), "defines": [], "include_paths": [], "include_files": [], @@ -73,7 +56,7 @@ def test_ptx(self): self.assertRaises( RuntimeError, finder.find, - self.rootdir, + str(self.rootdir), codebase, configuration, ) diff --git a/tests/basic_fortran/test_basic_fortran.py b/tests/basic_fortran/test_basic_fortran.py index 623224c..a9455d2 100644 --- a/tests/basic_fortran/test_basic_fortran.py +++ b/tests/basic_fortran/test_basic_fortran.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,7 +15,7 @@ class TestBasicFortran(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/basic_fortran/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -26,19 +26,11 @@ def setUp(self): def test_yaml(self): """basic_fortran/basic_fortran.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "test.f90")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "test.f90"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -46,14 +38,14 @@ def test_yaml(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "test.f90"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/build-dir/test_build_dir.py b/tests/build-dir/test_build_dir.py index 1020234..ed08211 100644 --- a/tests/build-dir/test_build_dir.py +++ b/tests/build-dir/test_build_dir.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path -from codebasin import config, finder +from codebasin import CodeBase, config, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -17,7 +17,7 @@ class TestBuildDirectories(unittest.TestCase): """ def setUp(self): - self.rootdir = str(Path(__file__).parent) + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = False def test_absolute_paths(self): @@ -26,11 +26,11 @@ def test_absolute_paths(self): All "file" fields are absolute paths. """ - source = str(Path(__file__).parent.joinpath("foo.cpp")) + source = self.rootdir / "foo.cpp" # CBI only understands how to load compilation databases from file. # For now, create temporary files every time we test. - dir1 = str(Path(__file__).parent.joinpath("build1/")) + dir1 = self.rootdir / "build1/" build1 = tempfile.NamedTemporaryFile() json1 = [ { @@ -42,7 +42,7 @@ def test_absolute_paths(self): with open(build1.name, "w") as f: json.dump(json1, f) - dir2 = str(Path(__file__).parent.joinpath("build2/")) + dir2 = self.rootdir / "build2/" build2 = tempfile.NamedTemporaryFile() json2 = [ { @@ -54,13 +54,7 @@ def test_absolute_paths(self): with open(build2.name, "w") as f: json.dump(json2, f) - codebase = { - "files": [source], - "platforms": ["one", "two"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = {} for name, path in [("one", build1.name), ("two", build2.name)]: @@ -69,7 +63,7 @@ def test_absolute_paths(self): expected_setmap = {frozenset(["one", "two"]): 1} - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap") @@ -80,11 +74,11 @@ def test_empty_platform(self): This may be a sign that the compilation database has incorrect paths. """ - source = str(Path(__file__).parent.joinpath("foo.cpp")) + source = self.rootdir / "foo.cpp" # CBI only understands how to load compilation databases from file. # For now, create temporary files every time we test. - build = str(Path(__file__).parent.joinpath("build/")) + build = self.rootdir / "build/" tmp = tempfile.NamedTemporaryFile() obj = [ { diff --git a/tests/code-base/__init__.py b/tests/code-base/__init__.py new file mode 100644 index 0000000..94adb81 --- /dev/null +++ b/tests/code-base/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2019-2024 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause diff --git a/tests/code-base/test_code_base.py b/tests/code-base/test_code_base.py new file mode 100644 index 0000000..72b0daf --- /dev/null +++ b/tests/code-base/test_code_base.py @@ -0,0 +1,106 @@ +# Copyright (C) 2019-2024 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause + +import logging +import tempfile +import unittest +import warnings +from pathlib import Path + +from codebasin import CodeBase + + +class TestCodeBase(unittest.TestCase): + """ + Test CodeBase class. + """ + + def setUp(self): + logging.getLogger("codebasin").disabled = False + warnings.simplefilter("ignore", ResourceWarning) + + # Create a temporary codebase spread across two directories + self.tmp1 = tempfile.TemporaryDirectory() + self.tmp2 = tempfile.TemporaryDirectory() + p1 = Path(self.tmp1.name) + p2 = Path(self.tmp2.name) + open(p1 / "foo.cpp", mode="w").close() + open(p1 / "bar.cpp", mode="w").close() + open(p1 / "baz.h", mode="w").close() + open(p1 / "README.md", mode="w").close() + open(p2 / "qux.cpp", mode="w").close() + open(p2 / "quux.h", mode="w").close() + open(p2 / "README.md", mode="w").close() + + def test_constructor(self): + """Check directories and exclude_patterns are handled correctly""" + path = Path(self.tmp1.name) + codebase = CodeBase(path, exclude_patterns=["*.h"]) + self.assertTrue(codebase.directories == [str(path)]) + self.assertTrue(codebase.exclude_patterns == ["*.h"]) + + def test_constructor_validation(self): + """Check directories and exclude_patterns are valid""" + + with self.assertRaises(TypeError): + CodeBase(exclude_patterns="*") + + with self.assertRaises(TypeError): + CodeBase(1, "2", 3) + + with self.assertRaises(TypeError): + CodeBase(exclude_patterns=[1, "2", 3]) + + def test_repr(self): + """Check implementation of __repr__""" + path = Path(self.tmp1.name) + codebase = CodeBase(path, exclude_patterns=["*.h"]) + self.assertTrue( + codebase.__repr__(), + f'CodeBase(directories=[{path}], exclude_patterns=[".h"])', + ) + + def test_contains(self): + """Check implementation of __contains__""" + p1 = Path(self.tmp1.name) + p2 = Path(self.tmp2.name) + codebase = CodeBase(p1, p2, exclude_patterns=["*.h"]) + + # Files in the temporary directories should be in the code base. + self.assertTrue(p1 / "foo.cpp" in codebase) + self.assertTrue(p1 / "bar.cpp" in codebase) + self.assertTrue(p2 / "qux.cpp" in codebase) + + # Files that match exclude pattern(s) should not be in the code base. + self.assertFalse(p1 / "baz.h" in codebase) + self.assertFalse(p2 / "quux.h" in codebase) + + # Files that don't exist should not be in the code base. + self.assertFalse(p1 / "asdf.cpp" in codebase) + self.assertFalse(p2 / "asdf.cpp" in codebase) + + # The temporary directories themselves should not be in the code base. + self.assertFalse(p1 in codebase) + self.assertFalse(p2 in codebase) + + # Non-source files should not be in the code base. + self.assertFalse(p1 / "README.md" in codebase) + self.assertFalse(p2 / "README.md" in codebase) + + def test_iterator(self): + """Check implementation of __iter__""" + p1 = Path(self.tmp1.name) + p2 = Path(self.tmp2.name) + codebase = CodeBase(p1, p2, exclude_patterns=["*.h"]) + + files = [f for f in codebase] + expected = [ + str(p1 / "bar.cpp"), + str(p1 / "foo.cpp"), + str(p2 / "qux.cpp"), + ] + self.assertEqual(files, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/commented_directive/test_commented_directive.py b/tests/commented_directive/test_commented_directive.py index 6b882de..e4f3f42 100644 --- a/tests/commented_directive/test_commented_directive.py +++ b/tests/commented_directive/test_commented_directive.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -16,7 +16,7 @@ class TestCommentedDirective(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/commented_directive/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = {frozenset(["CPU", "GPU"]): 5} @@ -30,19 +30,11 @@ def count_children_nodes(self, node): def test_yaml(self): """commented_directive/commented_directive.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -50,14 +42,14 @@ def test_yaml(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) diff --git a/tests/define/test_define.py b/tests/define/test_define.py index 15ca3ff..7814230 100644 --- a/tests/define/test_define.py +++ b/tests/define/test_define.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,7 +15,7 @@ class TestDefine(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/define/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -25,19 +25,11 @@ def setUp(self): def test_yaml(self): """define/define.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -45,14 +37,14 @@ def test_yaml(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/disjoint/test_disjoint.py b/tests/disjoint/test_disjoint.py index d412a6e..1f29f58 100644 --- a/tests/disjoint/test_disjoint.py +++ b/tests/disjoint/test_disjoint.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -17,59 +17,33 @@ class TestDisjointCodebase(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/disjoint/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = {frozenset(["CPU"]): 6, frozenset(["GPU"]): 6} def test_yaml(self): """disjoint/disjoint.yaml""" - files = [ - "cpu.cpp", - "gpu.cpp", - "cpu_headers/header.h", - "gpu_headers/header.h", - ] - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, f)) for f in files - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": os.path.realpath( - os.path.join(self.rootdir, "cpu.cpp"), - ), + "file": str(self.rootdir / "cpu.cpp"), "defines": ["CPU"], - "include_paths": [ - os.path.realpath( - os.path.join(self.rootdir, "cpu_headers"), - ), - ], + "include_paths": [str(self.rootdir / "cpu_headers")], "include_files": [], }, ], "GPU": [ { - "file": os.path.realpath( - os.path.join(self.rootdir, "gpu.cpp"), - ), + "file": str(self.rootdir / "gpu.cpp"), "defines": ["GPU"], - "include_paths": [ - os.path.realpath( - os.path.join(self.rootdir, "gpu_headers"), - ), - ], + "include_paths": [str(self.rootdir / "gpu_headers")], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/duplicates/test_duplicates.py b/tests/duplicates/test_duplicates.py index 5df13fc..c5b2b36 100644 --- a/tests/duplicates/test_duplicates.py +++ b/tests/duplicates/test_duplicates.py @@ -5,7 +5,7 @@ import unittest from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,22 +15,16 @@ class TestDuplicates(unittest.TestCase): """ def setUp(self): - self.rootdir = str(Path(__file__).parent) + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True def test_duplicates(self): """Check that duplicate files count towards divergence.""" - cpufile = str(Path(__file__).parent.joinpath("cpu/foo.cpp")) - gpufile = str(Path(__file__).parent.joinpath("gpu/foo.cpp")) + cpufile = str(self.rootdir / "cpu/foo.cpp") + gpufile = str(self.rootdir / "gpu/foo.cpp") - codebase = { - "files": [cpufile, gpufile], - "platforms": ["cpu", "gpu"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "cpu": [ @@ -53,7 +47,7 @@ def test_duplicates(self): expected_setmap = {frozenset(["cpu"]): 1, frozenset(["gpu"]): 1} - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap") diff --git a/tests/exclude/test_exclude.py b/tests/exclude/test_exclude.py index ba5f13b..82c3693 100644 --- a/tests/exclude/test_exclude.py +++ b/tests/exclude/test_exclude.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import config, finder +from codebasin import CodeBase, config, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,20 +15,19 @@ class TestExclude(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/exclude/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True def _get_setmap(self, excludes): - codebase = { - "files": [], - "platforms": ["test"], - "exclude_files": set(), - "exclude_patterns": excludes, - "rootdir": os.path.realpath(self.rootdir), + codebase = CodeBase( + self.rootdir, + exclude_patterns=excludes, + ) + dbpath = self.rootdir / "commands.json" + configuration = { + "test": config.load_database(str(dbpath), str(self.rootdir)), } - dbpath = os.path.realpath(os.path.join(self.rootdir, "commands.json")) - configuration = {"test": config.load_database(dbpath, self.rootdir)} - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) return setmap diff --git a/tests/include/test_include.py b/tests/include/test_include.py index 74a0fcf..4faa0b3 100644 --- a/tests/include/test_include.py +++ b/tests/include/test_include.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import config, finder +from codebasin import CodeBase, config, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -16,7 +16,7 @@ class TestInclude(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/include/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -27,26 +27,16 @@ def setUp(self): def test_include(self): """include/include.yaml""" - codebase = { - "files": [], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": os.path.realpath(self.rootdir), - } + codebase = CodeBase(self.rootdir) - cpu_path = os.path.realpath( - os.path.join(self.rootdir, "cpu_commands.json"), - ) - gpu_path = os.path.realpath( - os.path.join(self.rootdir, "gpu_commands.json"), - ) + cpu_path = self.rootdir / "cpu_commands.json" + gpu_path = self.rootdir / "gpu_commands.json" configuration = { - "CPU": config.load_database(cpu_path, self.rootdir), - "GPU": config.load_database(gpu_path, self.rootdir), + "CPU": config.load_database(str(cpu_path), str(self.rootdir)), + "GPU": config.load_database(str(gpu_path), str(self.rootdir)), } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/literals/test_literals.py b/tests/literals/test_literals.py index ecfe582..0f38d52 100644 --- a/tests/literals/test_literals.py +++ b/tests/literals/test_literals.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder, preprocessor +from codebasin import CodeBase, finder, preprocessor from codebasin.walkers.platform_mapper import PlatformMapper @@ -16,26 +16,18 @@ class TestLiterals(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/literals/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = {frozenset(["CPU", "GPU"]): 9} def test_literals(self): """literals/literals.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["USE_CPU"], "include_paths": [], "include_files": [], @@ -43,14 +35,14 @@ def test_literals(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["USE_GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/macro_expansion/test_macro_expansion.py b/tests/macro_expansion/test_macro_expansion.py index b7bb6a4..16ac787 100644 --- a/tests/macro_expansion/test_macro_expansion.py +++ b/tests/macro_expansion/test_macro_expansion.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder, platform, preprocessor +from codebasin import CodeBase, finder, platform, preprocessor from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,7 +15,7 @@ class TestMacroExpansion(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/macro_expansion/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -27,27 +27,13 @@ def setUp(self): def test_macro_expansion(self): """macro_expansion/macro_expansion.yaml""" - files = [ - "defined_undefined_test.cpp", - "infinite_loop_test.cpp", - "function_like_test.cpp", - "max_level.cpp", - ] - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, f)) for f in files - ], - "platforms": ["CPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) cpu_entries = [] gpu_entries = [] - for f in files: + for f in list(codebase): cpu_entries.append( { - "file": os.path.realpath(os.path.join(self.rootdir, f)), + "file": str(f), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -55,7 +41,7 @@ def test_macro_expansion(self): ) gpu_entries.append( { - "file": os.path.realpath(os.path.join(self.rootdir, f)), + "file": str(f), "defines": ["GPU"], "include_paths": [], "include_files": [], @@ -65,7 +51,7 @@ def test_macro_expansion(self): "CPU": cpu_entries, "GPU": gpu_entries, } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/multi_line/test_multi_line.py b/tests/multi_line/test_multi_line.py index 4567300..958f3f0 100644 --- a/tests/multi_line/test_multi_line.py +++ b/tests/multi_line/test_multi_line.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,7 +15,7 @@ class TestMultiLine(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/multi_line/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -25,19 +25,11 @@ def setUp(self): def test_yaml(self): """multi_line/multi_line.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -45,14 +37,14 @@ def test_yaml(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/nesting/test_nesting.py b/tests/nesting/test_nesting.py index 7c504ff..1684f04 100644 --- a/tests/nesting/test_nesting.py +++ b/tests/nesting/test_nesting.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,7 +15,7 @@ class TestNesting(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/nesting/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -26,19 +26,11 @@ def setUp(self): def test_yaml(self): """nesting/nesting.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -46,14 +38,14 @@ def test_yaml(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/once/test_once.py b/tests/once/test_once.py index 933aee4..e380ac5 100644 --- a/tests/once/test_once.py +++ b/tests/once/test_once.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder +from codebasin import CodeBase, finder from codebasin.walkers.platform_mapper import PlatformMapper @@ -15,7 +15,7 @@ class TestOnce(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/once/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = { @@ -25,20 +25,11 @@ def setUp(self): def test_yaml(self): """once/once.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - os.path.realpath(os.path.join(self.rootdir, "once.h")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -46,14 +37,14 @@ def test_yaml(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/operators/test_operators.py b/tests/operators/test_operators.py index bbd6201..d0c4256 100644 --- a/tests/operators/test_operators.py +++ b/tests/operators/test_operators.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: BSD-3-Clause import logging -import os import unittest +from pathlib import Path -from codebasin import finder, platform, preprocessor +from codebasin import CodeBase, finder, platform, preprocessor from codebasin.walkers.platform_mapper import PlatformMapper @@ -16,26 +16,18 @@ class TestOperators(unittest.TestCase): """ def setUp(self): - self.rootdir = "./tests/operators/" + self.rootdir = Path(__file__).parent.resolve() logging.getLogger("codebasin").disabled = True self.expected_setmap = {frozenset(["CPU", "GPU"]): 32} def test_operators(self): """operators/operators.yaml""" - codebase = { - "files": [ - os.path.realpath(os.path.join(self.rootdir, "main.cpp")), - ], - "platforms": ["CPU", "GPU"], - "exclude_files": set(), - "exclude_patterns": [], - "rootdir": self.rootdir, - } + codebase = CodeBase(self.rootdir) configuration = { "CPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["CPU"], "include_paths": [], "include_files": [], @@ -43,14 +35,14 @@ def test_operators(self): ], "GPU": [ { - "file": codebase["files"][0], + "file": str(self.rootdir / "main.cpp"), "defines": ["GPU"], "include_paths": [], "include_files": [], }, ], } - state = finder.find(self.rootdir, codebase, configuration) + state = finder.find(str(self.rootdir), codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( From c13618d07d86e34cbd8e54c99d79785426e85e96 Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Wed, 1 May 2024 07:57:05 -0700 Subject: [PATCH 4/8] Update documentation to reflect CodeBase behavior A side-effect of tracking all source files in the code base is that analysis can now pick up unexpected files that were previously never encountered in a compilation database. For example, any C++ files automatically generated by CMake will be identified as unused code if codebasin is invoked in a directory containing both build/ and src/ directories. This commit updates the documentation to highlight the importance of running codebasin in the right directory (or otherwise separating build and src). Signed-off-by: John Pennycook --- docs/source/analysis.rst | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/source/analysis.rst b/docs/source/analysis.rst index 4ee15fe..bdf5597 100644 --- a/docs/source/analysis.rst +++ b/docs/source/analysis.rst @@ -36,6 +36,13 @@ The table's name is the name of the platform, and we can use any meaningful string. The ``commands`` key tells CBI where to find the compilation database for this platform. +.. important:: + + By default, ``codebasin`` searches the current working directory for source + files to include in its analysis. Since we'll be running in the ``src`` + directory, we need to specify the ``commands`` paths relative to the + ``src`` directory or as absolute paths. + In our example, we have two platforms that we're calling "cpu" and "gpu", and our build directories are called ``build-cpu`` and ``build-gpu``, so our platform definitions should look like this: @@ -43,10 +50,10 @@ our platform definitions should look like this: .. code-block:: toml [platform.cpu] - commands = "build-cpu/compile_commands.json" + commands = "../build-cpu/compile_commands.json" [platform.gpu] - commands = "build-gpu/compile_commands.json" + commands = "../build-gpu/compile_commands.json" .. warning:: Platform names are case sensitive! The names "cpu" and "CPU" would refer to @@ -56,7 +63,8 @@ our platform definitions should look like this: Running ``codebasin`` ##################### -Running ``codebasin`` with this analysis file gives the following output: +Running ``codebasin`` in the ``src`` directory with this analysis file gives +the following output: .. code-block:: text :emphasize-lines: 4,5,6,7,9 @@ -86,6 +94,15 @@ used only by the GPU compilation, and 17 lines of code shared by both platforms. Plugging these numbers into the equation for code divergence gives 0.45. +.. caution:: + If we had run ``codebasin`` in the parent directory, everything in the + ``src``, ``build-cpu`` and ``build-gpu`` directories would have been + included in the analysis. For our sample code base, this would have + resulted in over 2000 lines of code being identified as unused! Why so + many? CMake generates multiple ``*.cpp`` files, which it uses as part of + the build process. ``codebasin`` will analyze such files unless we tell it + not to (more on that later). + Filtering Platforms ################### From b6ae8f114eafaf8eeeadb611272e4aadd2d88bdd Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Thu, 22 Aug 2024 15:35:47 +0100 Subject: [PATCH 5/8] Fix use of _ instead of - in docstrings Signed-off-by: John Pennycook --- codebasin/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebasin/__init__.py b/codebasin/__init__.py index d215d52..a3fa8dc 100644 --- a/codebasin/__init__.py +++ b/codebasin/__init__.py @@ -184,7 +184,7 @@ def exclude_patterns(self): def __contains__(self, path: os.PathLike) -> bool: """ Returns - _______ + ------- bool True if `path` is a recognized source file in one of the code base's listed directories and does not match any exclude From 55a62ce9fcf918464651bec9a64f0400fb888a5f Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Mon, 26 Aug 2024 08:04:00 +0100 Subject: [PATCH 6/8] Switch list to Iterable in type hints Signed-off-by: John Pennycook --- codebasin/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codebasin/__init__.py b/codebasin/__init__.py index a3fa8dc..ce0800a 100644 --- a/codebasin/__init__.py +++ b/codebasin/__init__.py @@ -3,6 +3,7 @@ import os import shlex import warnings +from collections.abc import Iterable from pathlib import Path import pathspec @@ -145,7 +146,7 @@ class CodeBase: def __init__( self, *directories: str | os.PathLike[str], - exclude_patterns: list[str] = [], + exclude_patterns: Iterable[str] = [], ): """ Raises From d08658412e5c32c39118da2f776ef929c47cf5fc Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Mon, 26 Aug 2024 08:07:13 +0100 Subject: [PATCH 7/8] Rewrite CodeBase test to not assume a sorted list rglob is not guaranteed to return a sorted list, and the output is OS-dependent. We may want to revisit this decision in future, but it will be easier to further constrain the behavior of the CodeBase iterator than to remove functionality that users rely on. Signed-off-by: John Pennycook --- tests/code-base/test_code_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/code-base/test_code_base.py b/tests/code-base/test_code_base.py index 72b0daf..8e1db81 100644 --- a/tests/code-base/test_code_base.py +++ b/tests/code-base/test_code_base.py @@ -99,7 +99,7 @@ def test_iterator(self): str(p1 / "foo.cpp"), str(p2 / "qux.cpp"), ] - self.assertEqual(files, expected) + self.assertCountEqual(files, expected) if __name__ == "__main__": From f7e6b024c986d9be65d68a52ceac3d517e7f992a Mon Sep 17 00:00:00 2001 From: John Pennycook Date: Mon, 26 Aug 2024 08:39:57 +0100 Subject: [PATCH 8/8] Move cast from Path to str into finder.find() finder.find() expects a string for compatibility with legacy interfaces. Performing the cast inside of the function is less error-prone, and will mean fewer updates later when all of the strings are replaced with Paths. Signed-off-by: John Pennycook --- codebasin/finder.py | 6 ++++++ tests/basic_asm/test_basic_asm.py | 4 ++-- tests/basic_fortran/test_basic_fortran.py | 2 +- tests/build-dir/test_build_dir.py | 2 +- tests/commented_directive/test_commented_directive.py | 2 +- tests/define/test_define.py | 2 +- tests/disjoint/test_disjoint.py | 2 +- tests/duplicates/test_duplicates.py | 2 +- tests/exclude/test_exclude.py | 2 +- tests/include/test_include.py | 2 +- tests/literals/test_literals.py | 2 +- tests/macro_expansion/test_macro_expansion.py | 2 +- tests/multi_line/test_multi_line.py | 2 +- tests/nesting/test_nesting.py | 2 +- tests/once/test_once.py | 2 +- tests/operators/test_operators.py | 2 +- 16 files changed, 22 insertions(+), 16 deletions(-) diff --git a/codebasin/finder.py b/codebasin/finder.py index 9db03af..819b6f2 100644 --- a/codebasin/finder.py +++ b/codebasin/finder.py @@ -8,6 +8,7 @@ import collections import logging import os +from pathlib import Path from codebasin import file_parser, platform, preprocessor, util from codebasin.language import FileLanguage @@ -140,6 +141,11 @@ def find( lines to platforms. """ + # Ensure rootdir is a string for compatibility with legacy code. + # TODO: Remove this once all other functionality is ported to Path. + if isinstance(rootdir, Path): + rootdir = str(rootdir) + # Build a tree for each unique file for all platforms. state = ParserState(summarize_only) for f in codebase: diff --git a/tests/basic_asm/test_basic_asm.py b/tests/basic_asm/test_basic_asm.py index 32265f6..952a186 100644 --- a/tests/basic_asm/test_basic_asm.py +++ b/tests/basic_asm/test_basic_asm.py @@ -34,7 +34,7 @@ def test_yaml(self): }, ) configuration = {"CPU": entries} - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( @@ -56,7 +56,7 @@ def test_ptx(self): self.assertRaises( RuntimeError, finder.find, - str(self.rootdir), + self.rootdir, codebase, configuration, ) diff --git a/tests/basic_fortran/test_basic_fortran.py b/tests/basic_fortran/test_basic_fortran.py index a9455d2..2995ad6 100644 --- a/tests/basic_fortran/test_basic_fortran.py +++ b/tests/basic_fortran/test_basic_fortran.py @@ -45,7 +45,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/build-dir/test_build_dir.py b/tests/build-dir/test_build_dir.py index ed08211..dc12307 100644 --- a/tests/build-dir/test_build_dir.py +++ b/tests/build-dir/test_build_dir.py @@ -63,7 +63,7 @@ def test_absolute_paths(self): expected_setmap = {frozenset(["one", "two"]): 1} - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap") diff --git a/tests/commented_directive/test_commented_directive.py b/tests/commented_directive/test_commented_directive.py index e4f3f42..42dcd54 100644 --- a/tests/commented_directive/test_commented_directive.py +++ b/tests/commented_directive/test_commented_directive.py @@ -49,7 +49,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) diff --git a/tests/define/test_define.py b/tests/define/test_define.py index 7814230..99e2169 100644 --- a/tests/define/test_define.py +++ b/tests/define/test_define.py @@ -44,7 +44,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/disjoint/test_disjoint.py b/tests/disjoint/test_disjoint.py index 1f29f58..beeb83a 100644 --- a/tests/disjoint/test_disjoint.py +++ b/tests/disjoint/test_disjoint.py @@ -43,7 +43,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/duplicates/test_duplicates.py b/tests/duplicates/test_duplicates.py index c5b2b36..20e7c06 100644 --- a/tests/duplicates/test_duplicates.py +++ b/tests/duplicates/test_duplicates.py @@ -47,7 +47,7 @@ def test_duplicates(self): expected_setmap = {frozenset(["cpu"]): 1, frozenset(["gpu"]): 1} - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap") diff --git a/tests/exclude/test_exclude.py b/tests/exclude/test_exclude.py index 82c3693..4d51234 100644 --- a/tests/exclude/test_exclude.py +++ b/tests/exclude/test_exclude.py @@ -27,7 +27,7 @@ def _get_setmap(self, excludes): configuration = { "test": config.load_database(str(dbpath), str(self.rootdir)), } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) return setmap diff --git a/tests/include/test_include.py b/tests/include/test_include.py index 4faa0b3..2b347d8 100644 --- a/tests/include/test_include.py +++ b/tests/include/test_include.py @@ -36,7 +36,7 @@ def test_include(self): "GPU": config.load_database(str(gpu_path), str(self.rootdir)), } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/literals/test_literals.py b/tests/literals/test_literals.py index 0f38d52..44651f2 100644 --- a/tests/literals/test_literals.py +++ b/tests/literals/test_literals.py @@ -42,7 +42,7 @@ def test_literals(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/macro_expansion/test_macro_expansion.py b/tests/macro_expansion/test_macro_expansion.py index 16ac787..2b61826 100644 --- a/tests/macro_expansion/test_macro_expansion.py +++ b/tests/macro_expansion/test_macro_expansion.py @@ -51,7 +51,7 @@ def test_macro_expansion(self): "CPU": cpu_entries, "GPU": gpu_entries, } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/multi_line/test_multi_line.py b/tests/multi_line/test_multi_line.py index 958f3f0..b658234 100644 --- a/tests/multi_line/test_multi_line.py +++ b/tests/multi_line/test_multi_line.py @@ -44,7 +44,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/nesting/test_nesting.py b/tests/nesting/test_nesting.py index 1684f04..8f51c6f 100644 --- a/tests/nesting/test_nesting.py +++ b/tests/nesting/test_nesting.py @@ -45,7 +45,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/once/test_once.py b/tests/once/test_once.py index e380ac5..ae44d01 100644 --- a/tests/once/test_once.py +++ b/tests/once/test_once.py @@ -44,7 +44,7 @@ def test_yaml(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual( diff --git a/tests/operators/test_operators.py b/tests/operators/test_operators.py index d0c4256..c043f73 100644 --- a/tests/operators/test_operators.py +++ b/tests/operators/test_operators.py @@ -42,7 +42,7 @@ def test_operators(self): }, ], } - state = finder.find(str(self.rootdir), codebase, configuration) + state = finder.find(self.rootdir, codebase, configuration) mapper = PlatformMapper(codebase) setmap = mapper.walk(state) self.assertDictEqual(