Moved data files into Python module ForensicArtifacts#594 (ForensicAr…

…tifacts#592)
joachimmetz · Jan 7, 2024 · 86064d7 · 86064d7
1 parent 142c986
commit 86064d7
Show file tree

Hide file tree

Showing 45 changed files with 27 additions and 29 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,13 +3,12 @@ include dependencies.ini run_tests.py utils/__init__.py utils/dependencies.py
 include utils/check_dependencies.py
 include requirements.txt test_requirements.txt
 recursive-include config *
-recursive-include data *
 recursive-include test_data *
 exclude .gitignore
 exclude *.pyc
+recursive-exclude artifacts *.pyc
 recursive-include tools *.py
 recursive-exclude tools *.pyc
-recursive-exclude artifacts *.pyc
 # The test scripts are not required in a binary distribution package they
 # are considered source distribution files and excluded in find_package()
 # in setup.py.

diff --git a/artifacts/__init__.py b/artifacts/__init__.py
@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 """ForensicArtifacts.com Artifact Repository."""
 
-__version__ = '20230928'
+__version__ = '20240107'
diff --git a/data/antivirus.yaml → artifacts/data/antivirus.yaml b/data/antivirus.yaml → artifacts/data/antivirus.yaml
diff --git a/data/applications.yaml → artifacts/data/applications.yaml b/data/applications.yaml → artifacts/data/applications.yaml
diff --git a/data/cloud_services.yaml → artifacts/data/cloud_services.yaml b/data/cloud_services.yaml → artifacts/data/cloud_services.yaml
diff --git a/data/config_files.yaml → artifacts/data/config_files.yaml b/data/config_files.yaml → artifacts/data/config_files.yaml
diff --git a/data/containerd.yaml → artifacts/data/containerd.yaml b/data/containerd.yaml → artifacts/data/containerd.yaml
diff --git a/data/database_servers.yaml → artifacts/data/database_servers.yaml b/data/database_servers.yaml → artifacts/data/database_servers.yaml
diff --git a/data/docker.yaml → artifacts/data/docker.yaml b/data/docker.yaml → artifacts/data/docker.yaml
diff --git a/data/esxi.yaml → artifacts/data/esxi.yaml b/data/esxi.yaml → artifacts/data/esxi.yaml
diff --git a/data/file_systems.yaml → artifacts/data/file_systems.yaml b/data/file_systems.yaml → artifacts/data/file_systems.yaml
diff --git a/data/hadoop.yaml → artifacts/data/hadoop.yaml b/data/hadoop.yaml → artifacts/data/hadoop.yaml
diff --git a/data/ics.yaml → artifacts/data/ics.yaml b/data/ics.yaml → artifacts/data/ics.yaml
diff --git a/data/installed_module_paths.yaml → artifacts/data/installed_module_paths.yaml b/data/installed_module_paths.yaml → artifacts/data/installed_module_paths.yaml
diff --git a/data/installed_modules.yaml → artifacts/data/installed_modules.yaml b/data/installed_modules.yaml → artifacts/data/installed_modules.yaml
diff --git a/data/instant_messaging.yaml → artifacts/data/instant_messaging.yaml b/data/instant_messaging.yaml → artifacts/data/instant_messaging.yaml
diff --git a/data/java.yaml → artifacts/data/java.yaml b/data/java.yaml → artifacts/data/java.yaml
diff --git a/data/kaspersky_careto.yaml → artifacts/data/kaspersky_careto.yaml b/data/kaspersky_careto.yaml → artifacts/data/kaspersky_careto.yaml
diff --git a/data/kubernetes.yaml → artifacts/data/kubernetes.yaml b/data/kubernetes.yaml → artifacts/data/kubernetes.yaml
diff --git a/data/legacy.yaml → artifacts/data/legacy.yaml b/data/legacy.yaml → artifacts/data/legacy.yaml
diff --git a/data/linux.yaml → artifacts/data/linux.yaml b/data/linux.yaml → artifacts/data/linux.yaml
diff --git a/data/linux_proc.yaml → artifacts/data/linux_proc.yaml b/data/linux_proc.yaml → artifacts/data/linux_proc.yaml
diff --git a/data/linux_services.yaml → artifacts/data/linux_services.yaml b/data/linux_services.yaml → artifacts/data/linux_services.yaml
diff --git a/data/macos.yaml → artifacts/data/macos.yaml b/data/macos.yaml → artifacts/data/macos.yaml
diff --git a/data/shell.yaml → artifacts/data/shell.yaml b/data/shell.yaml → artifacts/data/shell.yaml
diff --git a/data/tomcat.yaml → artifacts/data/tomcat.yaml b/data/tomcat.yaml → artifacts/data/tomcat.yaml
diff --git a/data/triage.yaml → artifacts/data/triage.yaml b/data/triage.yaml → artifacts/data/triage.yaml
diff --git a/data/unix_common.yaml → artifacts/data/unix_common.yaml b/data/unix_common.yaml → artifacts/data/unix_common.yaml
diff --git a/data/user.yaml → artifacts/data/user.yaml b/data/user.yaml → artifacts/data/user.yaml
diff --git a/data/webbrowser.yaml → artifacts/data/webbrowser.yaml b/data/webbrowser.yaml → artifacts/data/webbrowser.yaml
diff --git a/data/webservers.yaml → artifacts/data/webservers.yaml b/data/webservers.yaml → artifacts/data/webservers.yaml
diff --git a/data/windows.yaml → artifacts/data/windows.yaml b/data/windows.yaml → artifacts/data/windows.yaml
diff --git a/data/windows_dll_hijacking.yaml → artifacts/data/windows_dll_hijacking.yaml b/data/windows_dll_hijacking.yaml → artifacts/data/windows_dll_hijacking.yaml
diff --git a/data/wmi.yaml → artifacts/data/wmi.yaml b/data/wmi.yaml → artifacts/data/wmi.yaml
diff --git a/config/dpkg/changelog b/config/dpkg/changelog
@@ -1,5 +1,5 @@
-artifacts (20230928-1) unstable; urgency=low
+artifacts (20240107-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Forensic artifacts <[email protected]>  Thu, 28 Sep 2023 13:23:32 +0200
+ -- Forensic artifacts <[email protected]>  Sun, 07 Jan 2024 10:05:41 +0100
diff --git a/config/dpkg/control b/config/dpkg/control
@@ -7,16 +7,9 @@ Standards-Version: 4.1.4
 X-Python3-Version: >= 3.6
 Homepage: https://github.com/ForensicArtifacts/artifacts
 
-Package: artifacts-data
-Architecture: all
-Depends: ${misc:Depends}
-Description: Data files for Digital Forensics Artifacts Repository
- A free, community-sourced, machine-readable knowledge base of forensic
- artifacts that the world can use both as an information source and within other tools.
-
 Package: python3-artifacts
 Architecture: all
-Depends: artifacts-data (>= ${binary:Version}), python3-yaml (>= 3.10), ${misc:Depends}
+Depends: python3-yaml (>= 3.10), ${misc:Depends}
 Description: Python 3 module of Digital Forensics Artifacts Repository
  A free, community-sourced, machine-readable knowledge base of forensic
  artifacts that the world can use both as an information source and within other tools.

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,3 +1,4 @@
+certifi >= 2023.11.17
 docutils
 Markdown
 recommonmark

diff --git a/docs/sources/background/Stats.md b/docs/sources/background/Stats.md
@@ -4,12 +4,12 @@ The artifact definitions can be found in the
 [data directory](https://github.com/ForensicArtifacts/artifacts/tree/main/data) and the format is described in detail
 in the [Style Guide](https://artifacts.readthedocs.io/en/latest/sources/Format-specification.html).
 
-Status of the repository as of 2023-09-28
+Status of the repository as of 2024-01-07
 
 Description | Number
 --- | ---
-Number of artifact definitions: | 801
-Number of file paths: | 2047
+Number of artifact definitions: | 808
+Number of file paths: | 2058
 Number of Windows Registry key paths: | 677
 
 ### Artifact definition source types
@@ -18,7 +18,7 @@ Identifier | Number
 --- | ---
 ARTIFACT_GROUP | 47
 COMMAND | 10
-FILE | 516
+FILE | 523
 PATH | 28
 REGISTRY_KEY | 57
 REGISTRY_VALUE | 116
@@ -28,8 +28,8 @@ WMI | 27
 
 Identifier | Number
 --- | ---
-Darwin | 200
+Darwin | 202
 ESXi | 16
-Linux | 243
-Windows | 367
+Linux | 246
+Windows | 368
 
diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = artifacts
-version = 20231223
+version = 20240107
 description = ForensicArtifacts.com Artifact Repository.
 long_description = A free, community-sourced, machine-readable knowledge base of forensic artifacts that the world can use both as an information source and within other tools.
 long_description_content_type = text/plain
@@ -27,6 +27,10 @@ scripts =
   tools/stats.py
   tools/validator.py
 
+[options.package_data]
+artifacts.data =
+  data/*.yaml
+
 [options.packages.find]
 exclude =
   docs

diff --git a/tests/reader_test.py b/tests/reader_test.py
@@ -318,10 +318,8 @@ def testDefinitionsAsDict(self):
     """Tests the AsDict function."""
     artifact_reader = reader.YamlArtifactsReader()
 
-    artifact_definitions = list(artifact_reader.ReadDirectory('data'))
-
     last_artifact_definition = None
-    for artifact in artifact_definitions:
+    for artifact in artifact_reader.ReadDirectory(self._DATA_PATH):
       try:
         artifact_definition = artifact.AsDict()
       except errors.FormatError:

diff --git a/tests/test_lib.py b/tests/test_lib.py
@@ -10,7 +10,7 @@
 class BaseTestCase(unittest.TestCase):
   """The base test case."""
 
-  _DATA_PATH = os.path.join(os.getcwd(), 'data')
+  _DATA_PATH = os.path.join(os.getcwd(), 'artifacts', 'data')
   _TEST_DATA_PATH = os.path.join(os.getcwd(), 'test_data')
 
   # Show full diff results, part of TestCase so does not follow our naming

diff --git a/tests/validator_test.py b/tests/validator_test.py
@@ -19,7 +19,8 @@ def testArtifactDefinitionsValidator(self):
     """Runs the validator over all the YAML artifact definitions files."""
     validator_object = validator.ArtifactDefinitionsValidator()
 
-    for definitions_file in glob.glob(os.path.join('data', '*.yaml')):
+    data_files_glob = os.path.join(self._DATA_PATH, '*.yaml')
+    for definitions_file in glob.glob(data_files_glob):
       result = validator_object.CheckFile(definitions_file)
       self.assertTrue(
           result, msg=f'in definitions file: {definitions_file:s}')

diff --git a/tools/stats.py b/tools/stats.py
@@ -3,6 +3,7 @@
 """Report statistics about the artifact collection."""
 
 import collections
+import os
 import sys
 import time
 
@@ -70,7 +71,8 @@ def BuildStats(self):
     self._source_type_counts = {}
     self._total_count = 0
 
-    for artifact_definition in artifact_reader.ReadDirectory('data'):
+    data_files_path = os.path.join('artifacts', 'data')
+    for artifact_definition in artifact_reader.ReadDirectory(data_files_path):
       sources_supported_os = set()
       for source in artifact_definition.sources:
         self._total_count += 1

diff --git a/tools/validator.py b/tools/validator.py
@@ -17,7 +17,7 @@
 class ArtifactDefinitionsValidator(object):
   """Artifact definitions validator."""
 
-  LEGACY_PATH = os.path.join('data', 'legacy.yaml')
+  LEGACY_PATH = os.path.join('artifacts', 'data', 'legacy.yaml')
 
   _MACOS_PRIVATE_SUB_PATHS = ('etc', 'tftpboot', 'tmp', 'var')
 

diff --git a/tox.ini b/tox.ini
@@ -51,4 +51,4 @@ commands =
   pylint --version
   yamllint -v
   pylint --rcfile=.pylintrc artifacts setup.py tests tools
-  yamllint -c .yamllint.yaml data test_data
+  yamllint -c .yamllint.yaml artifacts/data test_data