Applied updates and worked on schema extraction script

libyal · Jan 7, 2024 · 46e528f · 46e528f
1 parent dd7de55
commit 46e528f
Show file tree

Hide file tree

Showing 11 changed files with 87 additions and 26 deletions.
diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml
@@ -18,7 +18,7 @@ jobs:
     - name: Install dependencies
       run: |
         dnf copr -y enable @gift/dev
-        dnf install -y @development-tools python3 python3-devel libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-dtfabric python3-idna python3-pytsk3 python3-pyxattr python3-pyyaml python3-setuptools
+        dnf install -y @development-tools python3 python3-devel libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml python3-setuptools python3-xattr
     - name: Run tests
       env:
         LANG: C.utf8
@@ -58,7 +58,7 @@ jobs:
       run: |
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential python3 python3-dev libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-wheel python3-yaml
+        apt-get install -y build-essential python3 python3-dev libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-wheel python3-xattr python3-yaml
     - name: Run tests
       env:
         LANG: en_US.UTF-8

diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml
@@ -36,7 +36,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox

diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml
@@ -46,7 +46,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox
@@ -82,7 +82,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox
@@ -120,7 +120,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
+        apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libcreg-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-xattr python3-yaml
     - name: Install tox
       run: |
         python3 -m pip install tox

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,15 @@
+include LICENSE README
+include dependencies.ini run_tests.py utils/__init__.py utils/dependencies.py
+include utils/check_dependencies.py
+include requirements.txt test_requirements.txt
+recursive-include config *
+recursive-include test_data *
+exclude .gitignore
+exclude *.pyc
+recursive-exclude sqliterc *.pyc
+recursive-include scripts *.py
+recursive-exclude scripts *.pyc
+# The test scripts are not required in a binary distribution package they
+# are considered source distribution files and excluded in find_package()
+# in setup.py.
+recursive-include tests *.py
diff --git a/config/dpkg/control b/config/dpkg/control
@@ -16,7 +16,7 @@ Description: Data files for SQLite database knowledge base
 
 Package: python3-sqliterc
 Architecture: all
-Depends: sqlite-kb-data (>= ${binary:Version}), libbde-python3 (>= 20220121), libcreg-python3 (>= 20200725), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20220709), libfsext-python3 (>= 20220829), libfsfat-python3 (>= 20220925), libfshfs-python3 (>= 20220831), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20220829), libfvde-python3 (>= 20220121), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20220121), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220228), libqcow-python3 (>= 20201213), libregf-python3 (>= 20201002), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsapm-python3 (>= 20230506), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20220219), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20221112), python3-dfimagetools (>= 20230806), python3-dfvfs (>= 20220831), python3-dfwinreg (>= 20211207), python3-dtfabric (>= 20220219), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
+Depends: sqlite-kb-data (>= ${binary:Version}), libbde-python3 (>= 20220121), libcreg-python3 (>= 20200725), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20220709), libfsext-python3 (>= 20220829), libfsfat-python3 (>= 20220925), libfshfs-python3 (>= 20220831), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20220829), libfvde-python3 (>= 20220121), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20220121), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220228), libqcow-python3 (>= 20201213), libregf-python3 (>= 20201002), libsigscan-python3 (>= 20230109), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsapm-python3 (>= 20230506), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20220219), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20221112), python3-dfimagetools (>= 20220129), python3-dfvfs (>= 20220831), python3-dfwinreg (>= 20211207), python3-dtfabric (>= 20230518), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-xattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
 Description: Python 3 module of SQLite database knowledge base resources (sqliterc)
  sqliterc is a Python module part of sqlite-kb to allow reuse of
  SQLite database resources.

diff --git a/dependencies.ini b/dependencies.ini
@@ -25,7 +25,7 @@ version_property: __version__
 
 [dfimagetools]
 dpkg_name: python3-dfimagetools
-minimum_version: 20230806
+minimum_version: 20220129
 rpm_name: python3-dfimagetools
 version_property: __version__
 
@@ -43,7 +43,7 @@ version_property: __version__
 
 [dtfabric]
 dpkg_name: python3-dtfabric
-minimum_version: 20220219
+minimum_version: 20230518
 rpm_name: python3-dtfabric
 version_property: __version__
 
@@ -186,7 +186,7 @@ version_property: get_version()
 [pysigscan]
 dpkg_name: libsigscan-python3
 l2tbinaries_name: libsigscan
-minimum_version: 20191221
+minimum_version: 20230109
 pypi_name: libsigscan-python
 rpm_name: libsigscan-python3
 version_property: get_version()
@@ -262,11 +262,11 @@ rpm_name: libvslvm-python3
 version_property: get_version()
 
 [xattr]
-dpkg_name: python3-pyxattr
+dpkg_name: python3-xattr
 is_optional: true
 minimum_version: 0.7.2
-pypi_name: pyxattr
-rpm_name: python3-pyxattr
+pypi_name: xattr
+rpm_name: python3-xattr
 version_property: __version__
 
 [yaml]

diff --git a/requirements.txt b/requirements.txt
@@ -3,10 +3,10 @@ artifacts >= 20220219
 cffi >= 1.9.1
 cryptography >= 2.0.2
 dfdatetime >= 20221112
-dfimagetools >= 20230806
+dfimagetools >= 20220129
 dfvfs >= 20220831
 dfwinreg >= 20211207
-dtfabric >= 20220219
+dtfabric >= 20230518
 libbde-python >= 20220121
 libcreg-python >= 20200725
 libewf-python >= 20131210
@@ -23,7 +23,7 @@ libmodi-python >= 20210405
 libphdi-python >= 20220228
 libqcow-python >= 20201213
 libregf-python >= 20201002
-libsigscan-python >= 20191221
+libsigscan-python >= 20230109
 libsmdev-python >= 20140529
 libsmraw-python >= 20140612
 libvhdi-python >= 20201014
@@ -33,4 +33,4 @@ libvsgpt-python >= 20211115
 libvshadow-python >= 20160109
 libvslvm-python >= 20160109
 pytsk3 >= 20210419
-pyxattr >= 0.7.2 ; platform_system != "Windows"
+xattr >= 0.7.2 ; platform_system != "Windows"
diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = sqliterc
-version = 20231228
+version = 20240107
 description = SQLite database resources (sqliterc)
 long_description = sqliterc is a Python module part of sqlite-kb to allow reuse of SQLite database resources.
 long_description_content_type = text/plain
@@ -26,6 +26,10 @@ python_requires = >=3.7
 scripts =
   scripts/extract.py
 
+[options.package_data]
+sqliterc.data =
+  data/*.yaml
+
 [options.packages.find]
 exclude =
   docs
@@ -58,7 +62,7 @@ requires =
   libphdi-python3 >= 20220228
   libqcow-python3 >= 20201213
   libregf-python3 >= 20201002
-  libsigscan-python3 >= 20191221
+  libsigscan-python3 >= 20230109
   libsmdev-python3 >= 20140529
   libsmraw-python3 >= 20140612
   libvhdi-python3 >= 20201014
@@ -71,14 +75,14 @@ requires =
   python3-cffi >= 1.9.1
   python3-cryptography >= 2.0.2
   python3-dfdatetime >= 20221112
-  python3-dfimagetools >= 20230806
+  python3-dfimagetools >= 20220129
   python3-dfvfs >= 20220831
   python3-dfwinreg >= 20211207
-  python3-dtfabric >= 20220219
+  python3-dtfabric >= 20230518
   python3-idna >= 2.5
   python3-pytsk3 >= 20210419
-  python3-pyxattr >= 0.7.2
   python3-pyyaml >= 3.10
+  python3-xattr >= 0.7.2
 
 [bdist_wheel]
 universal = 1
diff --git a/data/known_databases.yaml → sqliterc/data/known_databases.yaml b/data/known_databases.yaml → sqliterc/data/known_databases.yaml
@@ -1,5 +1,20 @@
 # sqlite-kb database definitions
 ---
+artifact_definition: ChromiumBasedBrowsersCookiesDatabaseFile
+database_identifier: chromium_Cookies
+---
+artifact_definition: ChromiumBasedBrowsersFaviconsDatabaseFile
+database_identifier: chromium_Favicons
+---
+artifact_definition: ChromiumBasedBrowsersHistoryDatabaseFile
+database_identifier: chromium_History
+---
+artifact_definition: ChromiumBasedBrowsersLoginDataDatabaseFile
+database_identifier: chromium_Login_Data
+---
+artifact_definition: ChromiumBasedBrowsersWebDataDatabaseFile
+database_identifier: chromium_Web_Data
+---
 artifact_definition: MacOSAddressBookImagesSQLiteDatabaseFile
 database_identifier: macos_AddressBookImages.sqlitedb
 ---
@@ -128,3 +143,18 @@ database_identifier: safari_TouchIconCacheSettings.db
 ---
 artifact_definition: WebKitPubSubSQLiteDatabaseFile
 database_identifier: webkit_Database.sqlite3
+---
+artifact_definition: WindowsActivitiesCacheDatabase
+database_identifier: windows_ActivitiesCache.db
+---
+artifact_definition: WindowsPushNotificationDatabaseFile
+database_identifier: windows_wpndatabase.db
+---
+artifact_definition: WindowsStateRepositoryDeploymentDatabaseFile
+database_identifier: windows_StateRepository-Deployment.srd
+---
+artifact_definition: WindowsStateRepositoryMachineDatabaseFile
+database_identifier: windows_StateRepository-Machine.srd
+---
+artifact_definition: WindowsUpdateStoreDatabaseFile
+database_identifier: windows_UpdateStore_store.db