fix disk-usage, allow override for skipping deleted files by default

.github/workflows/push.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.13"
 
       - uses: FedericoCarboni/setup-ffmpeg@v3
 
@@ -75,7 +75,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - run: |
           python -m pip install pdm

pyproject.toml

@@ -19,6 +19,8 @@ dependencies = [
   "python-dateutil",
   "python-mpv-jsonipc",
   "regex",
+  "requests",
+  "urllib3>2",
   "rich",
   "screeninfo",
   "sqlite-utils",
@@ -35,7 +37,7 @@ requires-python = ">=3.11"
 
 [project.optional-dependencies]
 dev = ["black", "isort", "ssort"]
-test = ["ruff", "pytest", "pytest-regressions", "freezegun", "pyfakefs"]
+test = ["ruff", "pytest", "pytest-regressions", "freezegun", "pandas", "pyfakefs"]
 deluxe = [
   "aiohttp",
   "annoy",

xklb/createdb/web_add.py

@@ -269,7 +269,11 @@ def web_add(args=None) -> None:
             if db_media.exists(args, p):
                 media_known.add(p)
             else:
-                add_media(args, [consolidate_media(args, p)])
+                if isinstance(p, dict):
+                    m = consolidate_media(args, p['path']) | p
+                else:
+                    m = consolidate_media(args, p)
+                add_media(args, [m])
                 media_new.add(p)
             printing.print_overwrite(f"Link import: {len(media_new)} new [{len(media_known)} known]")
     else:

xklb/fsdb/disk_usage.py

@@ -100,7 +100,8 @@ def get_data(args) -> list[dict]:
     if args.database:
         media = list(args.db.query(*sqlgroups.fs_sql(args, limit=None)))
     else:
-        args.paths = [p for p in args.paths if os.path.exists(p)]
+        if args.hide_deleted:
+            args.paths = [p for p in args.paths if os.path.exists(p)]
         media = arg_utils.gen_d(args)
         media = [d if "size" in d else file_utils.get_filesize(d) for d in media]
 

xklb/utils/arg_utils.py

@@ -1,10 +1,11 @@
-import argparse, operator, random
+import argparse, operator, os.path, random
 from collections import defaultdict
 from copy import copy
 from pathlib import Path
 
 from xklb.utils import consts, file_utils, iterables, nums, processes, strings
 from xklb.utils.consts import SC
+from xklb.utils.log_utils import log
 
 
 def gen_paths(args, default_exts=None):
@@ -26,6 +27,11 @@ def gen_paths(args, default_exts=None):
                 p = Path(path)
                 if p.is_dir():
                     yield from file_utils.rglob(str(p), args.ext or default_exts, getattr(args, "exclude", None))[0]
+                elif getattr(args, "hide_deleted", False):
+                    if os.path.exists(p):
+                        yield {"path": path}
+                    else:
+                        log.info("Skipping non-existent file %s", path)
                 else:
                     yield path
 
@@ -38,9 +44,18 @@ def gen_d(args, default_exts=None):
         for path in args.paths:
             json_data = strings.safe_json_loads(path)
             if isinstance(json_data, list):
-                yield from json_data
+                for json_item in json_data:
+                    if getattr(args, "hide_deleted", False):
+                        if os.path.exists(json_item["path"]):
+                            yield json_item
+                    else:
+                        yield json_item
             elif isinstance(json_data, dict):
-                yield json_data
+                if getattr(args, "hide_deleted", False):
+                    if os.path.exists(json_data["path"]):
+                        yield json_data
+                else:
+                    yield json_data
             else:
                 raise TypeError
     else:
@@ -50,6 +65,11 @@ def gen_d(args, default_exts=None):
                 if p.is_dir():
                     for sp in file_utils.rglob(str(p), args.ext or default_exts, getattr(args, "exclude", None))[0]:
                         yield {"path": sp}
+                elif getattr(args, "hide_deleted", False):
+                    if os.path.exists(p):
+                        yield {"path": path}
+                    else:
+                        log.info("Skipping non-existent file %s", path)
                 else:
                     yield {"path": path}