From 5f525a52c74e658c5a38f5c4f3274288e1b60c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Thu, 30 Nov 2023 11:57:20 +0100 Subject: [PATCH] fix: count all files present, not only recipe files --- mpl_data_cast/cli/cli.py | 7 ++-- mpl_data_cast/gui/main.py | 6 ++- mpl_data_cast/gui/widget_tree.py | 67 +++++++++++++++----------------- mpl_data_cast/recipe.py | 6 +-- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/mpl_data_cast/cli/cli.py b/mpl_data_cast/cli/cli.py index c5a2d16..83aaa42 100644 --- a/mpl_data_cast/cli/cli.py +++ b/mpl_data_cast/cli/cli.py @@ -1,6 +1,7 @@ import inspect import pathlib import time +from typing import List import click @@ -105,14 +106,14 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None: self.print(f"Processed {self.counter} files (~{rate:.1f} MB/s).") print("") - def __call__(self, path: pathlib.Path) -> None: + def __call__(self, path_list: List[pathlib.Path]) -> None: self.counter += 1 - name = click.format_filename(path, shorten=True) + name = click.format_filename(path_list[0], shorten=True) message = f"Processing file {self.counter}: {name}" rate = self.get_rate() if rate: message += f" ({rate:.1f}MB/s)" - self.size += path.stat().st_size + self.size += sum([pp.stat().st_size for pp in path_list]) self.print(message) def get_rate(self) -> float: diff --git a/mpl_data_cast/gui/main.py b/mpl_data_cast/gui/main.py index 447ab50..e4ab957 100644 --- a/mpl_data_cast/gui/main.py +++ b/mpl_data_cast/gui/main.py @@ -185,6 +185,7 @@ def on_task_transfer(self) -> None: for path, tb in result["errors"]: text += f"PATH {path}:\n{tb}\n\n" pathlib.Path("mpldc-dump.txt").write_text(text) + self.label_file.setText("") self.pushButton_transfer.setEnabled(True) @@ -206,7 +207,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): pass - def __call__(self, path) -> None: + def __call__(self, path_list) -> None: + path = path_list[0] # Let the user know where we are self.gui.label_file.setText(f"Processing {path}...") @@ -219,7 +221,7 @@ def __call__(self, path) -> None: # go to undetermined state self.gui.progressBar.setRange(0, 0) - self.counter += 1 + self.counter += len(path_list) class CastingThread(threading.Thread): diff --git a/mpl_data_cast/gui/widget_tree.py b/mpl_data_cast/gui/widget_tree.py index 19705dd..cf7bf8b 100644 --- a/mpl_data_cast/gui/widget_tree.py +++ b/mpl_data_cast/gui/widget_tree.py @@ -6,7 +6,7 @@ from PyQt6 import QtWidgets, QtCore, QtGui, uic -from ..recipe import map_recipe_name_to_class +from ..recipe import IGNORED_FILE_NAMES, map_recipe_name_to_class from ..util import is_dir_writable @@ -61,40 +61,37 @@ def run(self): # start crawling the directory tree self.is_counting = True self.has_counted = False - try: - rcp = recipe(path, path) - except BaseException: - pass - else: - tree_iterator = rcp.get_raw_data_iterator() - while True: - # check whether we have to abort - if (self.must_break - or recipe != self.recipe or path != self.path): - self.num_objects = 0 - self.size_objects = 0 - break - try: - item = next(tree_iterator) - except StopIteration: - self.has_counted = True - break - except BaseException: - # Windows might encounter PermissionError. - pass - else: - with self.lock: - # check before incrementing - if self.abort_current_count: - self.abort_current_count = False - break - self.num_objects += 1 - try: - self.size_objects += sum( - [it.stat().st_size for it in item]) - except BaseException: - pass - self.is_counting = False + ignored_files = IGNORED_FILE_NAMES + recipe.ignored_file_names + tree_iterator = path.rglob("*") + while True: + # check whether we have to abort + if (self.must_break + or recipe != self.recipe or path != self.path): + self.num_objects = 0 + self.size_objects = 0 + break + try: + pp = next(tree_iterator) + except StopIteration: + self.has_counted = True + break + except BaseException: + # Windows might encounter PermissionError. + pass + else: + if pp.is_dir() or pp.name in ignored_files: + continue + with self.lock: + # check before incrementing + if self.abort_current_count: + self.abort_current_count = False + break + self.num_objects += 1 + try: + self.size_objects += pp.stat().st_size + except BaseException: + pass + self.is_counting = False time.sleep(0.5) diff --git a/mpl_data_cast/recipe.py b/mpl_data_cast/recipe.py index 9a93652..1db6489 100644 --- a/mpl_data_cast/recipe.py +++ b/mpl_data_cast/recipe.py @@ -54,7 +54,7 @@ def cast(self, path_callback: Callable = None, **kwargs) -> dict: Parameters ---------- path_callback: Callable - Callable function accepting a path; used for tracking + Callable function accepting a list of paths; used for tracking the progress (e.g. via the CLI) Returns @@ -72,7 +72,7 @@ def cast(self, path_callback: Callable = None, **kwargs) -> dict: for path_list in ds_iterator: known_files += path_list if path_callback is not None: - path_callback(path_list[0]) + path_callback(path_list) targ_path = self.get_target_path(path_list) temp_path = self.get_temp_path(path_list) try: @@ -98,7 +98,7 @@ def cast(self, path_callback: Callable = None, **kwargs) -> dict: continue else: if path_callback is not None: - path_callback(pp) + path_callback([pp]) prel = pp.relative_to(self.path_raw) target_path = self.path_tar / prel target_path.parent.mkdir(parents=True, exist_ok=True)