Enforce exact_match=True when listing JSON file for get_estimated_time for MPH

.github/workflows/test.yml

@@ -52,6 +52,9 @@ jobs:
         run: |
           poetry install --no-interaction
 
+      # - name: Debug
+      #   uses: lhotari/action-upterm@v1
+
       - name: Run Tests
         run: |
           poetry install --with test --no-interaction --no-root

pyproject.toml

@@ -44,7 +44,7 @@ numpy = "1.*"
 natsort = "^8"
 seaborn = "^0.13"
 scikit-learn = "^1"
-watchdog = "^3"
+watchdog = "^6"
 tqdm = "^4"
 scipy = "^1.10.1"
 pandas = "^2"
@@ -128,3 +128,10 @@ filterwarnings = [
 ]
 testpaths = ["tests"]
 norecursedirs = ["tests/utilities"]
+
+log_cli = true
+log_level = "INFO"
+log_file = "pytest.txt"
+log_file_level = "INFO"
+log_format = "%(asctime)s %(levelname)s %(message)s"
+log_date_format = "%Y-%m-%d %H:%M:%S"

src/toffy/fov_watcher.py

@@ -207,21 +207,23 @@ def __init__(
         self.run_folder = run_folder
 
         self.last_event_time = datetime.now()
-        self.timer_thread = threading.Thread(
-            target=self.file_timer, args=(fov_timeout, watcher_timeout)
-        )
-        self.timer_thread.daemon = True
-        self.timer_thread.start()
+        # self.timer_thread = threading.Thread(
+        #     target=self.file_timer, args=(fov_timeout, watcher_timeout)
+        # )
+        # self.timer_thread.daemon = True
+        # self.timer_thread.start()
 
         self.log_path = os.path.join(log_folder, f"{Path(run_folder).parts[-1]}_log.txt")
         if not os.path.exists(log_folder):
             os.makedirs(log_folder)
+        print(f"Setting log path to {self.log_path}")
         logging.basicConfig(
             level=logging.INFO,
             filename=self.log_path,
             filemode="a",
             format="%(name)s - %(levelname)s - %(message)s",
         )
+        logging.info(f"Starting run on {run_folder}\n")
 
         # create run structure
         self.run_structure = RunStructure(run_folder, fov_timeout=fov_timeout)
@@ -487,45 +489,45 @@ def on_created(self, event: FileCreatedEvent, check_last_fov: bool = True):
         if self.last_fov_num_processed == self.run_structure.highest_fov:
             return
 
-        with self.lock:
-            super().on_created(event)
-            self._run_callbacks(event, check_last_fov)
-
-    def file_timer(self, fov_timeout, watcher_timeout):
-        """Checks time since last file was generated.
-
-        Args:
-            fov_timeout (int):
-                how long to wait for fov data to be generated once file detected
-            watcher_timeout (int):
-                length to wait for new file generation before timing out
-        """
-        while True:
-            current_time = datetime.now()
-            time_elapsed = (current_time - self.last_event_time).total_seconds()
-
-            # 3 fov cycles and no new files --> timeout
-            if time_elapsed > watcher_timeout:
-                fov_num = self.last_fov_num_processed
-                fov_name = list(self.run_structure.fov_progress.keys())[fov_num]
-                print(f"Timed out waiting for {fov_name} files to be generated.")
-                logging.info(
-                    f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- Timed out'
-                    f"waiting for {fov_name} files to be generated.\n"
-                )
-                logging.info(
-                    f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- '
-                    f"Running {self.run_func.__name__} on FOVs\n"
-                )
-
-                # mark remaining fovs as completed to exit watcher
-                for fov_name in list(self.run_structure.fov_progress.keys()):
-                    self.run_structure.fov_progress[fov_name] = {"json": True, "bin": True}
-
-                # trigger run callbacks
-                self.run_func(self.run_folder)
-                break
-            time.sleep(fov_timeout)
+        # with self.lock:
+        super().on_created(event)
+        self._run_callbacks(event, check_last_fov)
+
+    # def file_timer(self, fov_timeout, watcher_timeout):
+    #     """Checks time since last file was generated.
+
+    #     Args:
+    #         fov_timeout (int):
+    #             how long to wait for fov data to be generated once file detected
+    #         watcher_timeout (int):
+    #             length to wait for new file generation before timing out
+    #     """
+    #     while True:
+    #         current_time = datetime.now()
+    #         time_elapsed = (current_time - self.last_event_time).total_seconds()
+
+    #         # 3 fov cycles and no new files --> timeout
+    #         if time_elapsed > watcher_timeout:
+    #             fov_num = self.last_fov_num_processed
+    #             fov_name = list(self.run_structure.fov_progress.keys())[fov_num]
+    #             print(f"Timed out waiting for {fov_name} files to be generated.")
+    #             logging.info(
+    #                 f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- Timed out'
+    #                 f"waiting for {fov_name} files to be generated.\n"
+    #             )
+    #             logging.info(
+    #                 f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- '
+    #                 f"Running {self.run_func.__name__} on FOVs\n"
+    #             )
+
+    #             # mark remaining fovs as completed to exit watcher
+    #             for fov_name in list(self.run_structure.fov_progress.keys()):
+    #                 self.run_structure.fov_progress[fov_name] = {"json": True, "bin": True}
+
+    #             # trigger run callbacks
+    #             self.run_func(self.run_folder)
+    #             break
+    #         time.sleep(fov_timeout)
 
     def on_moved(self, event: FileMovedEvent, check_last_fov: bool = True):
         """Handles file renaming events.
@@ -541,9 +543,9 @@ def on_moved(self, event: FileMovedEvent, check_last_fov: bool = True):
         if self.last_fov_num_processed == self.run_structure.highest_fov:
             return
 
-        with self.lock:
-            super().on_moved(event)
-            self._run_callbacks(event, check_last_fov)
+        # with self.lock:
+        super().on_moved(event)
+        self._run_callbacks(event, check_last_fov)
 
     def check_complete(self):
         """Checks run structure fov_progress status.

src/toffy/image_stitching.py

@@ -253,6 +253,13 @@ def stitch_images(
             num_cols = math.isqrt(len(folders))
             max_img_size = get_max_img_size(tiff_out_dir, img_sub_folder, run_dir)
 
+            print(tiff_out_dir)
+            print(img_sub_folder)
+            print(os.listdir(tiff_out_dir))
+            subfolder = "" if img_sub_folder is None else img_sub_folder
+            for fov in folders:
+                print(f"Here's what's in fov {fov}")
+                print(os.listdir(os.path.join(tiff_out_dir, fov, subfolder)))
             image_data = load_utils.load_imgs_from_tree(
                 tiff_out_dir,
                 img_sub_folder=img_sub_folder,

src/toffy/mph_comp.py

@@ -24,12 +24,12 @@ def get_estimated_time(bin_file_dir, fov):
     io_utils.validate_paths(bin_file_dir)
 
     # get fov json file in bin_file_path
-    json_file = io_utils.list_files(bin_file_dir, fov + ".json")
-    if len(json_file) == 0:
+    json_file = os.path.join(bin_file_dir, f"{fov}.json")
+    if not os.path.exists(json_file):
         raise FileNotFoundError(f"The FOV name supplied doesn't have a JSON file: {fov}")
 
     # retrieve estimated time (frame dimensions x pixel dwell time)
-    run_metadata = read_json_file(os.path.join(bin_file_dir, json_file[0]), encoding="utf-8")
+    run_metadata = read_json_file(json_file, encoding="utf-8")
     try:
         size = run_metadata.get("frameSize")
         time = run_metadata.get("dwellTimeMillis")

src/toffy/normalize.py

@@ -65,14 +65,17 @@ def write_mph_per_mass(
         proficient (bool): whether proficient MPH data is written or not
     """
     # compute pulse heights
+    print("Inside writing MPH per mass")
     panel = make_panel(
         mass=masses, target_name=masses, low_range=start_offset, high_range=stop_offset
     )
 
     # generate the MPH values for each mass
+    print("About to get median pulse height")
     mph_vals = list(
         get_median_pulse_height(data_dir=base_dir, fov=fov, channels=masses, panel=panel).values()
     )
+    print("Computed median pulse height")
 
     fovs = np.repeat(fov, len(masses))
     out_df = pd.DataFrame({"mass": masses, "fov": fovs, "pulse_height": mph_vals})

src/toffy/watcher_callbacks.py

@@ -369,6 +369,7 @@ def generate_pulse_heights(self, pulse_out_dir: str, panel: pd.DataFrame = None,
              - start_offset
              - stop_offset
         """
+        print("Inside computing pulse heights")
         if not os.path.exists(pulse_out_dir):
             os.makedirs(pulse_out_dir)
 

tests/fov_watcher_test.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import shutil
 import tempfile
@@ -35,6 +36,72 @@
 SLOW_COPY_INTERVAL_S = 1
 
 
+# def _slow_copy_sample_tissue_data(
+#     dest: str, delta: int = 10, one_blank: bool = False, temp_bin: bool = False
+# ):
+#     """Slowly copies files from ./data/tissue/.
+
+#     Args:
+#         dest (str):
+#             Where to copy tissue files to
+#         delta (int):
+#             Time (in seconds) between each file copy
+#         one_blank (bool):
+#             Add a blank .bin file or not
+#         temp_bin (bool):
+#             Use initial temp bin file paths or not
+#     """
+#     num_bin_files = 0
+#     for tissue_file in sorted(os.listdir(COMBINED_DATA_PATH)):
+#         time.sleep(delta)
+#         if one_blank and ".bin" in tissue_file and tissue_file[0] != ".":
+#             # create blank (0 size) file
+#             open(os.path.join(dest, tissue_file), "w").close()
+#         else:
+#             tissue_path = os.path.join(COMBINED_DATA_PATH, tissue_file)
+#             if temp_bin and ".bin" in tissue_file:
+#                 # copy to a temporary file with hash extension, then move to dest folder
+#                 print(f"Copying over a temporary file with hash extension on {tissue_file}")
+#                 new_tissue_path = os.path.join(COMBINED_DATA_PATH, "." + tissue_file + ".aBcDeF")
+#                 shutil.copy(tissue_path, new_tissue_path)
+#                 shutil.copy(new_tissue_path, dest)
+#                 print(f"Removing the old tissue path at .{tissue_file}.aBcDeF")
+#                 os.remove(new_tissue_path)
+
+#                 # simulate a renaming event in dest
+#                 time.sleep(delta)
+#                 print(f"Renaming back to {tissue_file}")
+#                 copied_tissue_path = os.path.join(dest, "." + tissue_file + ".aBcDeF")
+#                 os.rename(copied_tissue_path, os.path.join(dest, tissue_file))
+
+#             else:
+#                 shutil.copy(tissue_path, dest)
+
+#             # simulate a timestamp update if .bin file has been extracted AND not blank
+#             # NOTE: this assumes .json file always copies after the .bin, which does happen on Ionpath
+#             tissue_data = os.path.splitext(tissue_file)
+#             if tissue_data[1] == ".json" and "_processing" not in tissue_data[0]:
+#                 if one_blank:
+#                     one_blank = False
+#                     continue
+#                 elif num_bin_files % 2 == 0:
+#                     bin_file_name = tissue_data[0] + ".bin"
+#                     print(f"Simulating .bin file update on {bin_file_name}")
+#                     shutil.copy(
+#                         os.path.join(COMBINED_DATA_PATH, bin_file_name),
+#                         os.path.join(dest, bin_file_name + ".temp"),
+#                     )
+#                     print("Renamed bin file to temp")
+#                     os.remove(os.path.join(dest, bin_file_name))
+#                     print("Removed old bin file")
+#                     os.rename(
+#                         os.path.join(dest, bin_file_name + ".temp"),
+#                         os.path.join(dest, bin_file_name),
+#                     )
+#                     print("Renamed temp bin file back to orig")
+#                 num_bin_files += 1
+
+
 def _slow_copy_sample_tissue_data(
     dest: str, delta: int = 10, one_blank: bool = False, temp_bin: bool = False
 ):
@@ -255,7 +322,6 @@ def test_watcher(
     add_blank,
     temp_bin,
 ):
-    print("The watcher start lag is: %d" % watcher_start_lag)
     try:
         with tempfile.TemporaryDirectory() as tmpdir:
             tiff_out_dir = os.path.join(tmpdir, "cb_0", RUN_DIR_NAME)
@@ -371,7 +437,11 @@ def test_watcher(
 
                     res_scan.get()
 
-            with open(os.path.join(log_out, "test_run_log.txt")) as f:
+            print("Testing log out status")
+            print(log_out)
+            # with open(os.path.join(log_out, "test_run_log.txt")) as f:
+            print(os.listdir("."))
+            with open(os.path.join(Path(__file__).parents[1], "pytest.txt")) as f:
                 logtxt = f.read()
                 assert add_blank == ("non-zero file size..." in logtxt)
 
@@ -390,6 +460,7 @@ def test_watcher(
                 fovs = fovs[1:]
 
             # extract tiffs check
+            print("TIFF validator check")
             validators[0](os.path.join(tmpdir, "cb_0", RUN_DIR_NAME), fovs, bad_fovs)
             if kwargs["extract_prof"]:
                 validators[0](
@@ -401,9 +472,11 @@ def test_watcher(
                 )
 
             # qc check
+            print("QC check")
             validators[1](os.path.join(tmpdir, "cb_1", RUN_DIR_NAME), fovs, bad_fovs)
 
             # mph check
+            print("MPH check")
             validators[2](
                 os.path.join(tmpdir, "cb_2", RUN_DIR_NAME),
                 os.path.join(tmpdir, "cb_2_plots", RUN_DIR_NAME),
@@ -412,13 +485,20 @@ def test_watcher(
             )
 
             # stitch images check
+            print("Stitch images check")
             validators[3](os.path.join(tmpdir, "cb_0", RUN_DIR_NAME, f"{RUN_DIR_NAME}_stitched"))
 
             # pulse heights check
+            print("Pulse heights check")
             validators[4](os.path.join(tmpdir, "cb_3", RUN_DIR_NAME), fovs, bad_fovs)
 
-    except OSError:
-        warnings.warn("Temporary file cleanup was incomplete.")
+            with open(os.path.join(Path(__file__).parents[1], "pytest.txt"), "w") as infile:
+                infile.write("")
+
+    except OSError as ose:
+        print("Ran into an error:")
+        print(ose)
+        # warnings.warn("Temporary file cleanup was incomplete.")
 
 
 def test_watcher_missing_fovs():