Many things

src/decolace/processing/cli_match_template.py

@@ -23,11 +23,11 @@ def run_matchtemplate(
     from decolace.processing.project_managment import MatchTemplateRun
     from pycistem.programs import match_template, generate_gpu_prefix, generate_num_procs
     import pycistem
-    pycistem.set_cistem_path("/groups/elferich/cistem_binaries/")
+    pycistem.set_cistem_path(ctx.obj.cistem_path)
     from pycistem.database import get_already_processed_images
     import pandas as pd
     logging.basicConfig(
-        level=logging.INFO,
+        level=logging.DEBUG,
         format="%(message)s",
         handlers=[
             RichHandler(),
@@ -47,6 +47,12 @@ def run_matchtemplate(
         "milano": 8,
         "sofia": 8,
         "manchester": 8,
+        "quebec": 8,
+        "boston": 8,
+        "orleans": 8,
+        "brno": 8,
+        "hannover": 8,
+        "muenchen": 8,
     }  
     if run_on != "all":
         run_profile = {run_on: run_profile[run_on]}
@@ -99,7 +105,7 @@ def run_matchtemplate(
             par.in_plane_angular_step = in_plane_angular_step
             par.defocus_step = defocus_step
             par.defocus_search_range = defocus_range
-            par.max_threads = 2
+            par.max_threads = 6
             par.my_symmetry = symmetry
             if save_mip:
                 par.mip_output_file = par.scaled_mip_output_file.replace("_scaled_mip.mrc", "_mip.mrc")
@@ -114,7 +120,7 @@ def run_matchtemplate(
     all_image_info = pd.concat(all_image_info)
     typer.echo(f"Total of {len(all_image_info)} tiles to process")
 
-    res = match_template.run(all_image_info,num_procs=generate_num_procs(run_profile),cmd_prefix=list(generate_gpu_prefix(run_profile)),cmd_suffix='"', sleep_time=1.0, write_directly_to_db=True)
+    res = match_template.run(all_image_info,num_procs=generate_num_procs(run_profile),cmd_prefix=list(generate_gpu_prefix(run_profile)),cmd_suffix=f'" 2>> /tmp/tmerror.txt 1>> /tmp/tmlog.txt', sleep_time=1.0, write_directly_to_db=True, save_output=False, save_output_path="/tmp/tm_debug/")
         #typer.echo(f"Writing results for {aa.area_name}")
 
         #match_template.write_results_to_database(aa.cistem_project,pars,res,image_info)
@@ -238,9 +244,11 @@ def precompute_filters(
             typer.echo(f"Filter values for matches already exist for {aa.area_name}")
             continue
         refined_matches = starfile.read(refined_matches_starfile)
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            fn = partial(get_distance_to_edge, refined_matches=refined_matches, binning_boxsize=binning_boxsize)
-            executor.map(fn, refined_matches["cisTEMOriginalImageFilename"].unique())
+        for filename in refined_matches["cisTEMOriginalImageFilename"].unique():
+            get_distance_to_edge(filename, refined_matches, binning_boxsize)
+        #with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+        #    fn = partial(get_distance_to_edge, refined_matches=refined_matches, binning_boxsize=binning_boxsize)
+        #    executor.map(fn, refined_matches["cisTEMOriginalImageFilename"].unique())
 
         starfile.write(refined_matches, filtered_matches_starfile, overwrite=True)  
 
@@ -269,6 +277,7 @@ def filter_matches(
             typer.echo(f"No refined matches for {aa.area_name}")
             continue
         refined_matches = starfile.read(refined_matches_starfile)
+        refined_matches["LACEBeamEdgeDistance"] = refined_matches["LACEBeamEdgeDistance"].astype(float)
         print(f"Starting with {len(refined_matches)} matches")
         refined_matches = refined_matches[refined_matches["cisTEMScore"] >= refined_score_cutoff]
         print(f"After {refined_score_cutoff} score criterion {len(refined_matches)} matches")
@@ -322,7 +331,7 @@ def join_matches(
             combined_matches.iloc[i,combined_matches.columns.get_loc('cisTEMOriginalImageFilename')] = "'"+str(new_filename)+"'"
         if use_different_pixel_size is not None:
             combined_matches['cisTEMPixelSize'] = use_different_pixel_size
-    starfile.write(combined_matches, combined_matches_starfile, overwrite=True)
+    starfile.write(combined_matches, combined_matches_starfile, overwrite=True, quote_character="'", quote_all_strings=True)
 
     database = create_project(f"processing_{name}", ctx.obj.project.project_path.absolute() / "cistem_projects")
     print(f"{combined_matches_starfile}")

src/decolace/processing/cli_montaging.py

@@ -31,6 +31,7 @@ def run_montage(
     redo_montage: bool = typer.Option(False, help="Redo only the creatin of the montage even if it already exists"),
     max_mean_density: Optional[float] = typer.Option(None, help="Maximum mean density of the tiles"),
     cc_cutoff_as_fraction_of_median: float = typer.Option(0.5, help="Cutoff for the cross-correlation as a fraction of the median cross-correlation"),
+    debug: bool = typer.Option(False, help="Debug mode")
 ):
     from rich.console import Console
     import starfile

src/decolace/processing/cli_preprocessing.py

@@ -81,6 +81,7 @@ def run_ctffind(
     cmd_suffix: str = typer.Option("", help="Suffix of run command"),
     num_cores: int = typer.Option(10, help="Number of cores to use"),
     fit_nodes: bool = typer.Option(True, help="Fit nodes"),
+    tilt: bool = typer.Option(True, help="Tilt"),
     fit_nodes_brute_force: bool = typer.Option(True, help="Fit nodes brute force"),
     fit_nodes_lowres: float = typer.Option(30.0, help="Fit nodes lowres"),
     fit_nodes_highres: float = typer.Option(4.0, help="Fit nodes highres"),
@@ -89,7 +90,8 @@ def run_ctffind(
     Run ctffind for each acquisition area
     """
     from pycistem.programs import ctffind
-
+    import pycistem
+    pycistem.set_cistem_path(ctx.obj.cistem_path)
 
     for aa in ctx.obj.acquisition_areas:
         if aa.ctffind_run:
@@ -98,6 +100,9 @@ def run_ctffind(
         pars, image_info = ctffind.parameters_from_database(aa.cistem_project,decolace=True)
 
         for par in pars:
+            par.determine_tilt = tilt
+            par.minimum_defocus = 10000
+            par.maximum_defocus = 50000
             par.fit_nodes = fit_nodes
             par.fit_nodes_1D_brute_force = fit_nodes_brute_force
             par.fit_nodes_low_resolution_limit = fit_nodes_lowres
@@ -141,19 +146,16 @@ def redo_projects(
 
 @app.command()
 def redo_unblur(
-    project_main: Path = typer.Option(None, help="Path to wanted project file")
+    ctx: typer.Context,
 ):
 
-    if project_main is None:
-        project_path = Path(glob.glob("*.decolace")[0])
-    project = ProcessingProject.read(project_path)
-
-    for aa in project.acquisition_areas:
+    for aa in ctx.obj.acquisition_areas:
         aa.unblur_run = False
 
-    project.write()
+    ctx.obj.project.write()
 
-@app.command()
+
+app.command()
 def redo_ctffind(
     project_main: Path = typer.Option(None, help="Path to wanted project file")
 ):

src/decolace/processing/cli_project_managment.py

@@ -39,13 +39,13 @@ def add_acquisition_area(
 
     aa = AcquisitionAreaSingle(area_name, area_directory.as_posix())
     aa.load_from_disk()
-    if np.sum(aa.state['positions_acquired']) == 0:
+    if np.sum(aa.state.positions_acquired) == 0:
                 print(f"{aa.name}: No Data")
     aa_pre = AcquisitionAreaPreProcessing(
                 area_name = f"{session_name}_{grid_name}_{aa.name}",
                 decolace_acquisition_area_info_path = sorted(Path(aa.directory).glob(f"{aa.name}*.npy"))[-1],
                 decolace_grid_info_path = None,
-                decolace_session_info_path = None,
+                decolace_session_info_path = sorted(Path(aa.directory).parent.parent.glob(f"{session_name}*.npy"))[-1],
                 frames_folder = aa.frames_directory,
             )
     ctx.obj.project.acquisition_areas.append(aa_pre)