From c7258bc836176cbfe67fde6155d6f65a6e67abcb Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 11:02:43 +0200 Subject: [PATCH 001/127] Changed UI to new (broke param names) --- omero/annotation_scripts/KeyVal_from_csv.py | 35 ++++++++++++++------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 1bd8f5797..474427bcb 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -305,28 +305,41 @@ def annotate_object(conn, obj, header, row, cols_to_ignore): def run_script(): - data_types = [rstring('Dataset'), rstring('Plate')] + source_types = [rstring("Project"), rstring("Dataset"), + rstring("Screen"), rstring("Plate"), + rstring("Run"), rstring("Well")] + + target_types = [rstring("Dataset"), rstring("Plate"), + rstring("Run"), rstring("Well"), + rstring("Image")] + client = scripts.client( 'Add_Key_Val_from_csv', """ - This script processes a csv file, attached to a Dataset + This script reads an attached CSV file to annotate objects with key-value pairs. """, scripts.String( - "Data_Type", optional=False, grouping="1", - description="Choose source of images", - values=data_types, default="Dataset"), + "Source object type", optional=False, grouping="1", + description="Choose the object type containing the objects to annotate", + values=source_types, default="Dataset"), scripts.List( - "IDs", optional=False, grouping="2", - description="Dataset or Plate ID(s).").ofType(rlong(0)), + "Source IDs", optional=False, grouping="1.1", + description="List of source IDs containing the images to annotate").ofType(rlong(0)), scripts.String( - "File_Annotation", grouping="3", + "File_Annotation ID", optional=False, grouping="1.2", description="File ID containing metadata to populate."), - authors=["Christian Evenhuis"], - institutions=["MIF UTS"], - contact="https://forum.image.sc/tag/omero" + scripts.String( + "Target object type", optional=False, grouping="2", + description="Choose the object type to annotate (must be bellow the chosen source object type)", + values=target_types, default="Image"), + + authors=["Christian Evenhuis", "Tom Boissonnet"], + institutions=["MIF UTS", "CAi HHU"], + contact="https://forum.image.sc/tag/omero", + version="2.0.0" ) try: From 0b22f161dc25b29a6f3e8ba05ef13d1f11a8859c Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 11:25:01 +0200 Subject: [PATCH 002/127] Assertion on Source Target relationship --- omero/annotation_scripts/KeyVal_from_csv.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 474427bcb..06b476d35 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -313,6 +313,15 @@ def run_script(): rstring("Run"), rstring("Well"), rstring("Image")] + allowed_relations = { + "Project": ["Dataset", "Image"], + "Dataset": ["Image"], + "Screen": ["Plate", "Run", "Well", "Image"], + "Plate": ["Run", "Well", "Image"], + "Run": ["Well", "Image"], + "Well": ["Image"] + } + client = scripts.client( 'Add_Key_Val_from_csv', """ @@ -342,6 +351,7 @@ def run_script(): version="2.0.0" ) + try: # process the list of args above. script_params = {} @@ -349,6 +359,10 @@ def run_script(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) + # validate that target is bellow source + source_name, target_name = script_params["Source object type"], script_params["Target object type"] + assert target_name in allowed_relations[source_name], f"Invalid {source_name} => {target_name}. The target type must be a child of the source type" + # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) print("script params") @@ -357,6 +371,10 @@ def run_script(): message = keyval_from_csv(conn, script_params) client.setOutput("Message", rstring(message)) + except AssertionError as err: #Display assertion errors in OMERO.web activities + client.setOutput("ERROR", rstring(err)) + raise AssertionError(str(err)) + finally: client.closeSession() From c875d38c159a695b2ccd9cfddb9cc079341d5f26 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 12:01:05 +0200 Subject: [PATCH 003/127] Rename param and changed AnnID to list input --- omero/annotation_scripts/KeyVal_from_csv.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 06b476d35..5f84f758b 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -328,20 +328,20 @@ def run_script(): This script reads an attached CSV file to annotate objects with key-value pairs. """, scripts.String( - "Source object type", optional=False, grouping="1", + "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to annotate", values=source_types, default="Dataset"), scripts.List( - "Source IDs", optional=False, grouping="1.1", - description="List of source IDs containing the images to annotate").ofType(rlong(0)), + "Source_IDs", optional=False, grouping="1.1", + description="List of source IDs containing the images to annotate.").ofType(rlong(0)), - scripts.String( - "File_Annotation ID", optional=False, grouping="1.2", - description="File ID containing metadata to populate."), + scripts.List( + "File_Annotation_ID", optional=True, grouping="1.2", + description="File IDs containing metadata to populate. If given, must match 'Source IDs'. Otherwise, uses the first CSV found on the source.").ofType(rlong(0)), scripts.String( - "Target object type", optional=False, grouping="2", + "Target_object_type", optional=False, grouping="2", description="Choose the object type to annotate (must be bellow the chosen source object type)", values=target_types, default="Image"), @@ -360,7 +360,7 @@ def run_script(): script_params[key] = client.getInput(key, unwrap=True) # validate that target is bellow source - source_name, target_name = script_params["Source object type"], script_params["Target object type"] + source_name, target_name = script_params["Source_object_type"], script_params["Target_object_type"] assert target_name in allowed_relations[source_name], f"Invalid {source_name} => {target_name}. The target type must be a child of the source type" # wrap client to use the Blitz Gateway From 1b611bd5986fa37a2f5d9ecce50f48f3e9b0d072 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 12:03:18 +0200 Subject: [PATCH 004/127] Stop AnnotationFile search on first item found --- omero/annotation_scripts/KeyVal_from_csv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 5f84f758b..9d758b41d 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -81,6 +81,7 @@ def get_original_file(omero_object, file_ann_id=None): if (file_ann_id is None and file_name.endswith(".csv")) or ( ann.getId() == file_ann_id): file_ann = ann + break # Stop on first matching item if file_ann is None: sys.stderr.write("Error: File does not exist.\n") sys.exit(1) From f570f73c2043024d5fc1a7301d7220abd10fda4d Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 15:03:25 +0200 Subject: [PATCH 005/127] Great refactor and use new inputs --- omero/annotation_scripts/KeyVal_from_csv.py | 259 +++++++++----------- 1 file changed, 110 insertions(+), 149 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 9d758b41d..a86e0acde 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -40,6 +40,15 @@ from collections import OrderedDict +HIERARCHY_OBJECTS = { + "Project": ["Dataset", "Image"], + "Dataset": ["Image"], + "Screen": ["Plate", "Well", "Image"], + "Plate": ["Well", "Image"], + #"Run": ["Well", "Image"], + "Well": ["Image"] + } + def get_existing_map_annotations(obj): """Get all Map Annotations linked to the object""" @@ -103,158 +112,116 @@ def link_file_ann(conn, object_type, object_id, file_ann_id): if len(links) == 0: omero_object.linkAnnotation(file_ann) - -def get_children_by_name(omero_obj): - - images_by_name = {} - wells_by_name = {} - - if omero_obj.OMERO_CLASS == "Dataset": - for img in omero_obj.listChildren(): - img_name = img.getName() - if img_name in images_by_name: - sys.stderr.write("File names not unique: {}".format(img_name)) - sys.exit(1) - images_by_name[img_name] = img - elif omero_obj.OMERO_CLASS == "Plate": - for well in omero_obj.listChildren(): - label = well.getWellPos() - wells_by_name[label] = well - for ws in well.listChildren(): - img = ws.getImage() - img_name = img.getName() - if img_name in images_by_name: - sys.stderr.write( - "File names not unique: {}".format(img_name)) - sys.exit(1) - images_by_name[img_name] = img - else: - sys.stderr.write(f'{omero_obj.OMERO_CLASS} objects not supported') - - return images_by_name, wells_by_name - - -def keyval_from_csv(conn, script_params): - data_type = script_params["Data_Type"] - ids = script_params["IDs"] - - nimg_processed = 0 - nimg_updated = 0 - missing_names = 0 - - for target_object in conn.getObjects(data_type, ids): - - # file_ann_id is Optional. If not supplied, use first .csv attached - file_ann_id = None - if "File_Annotation" in script_params: - file_ann_id = int(script_params["File_Annotation"]) - link_file_ann(conn, data_type, target_object.id, file_ann_id) - print("set ann id", file_ann_id) - - original_file = get_original_file(target_object, file_ann_id) - print("Original File", original_file.id.val, original_file.name.val) - provider = DownloadingOriginalFileProvider(conn) - - # read the csv - temp_file = provider.get_original_file_data(original_file) - # Needs omero-py 5.9.1 or later - temp_name = temp_file.name - file_length = original_file.size.val - with open(temp_name, 'rt', encoding='utf-8-sig') as file_handle: +def read_csv(conn, source_object, file_ann_id): #Dedicated function to read the CSV file + original_file = get_original_file(source_object, file_ann_id) + print("Original File", original_file.id.val, original_file.name.val) + provider = DownloadingOriginalFileProvider(conn) + # read the csv + temp_file = provider.get_original_file_data(original_file) + # Needs omero-py 5.9.1 or later + temp_name = temp_file.name + file_length = original_file.size.val + with open(temp_name, 'rt', encoding='utf-8-sig') as file_handle: + try: + delimiter = csv.Sniffer().sniff( + file_handle.read(floor(file_length/4)), ",;\t").delimiter + print("Using delimiter: ", delimiter, + f" after reading {floor(file_length/4)} characters") + except Exception: + file_handle.seek(0) try: delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length/4)), ",;\t").delimiter + file_handle.read(floor(file_length/2)), + ",;\t").delimiter print("Using delimiter: ", delimiter, - f" after reading {floor(file_length/4)} characters") + f"after reading {floor(file_length/2)} characters") except Exception: file_handle.seek(0) try: delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length/2)), + file_handle.read(floor(file_length*0.75)), ",;\t").delimiter print("Using delimiter: ", delimiter, - f"after reading {floor(file_length/2)} characters") + f" after reading {floor(file_length*0.75)}" + " characters") except Exception: - file_handle.seek(0) - try: - delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length*0.75)), - ",;\t").delimiter - print("Using delimiter: ", delimiter, - f" after reading {floor(file_length*0.75)}" - " characters") - except Exception: - print("Failed to sniff delimiter, using ','") - delimiter = "," - - # reset to start and read whole file... - file_handle.seek(0) - data = list(csv.reader(file_handle, delimiter=delimiter)) - - # keys are in the header row - header = data[0] - print("header", header) - - # create dictionaries for well/image name:object - images_by_name, wells_by_name = get_children_by_name(target_object) - nimg_processed += len(images_by_name) - - image_index = header.index("image") if "image" in header else -1 - well_index = header.index("well") if "well" in header else -1 - plate_index = header.index("plate") if "plate" in header else -1 - if image_index == -1: - # first header is the img-name column, if 'image' not found - image_index = 0 - print("image_index:", image_index, "well_index:", well_index, - "plate_index:", plate_index) - rows = data[1:] + print("Failed to sniff delimiter, using ','") + delimiter = "," - # loop over csv rows... - for row in rows: - # try to find 'image', then 'well', then 'plate' - image_name = row[image_index] - well_name = None - plate_name = None - obj = None - if len(image_name) > 0: - if image_name in images_by_name: - obj = images_by_name[image_name] - print("Annotating Image:", obj.id, image_name) - else: - missing_names += 1 - print("Image not found:", image_name) - if obj is None and well_index > -1 and len(row[well_index]) > 0: - well_name = row[well_index] - if well_name in wells_by_name: - obj = wells_by_name[well_name] - print("Annotating Well:", obj.id, well_name) + # reset to start and read whole file... + file_handle.seek(0) + data = list(csv.reader(file_handle, delimiter=delimiter)) + + # keys are in the header row + header = data[0] + print("header", header) + return data, header + +def get_children_recursive(source_object, target_type): + if HIERARCHY_OBJECTS[source_object.OMERO_CLASS][0] == target_type: # Stop condition, we return the source_obj children + return source_object.listChildren() + else: + result = [] + for child_obj in source_object.listChildren(): + # Going down in the Hierarchy list for all childs that aren't yet the target + result.extend(get_children_recursive(child_obj, target_type)) + return result + +def keyval_from_csv(conn, script_params): + source_type = script_params["Source_object_type"] + target_type = script_params["Target_object_type"] + source_ids = script_params["Source_IDs"] + file_ids = script_params["File_Annotation_ID"] + + ntarget_processed = 0 + ntarget_updated = 0 + missing_names = 0 + + for source_object, file_ann_id in zip(conn.getObjects(source_type, source_ids), file_ids): + #link_file_ann(conn, source_type, source_object.id, file_ann_id) # Make sure file is attached to the source + print("set ann id", file_ann_id) + data, header = read_csv(conn, source_object, file_ann_id) + + # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) + target_obj_l = get_children_recursive(source_object, target_type) + + # Finds the index of the column used to identify the targets. Try for IDs first + idx_id = header.index("target_id") if "target_id" in header else -1 + idx_name = header.index("target_name") if "target_name" in header else -1 + use_id = idx_id != -1 + + if not use_id: # Identify images by name must fail if two images have identical names + idx_id = idx_name + target_d = dict() + for target_obj in target_obj_l: + assert target_obj.getName() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getName()}" + target_d[target_obj.getName()] = target_obj + else: # Setting the dictionnary target_id:target_obj keys as string to match CSV reader output + target_d = {str(target_obj.getId()):target_obj for target_obj in target_obj_l} + ntarget_processed += len(target_d) + + rows = data[1:] + for row in rows: # Iterate the CSV rows and search for the matching target + target_id = row[idx_id] + if target_id in target_d.keys(): + target_obj = target_d[target_id] + if target_type in ["Dataset", "Image", "Plate"]: + print("Annotating Target:", f"{target_obj.getName()+':' if use_id else ''}{target_id}") else: - missing_names += 1 - print("Well not found:", well_name) - # always check that Plate name matches if it is given: - if data_type == "Plate" and plate_index > -1 and \ - len(row[plate_index]) > 0: - if row[plate_index] != target_object.name: - print("plate", row[plate_index], - "doesn't match object", target_object.name) - continue - if obj is None: - obj = target_object - print("Annotating Plate:", obj.id, plate_name) - if obj is None: - msg = "Can't find object by image, well or plate name" - print(msg) + print("Annotating Target:", f"{target_id}") # Some object don't have a name + else: + missing_names += 1 + print(f"Target not found: {target_id}") continue - cols_to_ignore = [image_index, well_index, plate_index] - updated = annotate_object(conn, obj, header, row, cols_to_ignore) + cols_to_ignore = [idx_id, idx_name] + updated = annotate_object(conn, target_obj, header, row, cols_to_ignore) if updated: - nimg_updated += 1 + ntarget_updated += 1 - message = "Added kv pairs to {}/{} files".format( - nimg_updated, nimg_processed) + message = f"Added kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" if missing_names > 0: - message += f". {missing_names} image names not found." + message += f". {missing_names} {target_type} not found (using {'ID' if use_id else 'name'} to identify them)." return message @@ -308,20 +275,10 @@ def run_script(): source_types = [rstring("Project"), rstring("Dataset"), rstring("Screen"), rstring("Plate"), - rstring("Run"), rstring("Well")] + rstring("Well")] target_types = [rstring("Dataset"), rstring("Plate"), - rstring("Run"), rstring("Well"), - rstring("Image")] - - allowed_relations = { - "Project": ["Dataset", "Image"], - "Dataset": ["Image"], - "Screen": ["Plate", "Run", "Well", "Image"], - "Plate": ["Run", "Well", "Image"], - "Run": ["Well", "Image"], - "Well": ["Image"] - } + rstring("Well"), rstring("Image")] client = scripts.client( 'Add_Key_Val_from_csv', @@ -355,14 +312,18 @@ def run_script(): try: # process the list of args above. - script_params = {} + script_params = {"File_Annotation_ID": [None]} # Set with defaults for optional parameters for key in client.getInputKeys(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) # validate that target is bellow source source_name, target_name = script_params["Source_object_type"], script_params["Target_object_type"] - assert target_name in allowed_relations[source_name], f"Invalid {source_name} => {target_name}. The target type must be a child of the source type" + assert target_name in HIERARCHY_OBJECTS[source_name], f"Invalid {source_name} => {target_name}. The target type must be a child of the source type" + + if len(script_params["File_Annotation_ID"]) == 1: # Poulate the parameter with None or same ID for all source + script_params["File_Annotation_ID"] = script_params["File_Annotation_ID"] * len(script_params["Source_IDs"]) + assert len(script_params["File_Annotation_ID"]) == len(script_params["Source_IDs"]), "Number of Source IDs and FileAnnotation IDs must match" # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) From 7348f2f492f1815e354c2c937488968e066d0e74 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 16:12:39 +0200 Subject: [PATCH 006/127] Changed csv file selection when multiple (highest ID) --- omero/annotation_scripts/KeyVal_from_csv.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index a86e0acde..42ecffdd8 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -87,10 +87,13 @@ def get_original_file(omero_object, file_ann_id=None): if isinstance(ann, omero.gateway.FileAnnotationWrapper): file_name = ann.getFile().getName() # Pick file by Ann ID (or name if ID is None) - if (file_ann_id is None and file_name.endswith(".csv")) or ( - ann.getId() == file_ann_id): - file_ann = ann - break # Stop on first matching item + if ann.getId() == file_ann_id: + file_ann = ann # Found it + break + elif file_ann_id is None and file_name.endswith(".csv"): + if (file_ann is None) or (ann.getId() > file_ann.getId()): + # Get the file with the biggest ID, that should be the most recent + file_ann = ann if file_ann is None: sys.stderr.write("Error: File does not exist.\n") sys.exit(1) @@ -296,7 +299,7 @@ def run_script(): scripts.List( "File_Annotation_ID", optional=True, grouping="1.2", - description="File IDs containing metadata to populate. If given, must match 'Source IDs'. Otherwise, uses the first CSV found on the source.").ofType(rlong(0)), + description="List of file IDs containing metadata to populate. If given, must match length of 'Source IDs'. Otherwise, uses the CSV file with the highest ID.").ofType(rlong(0)), scripts.String( "Target_object_type", optional=False, grouping="2", From 518d84e3c97767e19f1046b13cb482456a7aaca3 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 16:42:54 +0200 Subject: [PATCH 007/127] file attachment to sources fix for no AnnID provided --- omero/annotation_scripts/KeyVal_from_csv.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 42ecffdd8..c805767dd 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -98,7 +98,7 @@ def get_original_file(omero_object, file_ann_id=None): sys.stderr.write("Error: File does not exist.\n") sys.exit(1) - return file_ann.getFile()._obj + return file_ann def link_file_ann(conn, object_type, object_id, file_ann_id): @@ -115,8 +115,7 @@ def link_file_ann(conn, object_type, object_id, file_ann_id): if len(links) == 0: omero_object.linkAnnotation(file_ann) -def read_csv(conn, source_object, file_ann_id): #Dedicated function to read the CSV file - original_file = get_original_file(source_object, file_ann_id) +def read_csv(conn, original_file): #Dedicated function to read the CSV file print("Original File", original_file.id.val, original_file.name.val) provider = DownloadingOriginalFileProvider(conn) # read the csv @@ -181,9 +180,12 @@ def keyval_from_csv(conn, script_params): missing_names = 0 for source_object, file_ann_id in zip(conn.getObjects(source_type, source_ids), file_ids): - #link_file_ann(conn, source_type, source_object.id, file_ann_id) # Make sure file is attached to the source - print("set ann id", file_ann_id) - data, header = read_csv(conn, source_object, file_ann_id) + if file_ann_id is not None: # If the file ID is not defined, only already linked file will be used + link_file_ann(conn, source_type, source_object.id, file_ann_id) + file_ann = get_original_file(source_object, file_ann_id) + original_file = file_ann.getFile()._obj + print("set ann id", file_ann.getId()) + data, header = read_csv(conn, original_file) # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) target_obj_l = get_children_recursive(source_object, target_type) From 6d87ace068cd7b61ef5d0f5a99056ee92f9a7355 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 23 Oct 2023 18:16:44 +0200 Subject: [PATCH 008/127] added namespace to KV --- omero/annotation_scripts/KeyVal_from_csv.py | 30 ++++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index c805767dd..70a80f832 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -50,10 +50,10 @@ } -def get_existing_map_annotations(obj): +def get_existing_map_annotations(obj, namespace): """Get all Map Annotations linked to the object""" ord_dict = OrderedDict() - for ann in obj.listAnnotations(): + for ann in obj.listAnnotations(ns=namespace): if isinstance(ann, omero.gateway.MapAnnotationWrapper): kvs = ann.getValue() for k, v in kvs: @@ -63,9 +63,9 @@ def get_existing_map_annotations(obj): return ord_dict -def remove_map_annotations(conn, object): +def remove_map_annotations(conn, object, namespace): """Remove ALL Map Annotations on the object""" - anns = list(object.listAnnotations()) + anns = list(object.listAnnotations(ns=namespace)) mapann_ids = [ann.id for ann in anns if isinstance(ann, omero.gateway.MapAnnotationWrapper)] @@ -174,6 +174,7 @@ def keyval_from_csv(conn, script_params): target_type = script_params["Target_object_type"] source_ids = script_params["Source_IDs"] file_ids = script_params["File_Annotation_ID"] + namespace = script_params["Namespace (optional)"] ntarget_processed = 0 ntarget_updated = 0 @@ -220,7 +221,7 @@ def keyval_from_csv(conn, script_params): continue cols_to_ignore = [idx_id, idx_name] - updated = annotate_object(conn, target_obj, header, row, cols_to_ignore) + updated = annotate_object(conn, target_obj, header, row, cols_to_ignore, namespace) if updated: ntarget_updated += 1 @@ -230,10 +231,13 @@ def keyval_from_csv(conn, script_params): return message -def annotate_object(conn, obj, header, row, cols_to_ignore): +def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): + + if namespace is None: + namespace = omero.constants.metadata.NSCLIENTMAPANNOTATION obj_updated = False - existing_kv = get_existing_map_annotations(obj) + existing_kv = get_existing_map_annotations(obj, namespace) updated_kv = copy.deepcopy(existing_kv) print("Existing kv:") for k, vset in existing_kv.items(): @@ -257,9 +261,8 @@ def annotate_object(conn, obj, header, row, cols_to_ignore): if existing_kv != updated_kv: obj_updated = True print("The key-values pairs are different") - remove_map_annotations(conn, obj) + remove_map_annotations(conn, obj, namespace) map_ann = omero.gateway.MapAnnotationWrapper(conn) - namespace = omero.constants.metadata.NSCLIENTMAPANNOTATION map_ann.setNs(namespace) # convert the ordered dict to a list of lists kv_list = [] @@ -308,6 +311,10 @@ def run_script(): description="Choose the object type to annotate (must be bellow the chosen source object type)", values=target_types, default="Image"), + scripts.String( + "Namespace (optional)", optional=True, grouping="3", + description="Choose a namespace for the annotations"), + authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero", @@ -317,7 +324,10 @@ def run_script(): try: # process the list of args above. - script_params = {"File_Annotation_ID": [None]} # Set with defaults for optional parameters + script_params = { # Param dict with defaults for optional parameters + "File_Annotation_ID": [None], + "Namespace (optional)": None + } for key in client.getInputKeys(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) From 8181454306528d859cce4dfc636ee65a857d209d Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 11:09:11 +0200 Subject: [PATCH 009/127] removed existing kvpairs update/deletion --- omero/annotation_scripts/KeyVal_from_csv.py | 88 ++++----------------- 1 file changed, 16 insertions(+), 72 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 70a80f832..2845222fb 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -49,37 +49,6 @@ "Well": ["Image"] } - -def get_existing_map_annotations(obj, namespace): - """Get all Map Annotations linked to the object""" - ord_dict = OrderedDict() - for ann in obj.listAnnotations(ns=namespace): - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - kvs = ann.getValue() - for k, v in kvs: - if k not in ord_dict: - ord_dict[k] = set() - ord_dict[k].add(v) - return ord_dict - - -def remove_map_annotations(conn, object, namespace): - """Remove ALL Map Annotations on the object""" - anns = list(object.listAnnotations(ns=namespace)) - mapann_ids = [ann.id for ann in anns - if isinstance(ann, omero.gateway.MapAnnotationWrapper)] - - try: - delete = Delete2(targetObjects={'MapAnnotation': mapann_ids}) - handle = conn.c.sf.submit(delete) - conn.c.waitOnCmd(handle, loops=10, ms=500, failonerror=True, - failontimeout=False, closehandle=False) - - except Exception as ex: - print("Failed to delete links: {}".format(ex.message)) - return - - def get_original_file(omero_object, file_ann_id=None): """Find file linked to object. Option to filter by ID.""" file_ann = None @@ -174,7 +143,7 @@ def keyval_from_csv(conn, script_params): target_type = script_params["Target_object_type"] source_ids = script_params["Source_IDs"] file_ids = script_params["File_Annotation_ID"] - namespace = script_params["Namespace (optional)"] + namespace = script_params["Namespace (leave blank for default)"] ntarget_processed = 0 ntarget_updated = 0 @@ -233,50 +202,25 @@ def keyval_from_csv(conn, script_params): def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): - if namespace is None: - namespace = omero.constants.metadata.NSCLIENTMAPANNOTATION - - obj_updated = False - existing_kv = get_existing_map_annotations(obj, namespace) - updated_kv = copy.deepcopy(existing_kv) - print("Existing kv:") - for k, vset in existing_kv.items(): - for v in vset: - print(" ", k, v) - print("Adding kv:") + kv_list = [] + for i in range(len(row)): if i in cols_to_ignore or i >= len(header): continue key = header[i].strip() - vals = row[i].strip().split(';') - if len(vals) > 0: - for val in vals: - if len(val) > 0: - if key not in updated_kv: - updated_kv[key] = set() - print(" ", key, val) - updated_kv[key].add(val) - - if existing_kv != updated_kv: - obj_updated = True - print("The key-values pairs are different") - remove_map_annotations(conn, obj, namespace) - map_ann = omero.gateway.MapAnnotationWrapper(conn) - map_ann.setNs(namespace) - # convert the ordered dict to a list of lists - kv_list = [] - for k, vset in updated_kv.items(): - for v in vset: - kv_list.append([k, v]) - map_ann.setValue(kv_list) - map_ann.save() - print("Map Annotation created", map_ann.id) - obj.linkAnnotation(map_ann) - else: - print("No change change in kv") + value = row[i].strip() + kv_list.append([key, value]) + + map_ann = omero.gateway.MapAnnotationWrapper(conn) + map_ann.setNs(namespace) + map_ann.setValue(kv_list) + map_ann.save() + + print("Map Annotation created", map_ann.id) + obj.linkAnnotation(map_ann) - return obj_updated + return True def run_script(): @@ -312,7 +256,7 @@ def run_script(): values=target_types, default="Image"), scripts.String( - "Namespace (optional)", optional=True, grouping="3", + "Namespace (leave blank for default)", optional=True, grouping="3", description="Choose a namespace for the annotations"), authors=["Christian Evenhuis", "Tom Boissonnet"], @@ -326,7 +270,7 @@ def run_script(): # process the list of args above. script_params = { # Param dict with defaults for optional parameters "File_Annotation_ID": [None], - "Namespace (optional)": None + "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION } for key in client.getInputKeys(): if client.getInput(key): From f932f288b0a8941daf8a5dc3c557c6a0c9adeeb7 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 11:42:48 +0200 Subject: [PATCH 010/127] Updated script info --- omero/annotation_scripts/KeyVal_from_csv.py | 40 +++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 2845222fb..5f8c8c30a 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -1,11 +1,16 @@ # coding=utf-8 """ - MIF/Add_Key_Val_from_csv.py + KeyVal_from_csv.py - Adds key-value (kv) metadata to images in a dataset from a csv file - The first column contains the filenames - The first row of the file contains the keys - The rest is the values for each file/key + Adds key-value pairs to TARGETS on OMERO from a CSV file attached to a SOURCE container. + SOURCES can be: [Project, Dataset, Screen, Plate, Well] + TARGETS can be: [Dataset, Plate, Well, Image] + The targets are referenced in the CSV file either from their name (must then be unique, + and be called "target_name") or from their ID (must be called "target_id"). + In the case both are given, the ID will be used. + + Every row corresponds to a set of value to attach to the given TARGET with the key of the + correponding column. ----------------------------------------------------------------------------- Copyright (C) 2018 @@ -235,7 +240,30 @@ def run_script(): client = scripts.client( 'Add_Key_Val_from_csv', """ - This script reads an attached CSV file to annotate objects with key-value pairs. + This script reads an attached .csv file to annotate objects with key-value pairs. + + Only the child objects of the SOURCE will be searched and if they match an entry in + the .csv file, then a set of key-value pair will be added to the TARGET. + + In the .csv file, the TARGETs can be identified by their name (with a column named + "target_name"), in which case their names must be unique among all children objects + of the SOURCE. The TARGETs can also be identified by their IDs (with a column named + "target_id"). In case both are given, "target_name" will be ignored in favor of + "target_id". + + The .csv file must be imported in OMERO as a file annotation, and is passed as a + parameter to the script via the AnnotationID. + + Multiple SOURCE and AnnotationID can be passed to the script, and each will be + processed independantly. When using a single AnnotationID, the same .csv will be + used for each SOURCE. When no AnnotationID is given, each SOURCE will use the + most recently attached .csv on itself. + + The annotation can also be associated to a namespace (defaults to user namespace). + + Complementary scripts: + - "Export Key Value to csv": Export key value pairs of a given namespace + - "Delete Key Value": Delete the key value pairs associated to a namespace """, scripts.String( "Source_object_type", optional=False, grouping="1", From cbe1595a49925aa80cee300d3389a497b72409ca Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 11:58:51 +0200 Subject: [PATCH 011/127] compare file annotation on date rather than ID --- omero/annotation_scripts/KeyVal_from_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 5f8c8c30a..ef27d06be 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -65,8 +65,8 @@ def get_original_file(omero_object, file_ann_id=None): file_ann = ann # Found it break elif file_ann_id is None and file_name.endswith(".csv"): - if (file_ann is None) or (ann.getId() > file_ann.getId()): - # Get the file with the biggest ID, that should be the most recent + if (file_ann is None) or (ann.getDate() > file_ann.getDate()): + # Get the most recent file file_ann = ann if file_ann is None: sys.stderr.write("Error: File does not exist.\n") From 64fd963cdf0e3f12e68c31dad5edca4af7aad044 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 12:21:07 +0200 Subject: [PATCH 012/127] Minor tip update --- omero/annotation_scripts/KeyVal_from_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index ef27d06be..6f7097a94 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -272,7 +272,7 @@ def run_script(): scripts.List( "Source_IDs", optional=False, grouping="1.1", - description="List of source IDs containing the images to annotate.").ofType(rlong(0)), + description="List of source IDs containing the objects to annotate.").ofType(rlong(0)), scripts.List( "File_Annotation_ID", optional=True, grouping="1.2", From 4f26db0af817846ced058d46cf269fdbc0d0c77f Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 12:29:07 +0200 Subject: [PATCH 013/127] Modified deletion according to KeyVal_import_from_csv logic --- omero/annotation_scripts/Remove_KeyVal.py | 131 ++++++++++++---------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index d2bfb47f0..bf5f0c170 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -31,9 +31,17 @@ from omero.rtypes import rlong, rstring, wrap import omero.scripts as scripts - -def remove_map_annotations(conn, obj): - anns = list(obj.listAnnotations()) +HIERARCHY_OBJECTS = { + "Project": ["Dataset", "Image"], + "Dataset": ["Image"], + "Screen": ["Plate", "Well", "Image"], + "Plate": ["Well", "Image"], + #"Run": ["Well", "Image"], + "Well": ["Image"] + } + +def remove_map_annotations(conn, obj, namespace): + anns = list(obj.listAnnotations(ns=namespace)) mapann_ids = [ann.id for ann in anns if isinstance(ann, omero.gateway.MapAnnotationWrapper)] if len(mapann_ids) == 0: @@ -46,49 +54,50 @@ def remove_map_annotations(conn, obj): except Exception: print("Failed to delete links") return 1 - return -def get_objects(conn, script_params): +def get_children_recursive(source_object, target_type): + if HIERARCHY_OBJECTS[source_object.OMERO_CLASS][0] == target_type: # Stop condition, we return the source_obj children + return source_object.listChildren() + else: + result = [] + for child_obj in source_object.listChildren(): + # Going down in the Hierarchy list for all childs that aren't yet the target + result.extend(get_children_recursive(child_obj, target_type)) + return result + +def remove_keyvalue(conn, script_params): """ File the list of objects @param conn: Blitz Gateway connection wrapper @param script_params: A map of the input parameters """ - # we know script_params will have "Data_Type" and "IDs" since these - # parameters are not optional - data_type = script_params["Data_Type"] - ids = script_params["IDs"] - - # data_type is 'Dataset', 'Plate' or 'Image' so we can use it directly in - objs = list(conn.getObjects(data_type, ids)) - - if len(objs) == 0: - print("No {} found for specified IDs".format(data_type)) - return - - objs_ret = [] - - if data_type == 'Dataset': - for ds in objs: - print("Processing Images from Dataset: {}".format(ds.getName())) - objs_ret.append(ds) - imgs = list(ds.listChildren()) - objs_ret.extend(imgs) - elif data_type == "Plate": - for plate in objs: - print("Processing Wells and Images from Plate:", plate.getName()) - objs_ret.append(plate) - for well in plate.listChildren(): - objs_ret.append(well) - for ws in well.listChildren(): - img = ws.getImage() - objs_ret.append(img) + source_type = script_params["Source_object_type"] + target_type = script_params["Target_object_type"] + source_ids = script_params["Source_IDs"] + namespace = script_params["Namespace (leave blank for default)"] + + nfailed = 0 + ntotal = 0 + if source_type == target_type: # We remove annotation to the given object ID + for source_object in conn.getObjects(source_type, source_ids): + print("Processing object:", source_object) + ret = remove_map_annotations(conn, source_object, namespace) + nfailed = nfailed + ret + ntotal += 1 else: - print("Processing Images identified by ID") - objs_ret = objs + for source_object in conn.getObjects(source_type, source_ids): + # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) + target_obj_l = get_children_recursive(source_object, target_type) + for target_obj in target_obj_l: + print("Processing object:", target_obj) + ret = remove_map_annotations(conn, target_obj, namespace) + nfailed = nfailed + ret + ntotal += 1 + + message = f"Key value data deleted from {ntotal-nfailed} of {ntotal} objects" - return objs_ret + return message if __name__ == "__main__": @@ -97,7 +106,9 @@ def get_objects(conn, script_params): scripting service, passing the required parameters. """ - data_types = wrap(['Dataset', 'Plate', 'Image']) + data_types = [rstring("Project"), rstring("Dataset"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Image")] # Here we define the script name and description. # Good practice to put url here to give users more guidance on how to run @@ -110,21 +121,33 @@ def get_objects(conn, script_params): "scripts/user-guide.html for the tutorial that uses this script."), scripts.String( - "Data_Type", optional=False, grouping="1", - description="The data you want to work with.", values=data_types, - default="Dataset"), + "Source_object_type", optional=False, grouping="1", + description="Choose the object type containing the objects to delete annotation from", + values=data_types, default="Image"), scripts.List( - "IDs", optional=False, grouping="2", - description="List of Dataset IDs or Image IDs").ofType(rlong(0)), + "Source_IDs", optional=False, grouping="1.1", + description="List of source IDs").ofType(rlong(0)), + + scripts.String( + "Target_object_type", optional=False, grouping="2", + description="Choose the object type to delete annotation from", + values=data_types, default="Image"), + + scripts.String( + "Namespace (leave blank for default)", optional=True, grouping="3", + description="Choose a namespace for the annotations"), - authors=["Christian Evenhuis", "MIF"], - institutions=["University of Technology Sydney"], - contact="https://forum.image.sc/tag/omero" + authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], + institutions=["University of Technology Sydney", "CAi HHU"], + contact="https://forum.image.sc/tag/omero", + version="2.0.0" ) try: - script_params = {} + script_params = { + "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION + } for key in client.getInputKeys(): if client.getInput(key): # unwrap rtypes to String, Integer etc @@ -136,19 +159,7 @@ def get_objects(conn, script_params): conn = BlitzGateway(client_obj=client) # do the editing... - objs = get_objects(conn, script_params) - - nfailed = 0 - for obj in objs: - print("Processing object:", obj) - ret = remove_map_annotations(conn, obj) - nfailed = nfailed + ret - - # now handle the result, displaying message and returning image if - # appropriate - nobjs = len(objs) - message = "Key value data deleted from {} of {} objects".format( - nobjs-nfailed, nobjs) + message = remove_keyvalue(conn, script_params) client.setOutput("Message", rstring(message)) finally: From 2ab42d961785be27e632232f8149a5b89adf0b40 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 14:07:04 +0200 Subject: [PATCH 014/127] Checkbox to confirm consequences --- omero/annotation_scripts/Remove_KeyVal.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index bf5f0c170..85b424a78 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -110,6 +110,8 @@ def remove_keyvalue(conn, script_params): rstring("Screen"), rstring("Plate"), rstring("Well"), rstring("Image")] + agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" + # Here we define the script name and description. # Good practice to put url here to give users more guidance on how to run # your script. @@ -130,14 +132,18 @@ def remove_keyvalue(conn, script_params): description="List of source IDs").ofType(rlong(0)), scripts.String( - "Target_object_type", optional=False, grouping="2", - description="Choose the object type to delete annotation from", - values=data_types, default="Image"), + "Target_object_type", optional=True, grouping="1.2", + description="Choose the object type to delete annotation from (if empty, same as source)", + values=[rstring("")]+data_types, default=""), scripts.String( - "Namespace (leave blank for default)", optional=True, grouping="3", + "Namespace (leave blank for default)", optional=True, grouping="2", description="Choose a namespace for the annotations"), + scripts.Bool( + agreement, optional=False, grouping="3", + description="Make sure that you understood what this script does"), + authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", @@ -152,6 +158,10 @@ def remove_keyvalue(conn, script_params): if client.getInput(key): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) + if script_params["Target_object_type"] == "": + script_params["Target_object_type"] = script_params["Source_object_type"] + + assert script_params[agreement], "Please confirm that you understood the risks." print(script_params) # handy to have inputs in the std-out log @@ -161,6 +171,8 @@ def remove_keyvalue(conn, script_params): # do the editing... message = remove_keyvalue(conn, script_params) client.setOutput("Message", rstring(message)) - + except AssertionError as err: #Display assertion errors in OMERO.web activities + client.setOutput("ERROR", rstring(err)) + raise AssertionError(str(err)) finally: client.closeSession() From a1e01d28fb6ba1d4da44a5ae89d2e509d58b1567 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 14:29:30 +0200 Subject: [PATCH 015/127] special name handling for wells --- omero/annotation_scripts/KeyVal_from_csv.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 6f7097a94..edc44a9c2 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -174,8 +174,12 @@ def keyval_from_csv(conn, script_params): idx_id = idx_name target_d = dict() for target_obj in target_obj_l: - assert target_obj.getName() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getName()}" - target_d[target_obj.getName()] = target_obj + if target_type == "Well": + assert target_obj.getWellPos() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getWellPos()}" + target_d[target_obj.getWellPos()] = target_obj + else: + assert target_obj.getName() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getName()}" + target_d[target_obj.getName()] = target_obj else: # Setting the dictionnary target_id:target_obj keys as string to match CSV reader output target_d = {str(target_obj.getId()):target_obj for target_obj in target_obj_l} ntarget_processed += len(target_d) From e5f880fcf9551ea6fee1b2cb93f22b399cef0acf Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 14:30:06 +0200 Subject: [PATCH 016/127] fixed message --- omero/annotation_scripts/Remove_KeyVal.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 85b424a78..ba4730d95 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -45,15 +45,15 @@ def remove_map_annotations(conn, obj, namespace): mapann_ids = [ann.id for ann in anns if isinstance(ann, omero.gateway.MapAnnotationWrapper)] if len(mapann_ids) == 0: - return 0 + return 1 print("Map Annotation IDs to delete:", mapann_ids) try: conn.deleteObjects("Annotation", mapann_ids) - return 0 + return 1 except Exception: print("Failed to delete links") - return 1 + return 0 def get_children_recursive(source_object, target_type): @@ -77,13 +77,13 @@ def remove_keyvalue(conn, script_params): source_ids = script_params["Source_IDs"] namespace = script_params["Namespace (leave blank for default)"] - nfailed = 0 + nsuccess = 0 ntotal = 0 if source_type == target_type: # We remove annotation to the given object ID for source_object in conn.getObjects(source_type, source_ids): print("Processing object:", source_object) ret = remove_map_annotations(conn, source_object, namespace) - nfailed = nfailed + ret + nsuccess += ret ntotal += 1 else: for source_object in conn.getObjects(source_type, source_ids): @@ -92,10 +92,10 @@ def remove_keyvalue(conn, script_params): for target_obj in target_obj_l: print("Processing object:", target_obj) ret = remove_map_annotations(conn, target_obj, namespace) - nfailed = nfailed + ret + nsuccess += ret ntotal += 1 - message = f"Key value data deleted from {ntotal-nfailed} of {ntotal} objects" + message = f"Key value data deleted from {ntotal-nsuccess} of {ntotal} objects" return message From d48f685dec70045fb7af901f6bd55d0b9002a613 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 14:38:30 +0200 Subject: [PATCH 017/127] uppercasing well names to match OMERO style --- omero/annotation_scripts/KeyVal_from_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index edc44a9c2..99fce6d01 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -175,8 +175,8 @@ def keyval_from_csv(conn, script_params): target_d = dict() for target_obj in target_obj_l: if target_type == "Well": - assert target_obj.getWellPos() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getWellPos()}" - target_d[target_obj.getWellPos()] = target_obj + assert target_obj.getWellPos().upper() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getWellPos()}" + target_d[target_obj.getWellPos().upper()] = target_obj else: assert target_obj.getName() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getName()}" target_d[target_obj.getName()] = target_obj From 12cb5a4504ca587a60a63693685890203dca293d Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 17:55:00 +0200 Subject: [PATCH 018/127] typo fix --- omero/annotation_scripts/Remove_KeyVal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index ba4730d95..58aa979a2 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -79,7 +79,7 @@ def remove_keyvalue(conn, script_params): nsuccess = 0 ntotal = 0 - if source_type == target_type: # We remove annotation to the given object ID + if source_type == target_type: # We remove annotation to the given objects ID for source_object in conn.getObjects(source_type, source_ids): print("Processing object:", source_object) ret = remove_map_annotations(conn, source_object, namespace) From 66d1f9edd5d4da7969cbc4aeebadd063048f97d4 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 24 Oct 2023 17:55:48 +0200 Subject: [PATCH 019/127] Initial refactoring of the csv export --- omero/annotation_scripts/KeyVal_to_csv.py | 299 +++++++++++++--------- 1 file changed, 185 insertions(+), 114 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index fbab07b12..00f64c0a6 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -30,72 +30,90 @@ from omero.cmd import Delete2 import tempfile - import os - from collections import OrderedDict +HIERARCHY_OBJECTS = { + "Project": ["Dataset", "Image"], + "Dataset": ["Image"], + "Screen": ["Plate", "Well", "Image"], + "Plate": ["Well", "Image"], + #"Run": ["Well", "Image"], + "Well": ["Image"] + } + +ZERO_PADDING = 3 # To allow duplicated keys (3 means up to 1000 duplicate key on a single object) + +def get_existing_map_annotions(obj, namespace, zero_padding): + key_l = [] + result = OrderedDict() + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + for (k,v) in ann.getValue(): + n_occurence = key_l.count(k) + result[f"{str(n_occurence).rjust(zero_padding, '0')}{k}"] = v + key_l.append(k) # To count the multiple occurence of keys + return result + +def group_keyvalue_dictionaries(annotation_dicts, zero_padding): + """ Groups the keys and values of each object into a single dictionary """ + all_key = OrderedDict() # To keep the keys in order, for what it's worth + for annotation_dict in annotation_dicts: + all_key.update({k:None for k in annotation_dict.keys()}) + all_key = list(all_key.keys()) + + result = [] + for annotation_dict in annotation_dicts: + obj_dict = OrderedDict((k, "") for k in all_key) + obj_dict.update(annotation_dict) + for k,v in obj_dict.items(): + if v is None: + obj_dict[k] + result.append(list(obj_dict.values())) + + all_key = [key[zero_padding:] for key in all_key] # Removing temporary padding + return all_key, result + +def get_children_recursive(source_object, target_type): + if HIERARCHY_OBJECTS[source_object.OMERO_CLASS][0] == target_type: # Stop condition, we return the source_obj children + return source_object.listChildren() + else: + result = [] + for child_obj in source_object.listChildren(): + # Going down in the Hierarchy list for all childs that aren't yet the target + result.extend(get_children_recursive(child_obj, target_type)) + return result + +def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, annotation_dicts, separator): + def to_csv(ll): + """convience function to write a csv line""" + nl = len(ll) + fmstr = ("{}"+separator+" ")*(nl-1)+"{}\n" + return fmstr.format(*ll) -def get_existing_map_annotions(obj): - ord_dict = OrderedDict() - for ann in obj.listAnnotations(): - if(isinstance(ann, omero.gateway.MapAnnotationWrapper)): - kvs = ann.getValue() - for k, v in kvs: - if k not in ord_dict: - ord_dict[k] = set() - ord_dict[k].add(v) - return ord_dict - + all_key, whole_values_l = group_keyvalue_dictionaries(annotation_dicts, ZERO_PADDING) + all_key.insert(0, "target_id") + all_key.insert(1, "target_name") + for (obj_id, obj_name, whole_values) in zip(obj_id_l, obj_name_l, whole_values_l): + whole_values.insert(0, obj_id) + whole_values.insert(1, obj_name) -def attach_csv_file(conn, obj, data): - ''' writes the data (list of dicts) to a file - and attaches it to the object - conn : connection to OMERO (need to annotation creation - obj : the object to attach the file file to - data : the data - ''' # create the tmp directory tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) tfile = os.fdopen(fd, 'w') - - # get the list of keys and maximum number of occurences - # A key can appear multiple times, for example multiple dyes can be used - key_union = OrderedDict() - for img_n, img_kv in data.items(): - for key, vset in img_kv.items(): - key_union[key] = max(key_union.get(key, 0), len(vset)) - - # convience function to write a csv line - def to_csv(ll): - nl = len(ll) - fmstr = "{}, "*(nl-1)+"{}\n" - return fmstr.format(*ll) - - # construct the header of the CSV file - header = ['filename'] - for key, count in key_union.items(): - header.extend([key] * count) # keys can repeat multiple times - tfile.write(to_csv(header)) - + tfile.write(to_csv(all_key)) # write the keys values for each file - for filename, kv_dict in data.items(): - row = [""] * len(header) # empty row - row[0] = filename - for key, vset in kv_dict.items(): - n0 = header.index(key) # first occurence of key in header - for i, val in enumerate(vset): - row[n0 + i] = val - tfile.write(to_csv(row)) + for whole_values in whole_values_l: + tfile.write(to_csv(whole_values)) tfile.close() - name = "{}_metadata_out.csv".format(obj.getName()) + name = "{}_metadata_out.csv".format(source_object.getName()) # link it to the object ann = conn.createFileAnnfromLocalFile( tmp_file, origFilePathAndName=name, ns='MIF_test') - ann = obj.linkAnnotation(ann) + ann = source_object.linkAnnotation(ann) # remove the tmp file os.remove(tmp_file) @@ -103,89 +121,142 @@ def to_csv(ll): return "done" -def run_script(): +def main_loop(conn, script_params): + ''' writes the data (list of dicts) to a file + @param conn: Blitz Gateway connection wrapper + @param script_params: A map of the input parameters + ''' + source_type = script_params["Source_object_type"] + target_type = script_params["Target_object_type"] + source_ids = script_params["Source_IDs"] + namespace = script_params["Namespace (leave blank for default)"] + separator = script_params["Separator"] + + # One file output per given ID + for source_object in conn.getObjects(source_type, source_ids): + print("Processing object:", source_object) + if source_type == target_type: + annotation_dicts = [get_existing_map_annotions(source_object, namespace, ZERO_PADDING)] + obj_id_l = [source_object.getId()] + obj_name_l = [source_object.getWellPos() if source_object.OMERO_CLASS is "Well" else source_object.getName()] + + else: + annotation_dicts = [] + obj_id_l, obj_name_l = [], [] + # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) + for target_obj in get_children_recursive(source_object, target_type): + print("Processing object:", target_obj) + annotation_dicts.append(get_existing_map_annotions(target_obj, namespace, ZERO_PADDING)) + obj_id_l.append(target_obj.getId()) + obj_name_l.append(target_obj.getWellPos() if target_obj.OMERO_CLASS is "Well" else target_obj.getName()) + + mess = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, annotation_dicts, separator) + print(mess) + + # for ds in datasets: + # # name of the file + # csv_name = "{}_metadata_out.csv".format(ds.getName()) + # print(csv_name) + + # # remove the csv if it exists + # for ann in ds.listAnnotations(): + # if(isinstance(ann, omero.gateway.FileAnnotationWrapper)): + # if(ann.getFileName() == csv_name): + # # if the name matches delete it + # try: + # delete = Delete2( + # targetObjects={'FileAnnotation': + # [ann.getId()]}) + # handle = conn.c.sf.submit(delete) + # conn.c.waitOnCmd( + # handle, loops=10, + # ms=500, failonerror=True, + # failontimeout=False, closehandle=False) + # print("Deleted existing csv") + # except Exception as ex: + # print("Failed to delete existing csv: {}".format( + # ex.message)) + # else: + # print("No exisiting file") + + # assemble the metadata into an OrderedDict - data_types = [rstring('Dataset')] +def run_script(): + """ + The main entry point of the script, as called by the client via the + scripting service, passing the required parameters. + """ + + data_types = [rstring("Project"), rstring("Dataset"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Image")] + + agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" + separators = [";", ","] + # Here we define the script name and description. + # Good practice to put url here to give users more guidance on how to run + # your script. client = scripts.client( - 'Create_Metadata_csv', - """ - This script reads the metadata attached data set and creates - a csv file attached to the Dataset - """, + 'KeyVal_to_csv.py', + ("Export key-value pairs of targets to .csv file" + " \nSee" + " http://www.openmicroscopy.org/site/support/omero5.2/developers/" + "scripts/user-guide.html for the tutorial that uses this script."), + scripts.String( - "Data_Type", optional=False, grouping="1", - description="Choose source of images", - values=data_types, default="Dataset"), + "Source_object_type", optional=False, grouping="1", + description="Choose the object type containing the objects to delete annotation from", + values=data_types, default="Image"), scripts.List( - "IDs", optional=False, grouping="2", - description="Plate or Screen ID.").ofType(rlong(0)), + "Source_IDs", optional=False, grouping="1.1", + description="List of source IDs").ofType(rlong(0)), + scripts.String( + "Target_object_type", optional=True, grouping="1.2", + description="Choose the object type to delete annotation from (if empty, same as source)", + values=[rstring("")]+data_types, default=""), - authors=["Christian Evenhuis"], - institutions=["MIF UTS"], - contact="https://forum.image.sc/tag/omero" + scripts.String( + "Namespace (leave blank for default)", optional=True, grouping="2", + description="Choose a namespace for the annotations"), + + scripts.String( + "Separator", optional=False, grouping="3", + description="Choose the .csv separator", + values=separators, default=";"), + + authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], + institutions=["University of Technology Sydney", "CAi HHU"], + contact="https://forum.image.sc/tag/omero", + version="2.0.0" ) try: - # process the list of args above. - script_params = {} + script_params = { + "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION + } for key in client.getInputKeys(): if client.getInput(key): + # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) + if script_params["Target_object_type"] == "": + script_params["Target_object_type"] = script_params["Source_object_type"] + + print(script_params) # handy to have inputs in the std-out log # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - print("connection made") - - data_type = script_params["Data_Type"] - print(data_type) - ids = script_params["IDs"] - datasets = list(conn.getObjects(data_type, ids)) - print(ids) - print("datasets:") - print(datasets) - for ds in datasets: - # name of the file - csv_name = "{}_metadata_out.csv".format(ds.getName()) - print(csv_name) - - # remove the csv if it exists - for ann in ds.listAnnotations(): - if(isinstance(ann, omero.gateway.FileAnnotationWrapper)): - if(ann.getFileName() == csv_name): - # if the name matches delete it - try: - delete = Delete2( - targetObjects={'FileAnnotation': - [ann.getId()]}) - handle = conn.c.sf.submit(delete) - conn.c.waitOnCmd( - handle, loops=10, - ms=500, failonerror=True, - failontimeout=False, closehandle=False) - print("Deleted existing csv") - except Exception as ex: - print("Failed to delete existing csv: {}".format( - ex.message)) - else: - print("No exisiting file") - # assemble the metadata into an OrderedDict - kv_dict = OrderedDict() - for img in ds.listChildren(): - fn = img.getName() - kv_dict[fn] = get_existing_map_annotions(img) - - # attach the data - mess = attach_csv_file(conn, ds, kv_dict) - print(mess) - mess = "done" - client.setOutput("Message", rstring(mess)) + # do the editing... + message = main_loop(conn, script_params) + client.setOutput("Message", rstring(message)) + except AssertionError as err: #Display assertion errors in OMERO.web activities + client.setOutput("ERROR", rstring(err)) + raise AssertionError(str(err)) finally: client.closeSession() - if __name__ == "__main__": - run_script() + run_script() \ No newline at end of file From b746e0ddcaef62ed279dd493ffcde53df32ef94b Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Tue, 24 Oct 2023 19:45:08 +0200 Subject: [PATCH 020/127] Change user input 'same as source' for target --- omero/annotation_scripts/KeyVal_to_csv.py | 6 +++--- omero/annotation_scripts/Remove_KeyVal.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 00f64c0a6..c31a35df6 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -214,8 +214,8 @@ def run_script(): scripts.String( "Target_object_type", optional=True, grouping="1.2", - description="Choose the object type to delete annotation from (if empty, same as source)", - values=[rstring("")]+data_types, default=""), + description="Choose the object type to delete annotation from", + values=[rstring("")]+data_types, default=""), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="2", @@ -240,7 +240,7 @@ def run_script(): if client.getInput(key): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) - if script_params["Target_object_type"] == "": + if script_params["Target_object_type"] == "": script_params["Target_object_type"] = script_params["Source_object_type"] print(script_params) # handy to have inputs in the std-out log diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 58aa979a2..7a031f1e3 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -133,8 +133,8 @@ def remove_keyvalue(conn, script_params): scripts.String( "Target_object_type", optional=True, grouping="1.2", - description="Choose the object type to delete annotation from (if empty, same as source)", - values=[rstring("")]+data_types, default=""), + description="Choose the object type to delete annotation from.", + values=[rstring("")]+data_types, default=""), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="2", @@ -158,7 +158,7 @@ def remove_keyvalue(conn, script_params): if client.getInput(key): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) - if script_params["Target_object_type"] == "": + if script_params["Target_object_type"] == "": script_params["Target_object_type"] = script_params["Source_object_type"] assert script_params[agreement], "Please confirm that you understood the risks." From b6815fac5af36d2951fadc7a9db3cf7c61f18077 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Tue, 24 Oct 2023 23:07:07 +0200 Subject: [PATCH 021/127] Support of tags as source for delete or export --- omero/annotation_scripts/KeyVal_from_csv.py | 2 +- omero/annotation_scripts/KeyVal_to_csv.py | 21 ++++++++++++----- omero/annotation_scripts/Remove_KeyVal.py | 25 +++++++++++++-------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 99fce6d01..a0fcb2dcc 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -129,7 +129,7 @@ def read_csv(conn, original_file): #Dedicated function to read the CSV file data = list(csv.reader(file_handle, delimiter=delimiter)) # keys are in the header row - header = data[0] + header = [el.strip() for el in data[0]] print("header", header) return data, header diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index c31a35df6..98b03830e 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -108,7 +108,8 @@ def to_csv(ll): tfile.write(to_csv(whole_values)) tfile.close() - name = "{}_metadata_out.csv".format(source_object.getName()) + source_name = source_object.getWellPos() if source_object.OMERO_CLASS is "Well" else source_object.getName() + name = "{}_metadata_out.csv".format(source_name) # link it to the object ann = conn.createFileAnnfromLocalFile( tmp_file, origFilePathAndName=name, @@ -134,17 +135,22 @@ def main_loop(conn, script_params): # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): - print("Processing object:", source_object) if source_type == target_type: + print("Processing object:", source_object) annotation_dicts = [get_existing_map_annotions(source_object, namespace, ZERO_PADDING)] obj_id_l = [source_object.getId()] obj_name_l = [source_object.getWellPos() if source_object.OMERO_CLASS is "Well" else source_object.getName()] - else: annotation_dicts = [] obj_id_l, obj_name_l = [], [] + if source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) + target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later + source_object = target_obj_l[0] # Putting the csv file on the first child + else: + target_obj_l = get_children_recursive(source_object, target_type) # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - for target_obj in get_children_recursive(source_object, target_type): + for target_obj in target_obj_l: print("Processing object:", target_obj) annotation_dicts.append(get_existing_map_annotions(target_obj, namespace, ZERO_PADDING)) obj_id_l.append(target_obj.getId()) @@ -206,7 +212,7 @@ def run_script(): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to delete annotation from", - values=data_types, default="Image"), + values=data_types+[rstring("Tag")], default="Image"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -240,6 +246,11 @@ def run_script(): if client.getInput(key): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) + + if script_params["Source_object_type"] == "Tag": + script_params["Source_object_type"] = "TagAnnotation" + assert script_params["Target_object_type"] != "", "Tag as source is not compatible with target ''" + if script_params["Target_object_type"] == "": script_params["Target_object_type"] = script_params["Source_object_type"] diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 7a031f1e3..a8a548a87 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -44,9 +44,9 @@ def remove_map_annotations(conn, obj, namespace): anns = list(obj.listAnnotations(ns=namespace)) mapann_ids = [ann.id for ann in anns if isinstance(ann, omero.gateway.MapAnnotationWrapper)] - if len(mapann_ids) == 0: - return 1 + if len(mapann_ids) == 0: + return 0 print("Map Annotation IDs to delete:", mapann_ids) try: conn.deleteObjects("Annotation", mapann_ids) @@ -79,23 +79,26 @@ def remove_keyvalue(conn, script_params): nsuccess = 0 ntotal = 0 - if source_type == target_type: # We remove annotation to the given objects ID - for source_object in conn.getObjects(source_type, source_ids): + for source_object in conn.getObjects(source_type, source_ids): + if source_type == target_type: print("Processing object:", source_object) ret = remove_map_annotations(conn, source_object, namespace) nsuccess += ret ntotal += 1 - else: - for source_object in conn.getObjects(source_type, source_ids): + else: # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - target_obj_l = get_children_recursive(source_object, target_type) + if source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) + target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later + else: + target_obj_l = get_children_recursive(source_object, target_type) for target_obj in target_obj_l: print("Processing object:", target_obj) ret = remove_map_annotations(conn, target_obj, namespace) nsuccess += ret ntotal += 1 - message = f"Key value data deleted from {ntotal-nsuccess} of {ntotal} objects" + message = f"Key value data deleted from {nsuccess} of {ntotal} objects" return message @@ -125,7 +128,7 @@ def remove_keyvalue(conn, script_params): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to delete annotation from", - values=data_types, default="Image"), + values=data_types+[rstring("Tag")], default="Image"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -158,6 +161,10 @@ def remove_keyvalue(conn, script_params): if client.getInput(key): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) + if script_params["Source_object_type"] == "Tag": + script_params["Source_object_type"] = "TagAnnotation" + assert script_params["Target_object_type"] != "", "Tag as source is not compatible with target ''" + if script_params["Target_object_type"] == "": script_params["Target_object_type"] = script_params["Source_object_type"] From 3a97d45ba6fdb1c8d0297b34e9e9952a4e7407a0 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 11:16:16 +0200 Subject: [PATCH 022/127] Added supprt of ann from Tag as source and on source directly --- omero/annotation_scripts/KeyVal_from_csv.py | 64 +++++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index a0fcb2dcc..9eb4212a8 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -54,23 +54,20 @@ "Well": ["Image"] } -def get_original_file(omero_object, file_ann_id=None): - """Find file linked to object. Option to filter by ID.""" +def get_original_file(omero_object): + """Find last AnnotationFile linked to object""" file_ann = None for ann in omero_object.listAnnotations(): - if isinstance(ann, omero.gateway.FileAnnotationWrapper): + if ann.OMERO_TYPE == omero.model.FileAnnotationI: file_name = ann.getFile().getName() # Pick file by Ann ID (or name if ID is None) - if ann.getId() == file_ann_id: - file_ann = ann # Found it - break - elif file_ann_id is None and file_name.endswith(".csv"): + if file_name.endswith(".csv"): if (file_ann is None) or (ann.getDate() > file_ann.getDate()): # Get the most recent file file_ann = ann - if file_ann is None: - sys.stderr.write("Error: File does not exist.\n") - sys.exit(1) + + obj_name = omero_object.getWellPos() if omero_object.OMERO_CLASS is "Well" else omero_object.getName() + assert file_ann is not None, f"No .csv FileAnnotation was found on {omero_object.OMERO_CLASS}:{obj_name}:{omero_object.getId()}" return file_ann @@ -154,16 +151,29 @@ def keyval_from_csv(conn, script_params): ntarget_updated = 0 missing_names = 0 + # One file output per given ID for source_object, file_ann_id in zip(conn.getObjects(source_type, source_ids), file_ids): - if file_ann_id is not None: # If the file ID is not defined, only already linked file will be used - link_file_ann(conn, source_type, source_object.id, file_ann_id) - file_ann = get_original_file(source_object, file_ann_id) + #if file_ann_id is not None: # If the file ID is not defined, only already linked file will be used + # link_file_ann(conn, source_type, source_object.id, file_ann_id) # TODO do we want to keep that linking? + if file_ann_id is not None: + file_ann = conn.getObject("Annotation", oid=file_ann_id) + assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, "The provided annotation ID must reference a FileAnnotation, not a {file_ann.OMERO_TYPE}" + else: + file_ann = get_original_file(source_object, file_ann_id) original_file = file_ann.getFile()._obj print("set ann id", file_ann.getId()) data, header = read_csv(conn, original_file) - # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - target_obj_l = get_children_recursive(source_object, target_type) + if source_type == target_type: + print("Processing object:", source_object) + target_obj_l = [source_object] + else: + if source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) + target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later + else: + target_obj_l = get_children_recursive(source_object, target_type) + # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) # Finds the index of the column used to identify the targets. Try for IDs first idx_id = header.index("target_id") if "target_id" in header else -1 @@ -189,10 +199,8 @@ def keyval_from_csv(conn, script_params): target_id = row[idx_id] if target_id in target_d.keys(): target_obj = target_d[target_id] - if target_type in ["Dataset", "Image", "Plate"]: - print("Annotating Target:", f"{target_obj.getName()+':' if use_id else ''}{target_id}") - else: - print("Annotating Target:", f"{target_id}") # Some object don't have a name + obj_name = target_obj.getWellPos() if target_obj.OMERO_CLASS is "Well" else target_obj.getName() + print("Annotating Target:", f"{obj_name+':' if use_id else ''}{target_id}") else: missing_names += 1 print(f"Target not found: {target_id}") @@ -236,10 +244,12 @@ def run_script(): source_types = [rstring("Project"), rstring("Dataset"), rstring("Screen"), rstring("Plate"), - rstring("Well")] + rstring("Well"), rstring("Image"), + rstring("Tag")] - target_types = [rstring("Dataset"), rstring("Plate"), - rstring("Well"), rstring("Image")] + target_types = [rstring(""), rstring("Dataset"), + rstring("Plate"), rstring("Well"), + rstring("Image")] client = scripts.client( 'Add_Key_Val_from_csv', @@ -308,9 +318,13 @@ def run_script(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) - # validate that target is bellow source - source_name, target_name = script_params["Source_object_type"], script_params["Target_object_type"] - assert target_name in HIERARCHY_OBJECTS[source_name], f"Invalid {source_name} => {target_name}. The target type must be a child of the source type" + if script_params["Source_object_type"] == "Tag": + script_params["Source_object_type"] = "TagAnnotation" + assert script_params["Target_object_type"] != "", "Tag as source is not compatible with target ''" + assert None not in script_params["File_Annotation_ID"], "File annotation ID must be given when using Tag as source" + + if script_params["Target_object_type"] == "": + script_params["Target_object_type"] = script_params["Source_object_type"] if len(script_params["File_Annotation_ID"]) == 1: # Poulate the parameter with None or same ID for all source script_params["File_Annotation_ID"] = script_params["File_Annotation_ID"] * len(script_params["Source_IDs"]) From f2407b23942f56770553686e96da5d05fb79990b Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 11:16:37 +0200 Subject: [PATCH 023/127] minor rearangements --- omero/annotation_scripts/KeyVal_to_csv.py | 15 ++++++++++----- omero/annotation_scripts/Remove_KeyVal.py | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 98b03830e..1de8a657d 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -193,9 +193,14 @@ def run_script(): scripting service, passing the required parameters. """ - data_types = [rstring("Project"), rstring("Dataset"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image")] + source_types = [rstring("Project"), rstring("Dataset"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Image"), + rstring("Tag")] + + target_types = [rstring(""), rstring("Dataset"), + rstring("Plate"), rstring("Well"), + rstring("Image")] agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" separators = [";", ","] @@ -212,7 +217,7 @@ def run_script(): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to delete annotation from", - values=data_types+[rstring("Tag")], default="Image"), + values=source_types, default="Image"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -221,7 +226,7 @@ def run_script(): scripts.String( "Target_object_type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from", - values=[rstring("")]+data_types, default=""), + values=target_types, default=""), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="2", diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index a8a548a87..fa5a14eb5 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -109,9 +109,14 @@ def remove_keyvalue(conn, script_params): scripting service, passing the required parameters. """ - data_types = [rstring("Project"), rstring("Dataset"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image")] + source_types = [rstring("Project"), rstring("Dataset"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Image"), + rstring("Tag")] + + target_types = [rstring(""), rstring("Dataset"), + rstring("Plate"), rstring("Well"), + rstring("Image")] agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" @@ -128,7 +133,7 @@ def remove_keyvalue(conn, script_params): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to delete annotation from", - values=data_types+[rstring("Tag")], default="Image"), + values=source_types, default="Image"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -137,7 +142,7 @@ def remove_keyvalue(conn, script_params): scripts.String( "Target_object_type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", - values=[rstring("")]+data_types, default=""), + values=target_types, default=""), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="2", From 50e34020c76337034962c7e2e2b82bd67edf3fdd Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 11:38:19 +0200 Subject: [PATCH 024/127] Reworking the UI --- omero/annotation_scripts/KeyVal_from_csv.py | 31 +++++++++++---------- omero/annotation_scripts/KeyVal_to_csv.py | 26 +++++++---------- omero/annotation_scripts/Remove_KeyVal.py | 25 +++++++---------- 3 files changed, 37 insertions(+), 45 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 9eb4212a8..ec0fcca2e 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -242,14 +242,15 @@ def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): def run_script(): - source_types = [rstring("Project"), rstring("Dataset"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), + source_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image"), #Duplicate Image for UI, but not a problem for script rstring("Tag")] - target_types = [rstring(""), rstring("Dataset"), - rstring("Plate"), rstring("Well"), - rstring("Image")] + target_types = [rstring("Project"), + rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image")] client = scripts.client( 'Add_Key_Val_from_csv', @@ -282,7 +283,7 @@ def run_script(): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to annotate", - values=source_types, default="Dataset"), + values=source_types, default="- Dataset"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -293,12 +294,12 @@ def run_script(): description="List of file IDs containing metadata to populate. If given, must match length of 'Source IDs'. Otherwise, uses the CSV file with the highest ID.").ofType(rlong(0)), scripts.String( - "Target_object_type", optional=False, grouping="2", + "Target_object_type", optional=False, grouping="1.3", description="Choose the object type to annotate (must be bellow the chosen source object type)", - values=target_types, default="Image"), + values=target_types, default="-- Image"), scripts.String( - "Namespace (leave blank for default)", optional=True, grouping="3", + "Namespace (leave blank for default)", optional=True, grouping="1.4", description="Choose a namespace for the annotations"), authors=["Christian Evenhuis", "Tom Boissonnet"], @@ -318,14 +319,16 @@ def run_script(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) + # Getting rid of the trailing '---' added for the UI + tmp_src = script_params["Source_object_type"] + script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src + tmp_trg = script_params["Target_object_type"] + script_params["Source_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + if script_params["Source_object_type"] == "Tag": script_params["Source_object_type"] = "TagAnnotation" - assert script_params["Target_object_type"] != "", "Tag as source is not compatible with target ''" assert None not in script_params["File_Annotation_ID"], "File annotation ID must be given when using Tag as source" - if script_params["Target_object_type"] == "": - script_params["Target_object_type"] = script_params["Source_object_type"] - if len(script_params["File_Annotation_ID"]) == 1: # Poulate the parameter with None or same ID for all source script_params["File_Annotation_ID"] = script_params["File_Annotation_ID"] * len(script_params["Source_IDs"]) assert len(script_params["File_Annotation_ID"]) == len(script_params["Source_IDs"]), "Number of Source IDs and FileAnnotation IDs must match" diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 1de8a657d..097fd93aa 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -193,14 +193,15 @@ def run_script(): scripting service, passing the required parameters. """ - source_types = [rstring("Project"), rstring("Dataset"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), + source_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image"), #Duplicate Image for UI, but not a problem for script rstring("Tag")] - target_types = [rstring(""), rstring("Dataset"), - rstring("Plate"), rstring("Well"), - rstring("Image")] + target_types = [rstring("Project"), + rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image")] agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" separators = [";", ","] @@ -217,7 +218,7 @@ def run_script(): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to delete annotation from", - values=source_types, default="Image"), + values=source_types, default="- Dataset"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -226,10 +227,10 @@ def run_script(): scripts.String( "Target_object_type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from", - values=target_types, default=""), + values=target_types, default="-- Image"), scripts.String( - "Namespace (leave blank for default)", optional=True, grouping="2", + "Namespace (leave blank for default)", optional=True, grouping="1.3", description="Choose a namespace for the annotations"), scripts.String( @@ -252,13 +253,6 @@ def run_script(): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) - if script_params["Source_object_type"] == "Tag": - script_params["Source_object_type"] = "TagAnnotation" - assert script_params["Target_object_type"] != "", "Tag as source is not compatible with target ''" - - if script_params["Target_object_type"] == "": - script_params["Target_object_type"] = script_params["Source_object_type"] - print(script_params) # handy to have inputs in the std-out log # wrap client to use the Blitz Gateway diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index fa5a14eb5..63884e5ff 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -109,14 +109,15 @@ def remove_keyvalue(conn, script_params): scripting service, passing the required parameters. """ - source_types = [rstring("Project"), rstring("Dataset"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), + source_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image"), #Duplicate Image for UI, but not a problem for script rstring("Tag")] - target_types = [rstring(""), rstring("Dataset"), - rstring("Plate"), rstring("Well"), - rstring("Image")] + target_types = [rstring("Project"), + rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image")] agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" @@ -133,7 +134,7 @@ def remove_keyvalue(conn, script_params): scripts.String( "Source_object_type", optional=False, grouping="1", description="Choose the object type containing the objects to delete annotation from", - values=source_types, default="Image"), + values=source_types, default="- Dataset"), scripts.List( "Source_IDs", optional=False, grouping="1.1", @@ -142,10 +143,10 @@ def remove_keyvalue(conn, script_params): scripts.String( "Target_object_type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", - values=target_types, default=""), + values=target_types, default="-- Image"), scripts.String( - "Namespace (leave blank for default)", optional=True, grouping="2", + "Namespace (leave blank for default)", optional=True, grouping="1.3", description="Choose a namespace for the annotations"), scripts.Bool( @@ -166,12 +167,6 @@ def remove_keyvalue(conn, script_params): if client.getInput(key): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) - if script_params["Source_object_type"] == "Tag": - script_params["Source_object_type"] = "TagAnnotation" - assert script_params["Target_object_type"] != "", "Tag as source is not compatible with target ''" - - if script_params["Target_object_type"] == "": - script_params["Target_object_type"] = script_params["Source_object_type"] assert script_params[agreement], "Please confirm that you understood the risks." From 90f1fbc939fe6b9438ce331198572fb81fde16ca Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 14:44:13 +0200 Subject: [PATCH 025/127] Added option to add ancestry as columns --- omero/annotation_scripts/KeyVal_to_csv.py | 68 +++++++++++++++++------ 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 097fd93aa..1c662239a 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -33,13 +33,14 @@ import os from collections import OrderedDict -HIERARCHY_OBJECTS = { - "Project": ["Dataset", "Image"], - "Dataset": ["Image"], - "Screen": ["Plate", "Well", "Image"], - "Plate": ["Well", "Image"], +CHILD_OBJECTS = { + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", #"Run": ["Well", "Image"], - "Well": ["Image"] + "Well": "WellSample", + "WellSample": "Image" } ZERO_PADDING = 3 # To allow duplicated keys (3 means up to 1000 duplicate key on a single object) @@ -75,8 +76,11 @@ def group_keyvalue_dictionaries(annotation_dicts, zero_padding): return all_key, result def get_children_recursive(source_object, target_type): - if HIERARCHY_OBJECTS[source_object.OMERO_CLASS][0] == target_type: # Stop condition, we return the source_obj children - return source_object.listChildren() + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if source_object.OMERO_CLASS != "WellSample": + return source_object.listChildren() + else: + return [source_object.getImage()] else: result = [] for child_obj in source_object.listChildren(): @@ -84,7 +88,7 @@ def get_children_recursive(source_object, target_type): result.extend(get_children_recursive(child_obj, target_type)) return result -def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, annotation_dicts, separator): +def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator): def to_csv(ll): """convience function to write a csv line""" nl = len(ll) @@ -92,11 +96,20 @@ def to_csv(ll): return fmstr.format(*ll) all_key, whole_values_l = group_keyvalue_dictionaries(annotation_dicts, ZERO_PADDING) - all_key.insert(0, "target_id") - all_key.insert(1, "target_name") - for (obj_id, obj_name, whole_values) in zip(obj_id_l, obj_name_l, whole_values_l): - whole_values.insert(0, obj_id) - whole_values.insert(1, obj_name) + + counter = 0 + if len(obj_ancestry_l)>0: # If there's anything to add at all + for (parent_type, _) in obj_ancestry_l[0]: + all_key.insert(counter, parent_type); counter += 1 + all_key.insert(counter, "target_id") + all_key.insert(counter + 1, "target_name") + for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, obj_name_l, whole_values_l)): + counter = 0 + if len(obj_ancestry_l)>0: # If there's anything to add at all + for (_, parent_name) in obj_ancestry_l[k]: + whole_values.insert(counter, parent_name); counter += 1 + whole_values.insert(counter, obj_id) + whole_values.insert(counter + 1, obj_name) # create the tmp directory tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') @@ -108,7 +121,7 @@ def to_csv(ll): tfile.write(to_csv(whole_values)) tfile.close() - source_name = source_object.getWellPos() if source_object.OMERO_CLASS is "Well" else source_object.getName() + source_name = source_object.getWellPos() if source_object.OMERO_CLASS == "Well" else source_object.getName() name = "{}_metadata_out.csv".format(source_name) # link it to the object ann = conn.createFileAnnfromLocalFile( @@ -132,31 +145,40 @@ def main_loop(conn, script_params): source_ids = script_params["Source_IDs"] namespace = script_params["Namespace (leave blank for default)"] separator = script_params["Separator"] + include_parent = script_params["Include column(s) for parent objects name"] # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): + obj_ancestry_l = [] if source_type == target_type: print("Processing object:", source_object) annotation_dicts = [get_existing_map_annotions(source_object, namespace, ZERO_PADDING)] obj_id_l = [source_object.getId()] - obj_name_l = [source_object.getWellPos() if source_object.OMERO_CLASS is "Well" else source_object.getName()] + obj_name_l = [source_object.getWellPos() if source_object.OMERO_CLASS == "Well" else source_object.getName()] else: annotation_dicts = [] obj_id_l, obj_name_l = [], [] + if source_type == "TagAnnotation": target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later source_object = target_obj_l[0] # Putting the csv file on the first child else: + print(source_object, target_type) target_obj_l = get_children_recursive(source_object, target_type) # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) for target_obj in target_obj_l: print("Processing object:", target_obj) annotation_dicts.append(get_existing_map_annotions(target_obj, namespace, ZERO_PADDING)) obj_id_l.append(target_obj.getId()) - obj_name_l.append(target_obj.getWellPos() if target_obj.OMERO_CLASS is "Well" else target_obj.getName()) + obj_name_l.append(target_obj.getWellPos() if target_obj.OMERO_CLASS == "Well" else target_obj.getName()) + if include_parent: + ancestry = [(o.OMERO_CLASS, o.getWellPos() if o.OMERO_CLASS == "Well" else o.getName()) + for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"][::-1] + obj_ancestry_l.append(ancestry) - mess = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, annotation_dicts, separator) + + mess = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator) print(mess) # for ds in datasets: @@ -238,6 +260,10 @@ def run_script(): description="Choose the .csv separator", values=separators, default=";"), + scripts.Bool( + "Include column(s) for parent objects name", optional=False, grouping="3", + description="Weather to include or not the name of the parent(s) objects as columns in the .csv", default=False), + authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", @@ -253,6 +279,12 @@ def run_script(): # unwrap rtypes to String, Integer etc script_params[key] = client.getInput(key, unwrap=True) + # Getting rid of the trailing '---' added for the UI + tmp_src = script_params["Source_object_type"] + script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src + tmp_trg = script_params["Target_object_type"] + script_params["Target_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + print(script_params) # handy to have inputs in the std-out log # wrap client to use the Blitz Gateway From 3eee5bff664a9d8e4487f8240f9895e8a2de098d Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 14:44:29 +0200 Subject: [PATCH 026/127] fixes --- omero/annotation_scripts/Remove_KeyVal.py | 26 ++++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 63884e5ff..4f965547e 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -31,13 +31,14 @@ from omero.rtypes import rlong, rstring, wrap import omero.scripts as scripts -HIERARCHY_OBJECTS = { - "Project": ["Dataset", "Image"], - "Dataset": ["Image"], - "Screen": ["Plate", "Well", "Image"], - "Plate": ["Well", "Image"], +CHILD_OBJECTS = { + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", #"Run": ["Well", "Image"], - "Well": ["Image"] + "Well": "WellSample", + "WellSample": "Image" } def remove_map_annotations(conn, obj, namespace): @@ -57,8 +58,11 @@ def remove_map_annotations(conn, obj, namespace): def get_children_recursive(source_object, target_type): - if HIERARCHY_OBJECTS[source_object.OMERO_CLASS][0] == target_type: # Stop condition, we return the source_obj children - return source_object.listChildren() + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if source_object.OMERO_CLASS != "WellSample": + return source_object.listChildren() + else: + return [source_object.getImage()] else: result = [] for child_obj in source_object.listChildren(): @@ -170,6 +174,12 @@ def remove_keyvalue(conn, script_params): assert script_params[agreement], "Please confirm that you understood the risks." + # Getting rid of the trailing '---' added for the UI + tmp_src = script_params["Source_object_type"] + script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src + tmp_trg = script_params["Target_object_type"] + script_params["Target_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + print(script_params) # handy to have inputs in the std-out log # wrap client to use the Blitz Gateway From da83ee3ad0a65c8bc6d1225e7fee18b0997924e3 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 15:35:29 +0200 Subject: [PATCH 027/127] more parameter for csv import --- omero/annotation_scripts/KeyVal_from_csv.py | 113 ++++++++++++++------ omero/annotation_scripts/KeyVal_to_csv.py | 16 +-- omero/annotation_scripts/Remove_KeyVal.py | 16 +-- 3 files changed, 94 insertions(+), 51 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index ec0fcca2e..533a2ab7b 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -45,14 +45,15 @@ from collections import OrderedDict -HIERARCHY_OBJECTS = { - "Project": ["Dataset", "Image"], - "Dataset": ["Image"], - "Screen": ["Plate", "Well", "Image"], - "Plate": ["Well", "Image"], - #"Run": ["Well", "Image"], - "Well": ["Image"] - } +CHILD_OBJECTS = { + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + #"Run": ["Well", "Image"], + "Well": "WellSample", + "WellSample": "Image" + } def get_original_file(omero_object): """Find last AnnotationFile linked to object""" @@ -66,7 +67,7 @@ def get_original_file(omero_object): # Get the most recent file file_ann = ann - obj_name = omero_object.getWellPos() if omero_object.OMERO_CLASS is "Well" else omero_object.getName() + obj_name = omero_object.getWellPos() if omero_object.OMERO_CLASS == "Well" else omero_object.getName() assert file_ann is not None, f"No .csv FileAnnotation was found on {omero_object.OMERO_CLASS}:{obj_name}:{omero_object.getId()}" return file_ann @@ -86,7 +87,7 @@ def link_file_ann(conn, object_type, object_id, file_ann_id): if len(links) == 0: omero_object.linkAnnotation(file_ann) -def read_csv(conn, original_file): #Dedicated function to read the CSV file +def read_csv(conn, original_file, delimiter): #Dedicated function to read the CSV file print("Original File", original_file.id.val, original_file.name.val) provider = DownloadingOriginalFileProvider(conn) # read the csv @@ -95,31 +96,32 @@ def read_csv(conn, original_file): #Dedicated function to read the CSV file temp_name = temp_file.name file_length = original_file.size.val with open(temp_name, 'rt', encoding='utf-8-sig') as file_handle: - try: - delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length/4)), ",;\t").delimiter - print("Using delimiter: ", delimiter, - f" after reading {floor(file_length/4)} characters") - except Exception: - file_handle.seek(0) + if delimiter is None: try: delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length/2)), - ",;\t").delimiter + file_handle.read(floor(file_length/4)), ",;\t").delimiter print("Using delimiter: ", delimiter, - f"after reading {floor(file_length/2)} characters") + f" after reading {floor(file_length/4)} characters") except Exception: file_handle.seek(0) try: delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length*0.75)), + file_handle.read(floor(file_length/2)), ",;\t").delimiter print("Using delimiter: ", delimiter, - f" after reading {floor(file_length*0.75)}" - " characters") + f"after reading {floor(file_length/2)} characters") except Exception: - print("Failed to sniff delimiter, using ','") - delimiter = "," + file_handle.seek(0) + try: + delimiter = csv.Sniffer().sniff( + file_handle.read(floor(file_length*0.75)), + ",;\t").delimiter + print("Using delimiter: ", delimiter, + f" after reading {floor(file_length*0.75)}" + " characters") + except Exception: + print("Failed to sniff delimiter, using ','") + delimiter = "," # reset to start and read whole file... file_handle.seek(0) @@ -131,8 +133,12 @@ def read_csv(conn, original_file): #Dedicated function to read the CSV file return data, header def get_children_recursive(source_object, target_type): - if HIERARCHY_OBJECTS[source_object.OMERO_CLASS][0] == target_type: # Stop condition, we return the source_obj children - return source_object.listChildren() + print(source_object, target_type) + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if source_object.OMERO_CLASS != "WellSample": + return source_object.listChildren() + else: + return [source_object.getImage()] else: result = [] for child_obj in source_object.listChildren(): @@ -146,6 +152,10 @@ def keyval_from_csv(conn, script_params): source_ids = script_params["Source_IDs"] file_ids = script_params["File_Annotation_ID"] namespace = script_params["Namespace (leave blank for default)"] + to_exclude = script_params["Columns to exclude"] + target_id_colname = script_params["Column name of the 'target ID'"] + target_name_colname = script_params["Namespace (leave blank for default)"] + separator = script_params["Separator"] ntarget_processed = 0 ntarget_updated = 0 @@ -159,10 +169,10 @@ def keyval_from_csv(conn, script_params): file_ann = conn.getObject("Annotation", oid=file_ann_id) assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, "The provided annotation ID must reference a FileAnnotation, not a {file_ann.OMERO_TYPE}" else: - file_ann = get_original_file(source_object, file_ann_id) + file_ann = get_original_file(source_object) original_file = file_ann.getFile()._obj print("set ann id", file_ann.getId()) - data, header = read_csv(conn, original_file) + data, header = read_csv(conn, original_file, separator) if source_type == target_type: print("Processing object:", source_object) @@ -176,10 +186,13 @@ def keyval_from_csv(conn, script_params): # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) # Finds the index of the column used to identify the targets. Try for IDs first - idx_id = header.index("target_id") if "target_id" in header else -1 - idx_name = header.index("target_name") if "target_name" in header else -1 - use_id = idx_id != -1 + idx_id = header.index(target_id_colname) if target_id_colname in header else -1 + idx_name = header.index(target_name_colname) if target_name_colname in header else -1 + cols_to_ignore = [header.index(el) for el in to_exclude if el in header] + + assert (idx_id != -1) or (idx_name != -1), "Neither the column for the objects name or the objects index were found" + use_id = idx_id != -1 # If the column for the object index exist, use it if not use_id: # Identify images by name must fail if two images have identical names idx_id = idx_name target_d = dict() @@ -199,14 +212,13 @@ def keyval_from_csv(conn, script_params): target_id = row[idx_id] if target_id in target_d.keys(): target_obj = target_d[target_id] - obj_name = target_obj.getWellPos() if target_obj.OMERO_CLASS is "Well" else target_obj.getName() + obj_name = target_obj.getWellPos() if target_obj.OMERO_CLASS == "Well" else target_obj.getName() print("Annotating Target:", f"{obj_name+':' if use_id else ''}{target_id}") else: missing_names += 1 print(f"Target not found: {target_id}") continue - cols_to_ignore = [idx_id, idx_name] updated = annotate_object(conn, target_obj, header, row, cols_to_ignore, namespace) if updated: ntarget_updated += 1 @@ -252,6 +264,8 @@ def run_script(): rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] + separators = ["guess", ";", ",", ""] + client = scripts.client( 'Add_Key_Val_from_csv', """ @@ -302,6 +316,23 @@ def run_script(): "Namespace (leave blank for default)", optional=True, grouping="1.4", description="Choose a namespace for the annotations"), + scripts.List( + "Columns to exclude", optional=False, grouping="2", + description="List of columns to exclude from the key-value pair import", default=",").ofType(rstring("")), + + scripts.String( + "Column name of the 'target ID'", optional=True, grouping="2.1", + description="Set the column name containing the id of the target", default="target_id"), + + scripts.String( + "Column name of the 'target name'", optional=True, grouping="2.2", + description="Set the column name containing the id of the target", default="target_name"), + + scripts.String( + "Separator", optional=False, grouping="3", + description="Choose the .csv separator", + values=separators, default="guess"), + authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero", @@ -323,7 +354,7 @@ def run_script(): tmp_src = script_params["Source_object_type"] script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src tmp_trg = script_params["Target_object_type"] - script_params["Source_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + script_params["Target_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg if script_params["Source_object_type"] == "Tag": script_params["Source_object_type"] = "TagAnnotation" @@ -333,6 +364,18 @@ def run_script(): script_params["File_Annotation_ID"] = script_params["File_Annotation_ID"] * len(script_params["Source_IDs"]) assert len(script_params["File_Annotation_ID"]) == len(script_params["Source_IDs"]), "Number of Source IDs and FileAnnotation IDs must match" + to_exclude = list(map(lambda x: x.replace('', script_params["Column name of the 'target ID'"]), + script_params["Columns to exclude"])) + script_params["Columns to exclude"] = list(map(lambda x: x.replace('', script_params["Column name of the 'target name'"]), + to_exclude)) + + if script_params["Separator"] == "guess": + script_params["Separator"] = None + elif script_params["Separator"] == "": + script_params["Separator"] = "\t" + + + # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) print("script params") diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 1c662239a..b3b31ffee 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -34,14 +34,14 @@ from collections import OrderedDict CHILD_OBJECTS = { - "Project": "Dataset", - "Dataset": "Image", - "Screen": "Plate", - "Plate": "Well", - #"Run": ["Well", "Image"], - "Well": "WellSample", - "WellSample": "Image" - } + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + #"Run": ["Well", "Image"], + "Well": "WellSample", + "WellSample": "Image" + } ZERO_PADDING = 3 # To allow duplicated keys (3 means up to 1000 duplicate key on a single object) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 4f965547e..2f7d9e0c6 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -32,14 +32,14 @@ import omero.scripts as scripts CHILD_OBJECTS = { - "Project": "Dataset", - "Dataset": "Image", - "Screen": "Plate", - "Plate": "Well", - #"Run": ["Well", "Image"], - "Well": "WellSample", - "WellSample": "Image" - } + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + #"Run": ["Well", "Image"], + "Well": "WellSample", + "WellSample": "Image" + } def remove_map_annotations(conn, obj, namespace): anns = list(obj.listAnnotations(ns=namespace)) From 4e9e8d596e6b8fba59bbd5ab885a46e306bd75ed Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 17:59:09 +0200 Subject: [PATCH 028/127] Auto ID input and doc update --- omero/annotation_scripts/KeyVal_from_csv.py | 145 ++++++++++---------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 533a2ab7b..67c7a957b 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -147,14 +147,14 @@ def get_children_recursive(source_object, target_type): return result def keyval_from_csv(conn, script_params): - source_type = script_params["Source_object_type"] - target_type = script_params["Target_object_type"] - source_ids = script_params["Source_IDs"] - file_ids = script_params["File_Annotation_ID"] + source_type = script_params["Data_Type"] + target_type = script_params["Target Data_Type"] + source_ids = script_params["IDs"] + file_ids = script_params["File_Annotation"] namespace = script_params["Namespace (leave blank for default)"] to_exclude = script_params["Columns to exclude"] - target_id_colname = script_params["Column name of the 'target ID'"] - target_name_colname = script_params["Namespace (leave blank for default)"] + target_id_colname = script_params["Target ID colname"] + target_name_colname = script_params["Target name colname"] separator = script_params["Separator"] ntarget_processed = 0 @@ -254,128 +254,127 @@ def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): def run_script(): - source_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), - rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image"), #Duplicate Image for UI, but not a problem for script - rstring("Tag")] + source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Tag"), + rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + ] - target_types = [rstring("Project"), + target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] - separators = ["guess", ";", ",", ""] + separators = ["guess", ";", ",", "TAB"] client = scripts.client( 'Add_Key_Val_from_csv', """ - This script reads an attached .csv file to annotate objects with key-value pairs. - - Only the child objects of the SOURCE will be searched and if they match an entry in - the .csv file, then a set of key-value pair will be added to the TARGET. - - In the .csv file, the TARGETs can be identified by their name (with a column named - "target_name"), in which case their names must be unique among all children objects - of the SOURCE. The TARGETs can also be identified by their IDs (with a column named - "target_id"). In case both are given, "target_name" will be ignored in favor of - "target_id". - - The .csv file must be imported in OMERO as a file annotation, and is passed as a - parameter to the script via the AnnotationID. + This script reads a .csv file to annotate target objects with key-value pairs. + TODO: add hyperlink to readthedocs + \t + \t + Parameters: + \t + - Data Type: Type of the "parent objects" in which "target objects" are searched. + - IDs: IDs of the "parent objects". + - Target Data Type: Type of the "target objects" that will be annotated. + - File_Annotation: IDs of .csv FileAnnotation or input file. + - Namespace: Namespace that will be given to the annotations. + \t + - Columns to exclude: Columns name of the .csv file to exclude. + - Target ID colname: Column name in the .csv of the target IDs. + - Target name colname: Column name in the .csv of the target names. + - Separator: Separator used in the .csv file. + \t + """, # Tabs are needed to add line breaks in the HTML - Multiple SOURCE and AnnotationID can be passed to the script, and each will be - processed independantly. When using a single AnnotationID, the same .csv will be - used for each SOURCE. When no AnnotationID is given, each SOURCE will use the - most recently attached .csv on itself. - - The annotation can also be associated to a namespace (defaults to user namespace). - - Complementary scripts: - - "Export Key Value to csv": Export key value pairs of a given namespace - - "Delete Key Value": Delete the key value pairs associated to a namespace - """, scripts.String( - "Source_object_type", optional=False, grouping="1", - description="Choose the object type containing the objects to annotate", - values=source_types, default="- Dataset"), - - scripts.List( - "Source_IDs", optional=False, grouping="1.1", - description="List of source IDs containing the objects to annotate.").ofType(rlong(0)), + "Data_Type", optional=False, grouping="1", + description="Parent data type of the objects to annotate.", + values=source_types, default="Dataset"), scripts.List( - "File_Annotation_ID", optional=True, grouping="1.2", - description="List of file IDs containing metadata to populate. If given, must match length of 'Source IDs'. Otherwise, uses the CSV file with the highest ID.").ofType(rlong(0)), + "IDs", optional=False, grouping="1.1", + description="List of parent data IDs containing the objects to annotate.").ofType(rlong(0)), scripts.String( - "Target_object_type", optional=False, grouping="1.3", - description="Choose the object type to annotate (must be bellow the chosen source object type)", + "Target Data_Type", optional=False, grouping="1.2", + description="The data type on which will be annotated. Entries in the .csv correspond to these objects.", values=target_types, default="-- Image"), + scripts.String( + "File_Annotation", optional=True, grouping="1.3", + description="If no file is provided, list of file IDs containing metadata to populate (must match length of 'IDs'). If neither, searches the most recently attached CSV file on each parent object."), + scripts.String( "Namespace (leave blank for default)", optional=True, grouping="1.4", - description="Choose a namespace for the annotations"), + description="Namespace given to the created key-value pairs annotations."), scripts.List( "Columns to exclude", optional=False, grouping="2", - description="List of columns to exclude from the key-value pair import", default=",").ofType(rstring("")), + description="List of columns in the .csv file to exclude from the key-value pair import. and correspond to the two following parameters.", default=",").ofType(rstring("")), scripts.String( - "Column name of the 'target ID'", optional=True, grouping="2.1", - description="Set the column name containing the id of the target", default="target_id"), + "Target ID colname", optional=False, grouping="2.1", + description="The column name in the .csv containing the id of the objects to annotate. Correspond to in exclude parameter.", default="target_id"), scripts.String( - "Column name of the 'target name'", optional=True, grouping="2.2", - description="Set the column name containing the id of the target", default="target_name"), + "Target name colname", optional=False, grouping="2.2", + description="The column name in the .csv containing the name of the objects to annotate (used if no column ID is provided or found in the .csv). Correspond to in exclude parameter.", default="target_name"), scripts.String( - "Separator", optional=False, grouping="3", - description="Choose the .csv separator", + "Separator", optional=False, grouping="2.3", + description="The separator used in the .csv file. 'guess' will attempt to detetect automatically which of ,;\\t is used.", values=separators, default="guess"), authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], - contact="https://forum.image.sc/tag/omero", - version="2.0.0" + contact="https://forum.image.sc/tag/omero" ) try: # process the list of args above. script_params = { # Param dict with defaults for optional parameters - "File_Annotation_ID": [None], + "File_Annotation": None, "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION } for key in client.getInputKeys(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) + print(script_params["File_Annotation"]) + # Getting rid of the trailing '---' added for the UI - tmp_src = script_params["Source_object_type"] - script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src - tmp_trg = script_params["Target_object_type"] - script_params["Target_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + tmp_src = script_params["Data_Type"] + script_params["Data_Type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src + tmp_trg = script_params["Target Data_Type"] + script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg - if script_params["Source_object_type"] == "Tag": - script_params["Source_object_type"] = "TagAnnotation" - assert None not in script_params["File_Annotation_ID"], "File annotation ID must be given when using Tag as source" + if script_params["Data_Type"] == "Tag": + script_params["Data_Type"] = "TagAnnotation" + assert None not in script_params["File_Annotation"], "File annotation ID must be given when using Tag as source" - if len(script_params["File_Annotation_ID"]) == 1: # Poulate the parameter with None or same ID for all source - script_params["File_Annotation_ID"] = script_params["File_Annotation_ID"] * len(script_params["Source_IDs"]) - assert len(script_params["File_Annotation_ID"]) == len(script_params["Source_IDs"]), "Number of Source IDs and FileAnnotation IDs must match" + if (script_params["File_Annotation"]) is not None and ("," in script_params["File_Annotation"]): # List of ID provided + script_params["File_Annotation"] = script_params["File_Annotation"].split(",") + else: + script_params["File_Annotation"] = [script_params["File_Annotation"]] + if len(script_params["File_Annotation"]) == 1: # Poulate the parameter with None or same ID for all source + script_params["File_Annotation"] = script_params["File_Annotation"] * len(script_params["IDs"]) + assert len(script_params["File_Annotation"]) == len(script_params["IDs"]), "Number of Source IDs and FileAnnotation IDs must match" - to_exclude = list(map(lambda x: x.replace('', script_params["Column name of the 'target ID'"]), + # Replacing the placeholders and with the actual values from the parameters + to_exclude = list(map(lambda x: x.replace('', script_params["Target ID colname"]), script_params["Columns to exclude"])) - script_params["Columns to exclude"] = list(map(lambda x: x.replace('', script_params["Column name of the 'target name'"]), + script_params["Columns to exclude"] = list(map(lambda x: x.replace('', script_params["Target name colname"]), to_exclude)) if script_params["Separator"] == "guess": script_params["Separator"] = None - elif script_params["Separator"] == "": + elif script_params["Separator"] == "TAB": script_params["Separator"] = "\t" - - # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) print("script params") From 9a802aafff43fbfaa41cf2fac067664be99d3e28 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 18:18:34 +0200 Subject: [PATCH 029/127] Auto ID input and doc update --- omero/annotation_scripts/KeyVal_from_csv.py | 1 - omero/annotation_scripts/KeyVal_to_csv.py | 69 +++++++++++++-------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 67c7a957b..b2ba5ccd7 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -273,7 +273,6 @@ def run_script(): This script reads a .csv file to annotate target objects with key-value pairs. TODO: add hyperlink to readthedocs \t - \t Parameters: \t - Data Type: Type of the "parent objects" in which "target objects" are searched. diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index b3b31ffee..19fc68a24 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -140,12 +140,12 @@ def main_loop(conn, script_params): @param conn: Blitz Gateway connection wrapper @param script_params: A map of the input parameters ''' - source_type = script_params["Source_object_type"] - target_type = script_params["Target_object_type"] - source_ids = script_params["Source_IDs"] + source_type = script_params["Data_Type"] + target_type = script_params["Target Data_Type"] + source_ids = script_params["IDs"] namespace = script_params["Namespace (leave blank for default)"] separator = script_params["Separator"] - include_parent = script_params["Include column(s) for parent objects name"] + include_parent = script_params["Include column(s) of parents name"] # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): @@ -215,39 +215,52 @@ def run_script(): scripting service, passing the required parameters. """ - source_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), - rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image"), #Duplicate Image for UI, but not a problem for script - rstring("Tag")] + source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Tag"), + rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + ] - target_types = [rstring("Project"), + target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] - agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" - separators = [";", ","] + separators = [";", ",", "TAB"] # Here we define the script name and description. # Good practice to put url here to give users more guidance on how to run # your script. client = scripts.client( 'KeyVal_to_csv.py', - ("Export key-value pairs of targets to .csv file" - " \nSee" - " http://www.openmicroscopy.org/site/support/omero5.2/developers/" - "scripts/user-guide.html for the tutorial that uses this script."), + """ + This script exports key-value pairs of objects to a .csv file. + Can also export a blank .csv with only of target objects' name and IDs. + (for example by providing a non-existing namespace) + TODO: add hyperlink to readthedocs + \t + Parameters: + \t + - Data Type: Type of the "parent objects" in which "target objects" are searched. + - IDs: IDs of the "parent objects". + - Target Data Type: Type of the "target objects" of which KV-pairs are exported. + - Namespace: Only annotations with this namespace will be exported. + \t + - Separator: Separator to be used in the .csv file. + - Include column(s) of parents name: If checked, add columns for each object in the hierarchy of the target data. + \t + """, scripts.String( - "Source_object_type", optional=False, grouping="1", - description="Choose the object type containing the objects to delete annotation from", - values=source_types, default="- Dataset"), + "Data_Type", optional=False, grouping="1", + description="Parent data type of the objects to annotate.", + values=source_types, default="Dataset"), scripts.List( - "Source_IDs", optional=False, grouping="1.1", - description="List of source IDs").ofType(rlong(0)), + "IDs", optional=False, grouping="1.1", + description="List of parent data IDs containing the objects to delete annotation from.").ofType(rlong(0)), scripts.String( - "Target_object_type", optional=True, grouping="1.2", + "Target Data_Type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from", values=target_types, default="-- Image"), @@ -261,13 +274,12 @@ def run_script(): values=separators, default=";"), scripts.Bool( - "Include column(s) for parent objects name", optional=False, grouping="3", + "Include column(s) of parents name", optional=False, grouping="3", description="Weather to include or not the name of the parent(s) objects as columns in the .csv", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", - version="2.0.0" ) try: @@ -280,10 +292,13 @@ def run_script(): script_params[key] = client.getInput(key, unwrap=True) # Getting rid of the trailing '---' added for the UI - tmp_src = script_params["Source_object_type"] - script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src - tmp_trg = script_params["Target_object_type"] - script_params["Target_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + tmp_src = script_params["Data_Type"] + script_params["Data_Type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src + tmp_trg = script_params["Target Data_Type"] + script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + + if script_params["Separator"] == "TAB": + script_params["Separator"] = "\t" print(script_params) # handy to have inputs in the std-out log From 182aeb104a087ed5b73efa18b9fc75a4ed2e1765 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 18:27:45 +0200 Subject: [PATCH 030/127] Auto ID input and doc update for delete script --- omero/annotation_scripts/KeyVal_to_csv.py | 8 +-- omero/annotation_scripts/Remove_KeyVal.py | 59 ++++++++++++++--------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 19fc68a24..61e92b600 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -261,21 +261,21 @@ def run_script(): scripts.String( "Target Data_Type", optional=True, grouping="1.2", - description="Choose the object type to delete annotation from", + description="Choose the object type to delete annotation from.", values=target_types, default="-- Image"), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="1.3", - description="Choose a namespace for the annotations"), + description="Choose a namespace for the annotations."), scripts.String( "Separator", optional=False, grouping="3", - description="Choose the .csv separator", + description="Choose the .csv separator.", values=separators, default=";"), scripts.Bool( "Include column(s) of parents name", optional=False, grouping="3", - description="Weather to include or not the name of the parent(s) objects as columns in the .csv", default=False), + description="Weather to include or not the name of the parent(s) objects as columns in the .csv.", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 2f7d9e0c6..a4431c5d0 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -76,9 +76,9 @@ def remove_keyvalue(conn, script_params): @param conn: Blitz Gateway connection wrapper @param script_params: A map of the input parameters """ - source_type = script_params["Source_object_type"] - target_type = script_params["Target_object_type"] - source_ids = script_params["Source_IDs"] + source_type = script_params["Data_Type"] + target_type = script_params["Target Data_Type"] + source_ids = script_params["IDs"] namespace = script_params["Namespace (leave blank for default)"] nsuccess = 0 @@ -113,12 +113,13 @@ def remove_keyvalue(conn, script_params): scripting service, passing the required parameters. """ - source_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), - rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image"), #Duplicate Image for UI, but not a problem for script - rstring("Tag")] + source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Tag"), + rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + ] - target_types = [rstring("Project"), + target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] @@ -130,37 +131,47 @@ def remove_keyvalue(conn, script_params): # your script. client = scripts.client( 'Remove_Key_Value.py', - ("Remove key-value pairs from" - " Image IDs or by the Dataset IDs.\nSee" - " http://www.openmicroscopy.org/site/support/omero5.2/developers/" - "scripts/user-guide.html for the tutorial that uses this script."), + """ + This script deletes key-value pairs of all child objects founds. + Only key-value pairs of the namespace are deleted. + (default namespace correspond to editable KV pairs in web) + TODO: add hyperlink to readthedocs + \t + Parameters: + \t + - Data Type: Type of the "parent objects" in which "target objects" are searched. + - IDs: IDs of the "parent objects". + - Target Data Type: Type of the "target objects" of which KV-pairs are deleted. + - Namespace: Only annotations with this namespace will be deleted. + \t + """, scripts.String( - "Source_object_type", optional=False, grouping="1", - description="Choose the object type containing the objects to delete annotation from", - values=source_types, default="- Dataset"), + "Data_Type", optional=False, grouping="1", + description="Parent data type of the objects to annotate.", + values=source_types, default="Dataset"), scripts.List( - "Source_IDs", optional=False, grouping="1.1", - description="List of source IDs").ofType(rlong(0)), + "IDs", optional=False, grouping="1.1", + description="List of parent data IDs containing the objects to delete annotation from.").ofType(rlong(0)), scripts.String( - "Target_object_type", optional=True, grouping="1.2", + "Target Data_Type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", values=target_types, default="-- Image"), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="1.3", + default="NAMESPACE TO DELETE", description="Choose a namespace for the annotations"), scripts.Bool( agreement, optional=False, grouping="3", - description="Make sure that you understood what this script does"), + description="Make sure that you understood the scope of what will be deleted."), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", - version="2.0.0" ) try: @@ -175,10 +186,10 @@ def remove_keyvalue(conn, script_params): assert script_params[agreement], "Please confirm that you understood the risks." # Getting rid of the trailing '---' added for the UI - tmp_src = script_params["Source_object_type"] - script_params["Source_object_type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src - tmp_trg = script_params["Target_object_type"] - script_params["Target_object_type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + tmp_src = script_params["Data_Type"] + script_params["Data_Type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src + tmp_trg = script_params["Target Data_Type"] + script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg print(script_params) # handy to have inputs in the std-out log From 6993841e256da0496eef81ed276023d8dfa79e40 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 25 Oct 2023 19:24:10 +0200 Subject: [PATCH 031/127] target object name sorting before export --- omero/annotation_scripts/KeyVal_to_csv.py | 54 ++++++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 61e92b600..6a728ddc8 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -88,15 +88,51 @@ def get_children_recursive(source_object, target_type): result.extend(get_children_recursive(child_obj, target_type)) return result -def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator): +def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well): def to_csv(ll): """convience function to write a csv line""" nl = len(ll) - fmstr = ("{}"+separator+" ")*(nl-1)+"{}\n" + fmstr = ("{}"+separator)*(nl-1)+"{}\n" return fmstr.format(*ll) + def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): + result_name = [] + result_ancestry = [] + result_value = [] + + tmp_ancestry_l = [] # That's an imbricated list of list of tuples, making it simplier first + for ancestries in obj_ancestry_l: + tmp_ancestry_l.append(["".join(list(obj_name)) for obj_name in ancestries]) + + start_idx = 0 + stop_idx = 1 + while start_idx0: # If there's anything to add at all for (parent_type, _) in obj_ancestry_l[0]: @@ -135,6 +171,8 @@ def to_csv(ll): return "done" + + def main_loop(conn, script_params): ''' writes the data (list of dicts) to a file @param conn: Blitz Gateway connection wrapper @@ -147,6 +185,8 @@ def main_loop(conn, script_params): separator = script_params["Separator"] include_parent = script_params["Include column(s) of parents name"] + is_well = False + # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): obj_ancestry_l = [] @@ -168,17 +208,17 @@ def main_loop(conn, script_params): target_obj_l = get_children_recursive(source_object, target_type) # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) for target_obj in target_obj_l: + is_well = target_obj.OMERO_CLASS == "Well" print("Processing object:", target_obj) annotation_dicts.append(get_existing_map_annotions(target_obj, namespace, ZERO_PADDING)) obj_id_l.append(target_obj.getId()) - obj_name_l.append(target_obj.getWellPos() if target_obj.OMERO_CLASS == "Well" else target_obj.getName()) + obj_name_l.append(target_obj.getWellPos() if is_well else target_obj.getName()) if include_parent: ancestry = [(o.OMERO_CLASS, o.getWellPos() if o.OMERO_CLASS == "Well" else o.getName()) - for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"][::-1] - obj_ancestry_l.append(ancestry) - + for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"] + obj_ancestry_l.append(ancestry[::-1]) # Reverse the order to go from highest to lowest - mess = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator) + mess = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well) print(mess) # for ds in datasets: From 8ca4ef9181dd029ca954345385030617d844deec Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Wed, 25 Oct 2023 22:29:24 +0200 Subject: [PATCH 032/127] New script to convert namespace --- .../Convert_namespace_KeyVal.py | 214 ++++++++++++++++++ omero/annotation_scripts/KeyVal_from_csv.py | 8 +- omero/annotation_scripts/KeyVal_to_csv.py | 2 - omero/annotation_scripts/Remove_KeyVal.py | 2 - 4 files changed, 215 insertions(+), 11 deletions(-) create mode 100644 omero/annotation_scripts/Convert_namespace_KeyVal.py diff --git a/omero/annotation_scripts/Convert_namespace_KeyVal.py b/omero/annotation_scripts/Convert_namespace_KeyVal.py new file mode 100644 index 000000000..d652d3f2e --- /dev/null +++ b/omero/annotation_scripts/Convert_namespace_KeyVal.py @@ -0,0 +1,214 @@ +# coding=utf-8 +""" + Convert_namespace_KeyVal.py + + Convert the namespace of objects key-value pairs. +----------------------------------------------------------------------------- + Copyright (C) 2018 + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +------------------------------------------------------------------------------ +Created by Tom Boissonnet + +""" + +import omero +from omero.gateway import BlitzGateway +from omero.rtypes import rstring, rlong +import omero.scripts as scripts + +CHILD_OBJECTS = { + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + #"Run": ["Well", "Image"], + "Well": "WellSample", + "WellSample": "Image" + } + +def get_children_recursive(source_object, target_type): + print(source_object, target_type) + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if source_object.OMERO_CLASS != "WellSample": + return source_object.listChildren() + else: + return [source_object.getImage()] + else: + result = [] + for child_obj in source_object.listChildren(): + # Going down in the Hierarchy list for all childs that aren't yet the target + result.extend(get_children_recursive(child_obj, target_type)) + return result + +def get_existing_map_annotions(obj, namespace): + result = [] + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + result.extend([(k,v) for (k,v) in ann.getValue()]) + return result + +def remove_map_annotations(conn, obj, namespace): + anns = list(obj.listAnnotations(ns=namespace)) + mapann_ids = [ann.id for ann in anns + if isinstance(ann, omero.gateway.MapAnnotationWrapper)] + + if len(mapann_ids) == 0: + return 0 + print("Map Annotation IDs to delete:", mapann_ids) + try: + conn.deleteObjects("Annotation", mapann_ids) + return 1 + except Exception: + print("Failed to delete links") + return 0 + +def annotate_object(conn, obj, kv_list, namespace): + + print("Adding kv:") + + map_ann = omero.gateway.MapAnnotationWrapper(conn) + map_ann.setNs(namespace) + map_ann.setValue(kv_list) + map_ann.save() + + print("Map Annotation created", map_ann.id) + obj.linkAnnotation(map_ann) + +def replace_namespace(conn, script_params): + source_type = script_params["Data_Type"] + target_type = script_params["Target Data_Type"] + source_ids = script_params["IDs"] + old_namespace = script_params["Old Namespace (leave blank for default)"] + new_namespace = script_params["New Namespace (leave blank for default)"] + + ntarget_processed = 0 + ntarget_updated = 0 + + # One file output per given ID + for source_object in conn.getObjects(source_type, source_ids): + + if source_type == target_type: + target_obj_l = [source_object] + else: + if source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) + target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later + else: + target_obj_l = get_children_recursive(source_object, target_type) + # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) + + for target_obj in target_obj_l: + ntarget_processed += 1 + print("Processing object:", target_obj) + kv_list = get_existing_map_annotions(target_obj, old_namespace) + if len(kv_list) > 1: + remove_map_annotations(conn, target_obj, old_namespace) + annotate_object(conn, target_obj, kv_list, new_namespace) + ntarget_updated += 1 + + message = f"Updated kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" + return message + +def run_script(): + + source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), + rstring("Well"), rstring("Tag"), + rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + ] + + target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script + rstring("- Dataset"), rstring("-- Image"), + rstring("Screen"), rstring("- Plate"), + rstring("-- Well"), rstring("--- Image")] + + client = scripts.client( + 'Convert_KeyVal_namespace', + """ + This script converts the namespace of key-value pair annotations. + TODO: add hyperlink to readthedocs + \t + Parameters: + \t + - Data Type: Type of the "parent objects" in which "target objects" are searched. + - IDs: IDs of the "parent objects". + - Target Data Type: Type of the "target objects" that will be changed. + - Old Namespace: Namespace of the annotations to change. + - New Namespace: New namespace for the annotations. + \t + """, # Tabs are needed to add line breaks in the HTML + + scripts.String( + "Data_Type", optional=False, grouping="1", + description="Parent data type of the objects to annotate.", + values=source_types, default="Dataset"), + + scripts.List( + "IDs", optional=False, grouping="1.1", + description="List of parent data IDs containing the objects to annotate.").ofType(rlong(0)), + + scripts.String( + "Target Data_Type", optional=False, grouping="1.2", + description="The data type for which key-value pair annotations will be converted.", + values=target_types, default="-- Image"), + + scripts.String( + "Old Namespace (leave blank for default)", optional=True, grouping="1.4", + description="The namespace of the annotations to change"), + + scripts.String( + "New Namespace (leave blank for default)", optional=True, grouping="1.5", + description="The new namespace for the annotations."), + + authors=["Tom Boissonnet"], + institutions=["CAi HHU"], + contact="https://forum.image.sc/tag/omero" + ) + + + try: + # process the list of args above. + script_params = { # Param dict with defaults for optional parameters + "File_Annotation": None, + "Old Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION, + "New Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION + } + for key in client.getInputKeys(): + if client.getInput(key): + script_params[key] = client.getInput(key, unwrap=True) + + # Getting rid of the trailing '---' added for the UI + tmp_trg = script_params["Target Data_Type"] + script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg + + if script_params["Data_Type"] == "Tag": + script_params["Data_Type"] = "TagAnnotation" + + # wrap client to use the Blitz Gateway + conn = BlitzGateway(client_obj=client) + print("script params") + for k, v in script_params.items(): + print(k, v) + message = replace_namespace(conn, script_params) + client.setOutput("Message", rstring(message)) + + except AssertionError as err: #Display assertion errors in OMERO.web activities + client.setOutput("ERROR", rstring(err)) + raise AssertionError(str(err)) + + finally: + client.closeSession() + + +if __name__ == "__main__": + run_script() diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index b2ba5ccd7..5b651a5cf 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -34,11 +34,9 @@ from omero.gateway import BlitzGateway from omero.rtypes import rstring, rlong import omero.scripts as scripts -from omero.cmd import Delete2 import sys import csv -import copy from math import floor from omero.util.populate_roi import DownloadingOriginalFileProvider @@ -299,7 +297,7 @@ def run_script(): scripts.String( "Target Data_Type", optional=False, grouping="1.2", - description="The data type on which will be annotated. Entries in the .csv correspond to these objects.", + description="The data type which will be annotated. Entries in the .csv correspond to these objects.", values=target_types, default="-- Image"), scripts.String( @@ -342,12 +340,8 @@ def run_script(): for key in client.getInputKeys(): if client.getInput(key): script_params[key] = client.getInput(key, unwrap=True) - print(script_params["File_Annotation"]) - # Getting rid of the trailing '---' added for the UI - tmp_src = script_params["Data_Type"] - script_params["Data_Type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src tmp_trg = script_params["Target Data_Type"] script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 6a728ddc8..5c9b78c6f 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -332,8 +332,6 @@ def run_script(): script_params[key] = client.getInput(key, unwrap=True) # Getting rid of the trailing '---' added for the UI - tmp_src = script_params["Data_Type"] - script_params["Data_Type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src tmp_trg = script_params["Target Data_Type"] script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index a4431c5d0..08f6e8be7 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -186,8 +186,6 @@ def remove_keyvalue(conn, script_params): assert script_params[agreement], "Please confirm that you understood the risks." # Getting rid of the trailing '---' added for the UI - tmp_src = script_params["Data_Type"] - script_params["Data_Type"] = tmp_src.split(" ")[1] if " " in tmp_src else tmp_src tmp_trg = script_params["Target Data_Type"] script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg From 5cdaba9fb7eec37b983da8605f5975f8e22e992e Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Wed, 25 Oct 2023 22:29:58 +0200 Subject: [PATCH 033/127] renamed convert namespace script --- .../{Convert_namespace_KeyVal.py => Convert_KeyVal_namespace.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename omero/annotation_scripts/{Convert_namespace_KeyVal.py => Convert_KeyVal_namespace.py} (100%) diff --git a/omero/annotation_scripts/Convert_namespace_KeyVal.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py similarity index 100% rename from omero/annotation_scripts/Convert_namespace_KeyVal.py rename to omero/annotation_scripts/Convert_KeyVal_namespace.py From 4467d5451b969be24bab6739b4d488b490aed605 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 08:58:13 +0200 Subject: [PATCH 034/127] packed advanced parameters in the UI --- omero/annotation_scripts/KeyVal_from_csv.py | 22 ++++++++++++--------- omero/annotation_scripts/KeyVal_to_csv.py | 8 ++++++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 5b651a5cf..7eda37f14 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -279,10 +279,10 @@ def run_script(): - File_Annotation: IDs of .csv FileAnnotation or input file. - Namespace: Namespace that will be given to the annotations. \t + - Separator: Separator used in the .csv file. - Columns to exclude: Columns name of the .csv file to exclude. - Target ID colname: Column name in the .csv of the target IDs. - Target name colname: Column name in the .csv of the target names. - - Separator: Separator used in the .csv file. \t """, # Tabs are needed to add line breaks in the HTML @@ -308,23 +308,27 @@ def run_script(): "Namespace (leave blank for default)", optional=True, grouping="1.4", description="Namespace given to the created key-value pairs annotations."), + scripts.Bool( + "Advanced parameters", optional=True, grouping="2", + description="Ticking or unticking this has no effect", default=False), + + scripts.String( + "Separator", optional=False, grouping="2.1", + description="The separator used in the .csv file. 'guess' will attempt to detetect automatically which of ,;\\t is used.", + values=separators, default="guess"), + scripts.List( - "Columns to exclude", optional=False, grouping="2", + "Columns to exclude", optional=False, grouping="2.2", description="List of columns in the .csv file to exclude from the key-value pair import. and correspond to the two following parameters.", default=",").ofType(rstring("")), scripts.String( - "Target ID colname", optional=False, grouping="2.1", + "Target ID colname", optional=False, grouping="2.3", description="The column name in the .csv containing the id of the objects to annotate. Correspond to in exclude parameter.", default="target_id"), scripts.String( - "Target name colname", optional=False, grouping="2.2", + "Target name colname", optional=False, grouping="2.4", description="The column name in the .csv containing the name of the objects to annotate (used if no column ID is provided or found in the .csv). Correspond to in exclude parameter.", default="target_name"), - scripts.String( - "Separator", optional=False, grouping="2.3", - description="The separator used in the .csv file. 'guess' will attempt to detetect automatically which of ,;\\t is used.", - values=separators, default="guess"), - authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 5c9b78c6f..36624c548 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -308,13 +308,17 @@ def run_script(): "Namespace (leave blank for default)", optional=True, grouping="1.3", description="Choose a namespace for the annotations."), + scripts.Bool( + "Advanced parameters", optional=True, grouping="2", + description="Ticking or unticking this has no effect", default=False), + scripts.String( - "Separator", optional=False, grouping="3", + "Separator", optional=False, grouping="2.1", description="Choose the .csv separator.", values=separators, default=";"), scripts.Bool( - "Include column(s) of parents name", optional=False, grouping="3", + "Include column(s) of parents name", optional=False, grouping="2.2", description="Weather to include or not the name of the parent(s) objects as columns in the .csv.", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], From 63d5cd116f329a1c21bd059069a298ace50d872b Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 09:26:10 +0200 Subject: [PATCH 035/127] multiple namespace support --- .../Convert_KeyVal_namespace.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index d652d3f2e..ea342e259 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -50,17 +50,17 @@ def get_children_recursive(source_object, target_type): result.extend(get_children_recursive(child_obj, target_type)) return result -def get_existing_map_annotions(obj, namespace): - result = [] - for ann in obj.listAnnotations(ns=namespace): - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - result.extend([(k,v) for (k,v) in ann.getValue()]) - return result - -def remove_map_annotations(conn, obj, namespace): - anns = list(obj.listAnnotations(ns=namespace)) - mapann_ids = [ann.id for ann in anns - if isinstance(ann, omero.gateway.MapAnnotationWrapper)] +def get_existing_map_annotions(obj, namespace_l): + keyval_l, ann_l = [], [] + for namespace in namespace_l: + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + keyval_l.extend([(k,v) for (k,v) in ann.getValue()]) + ann_l.append(ann) + return keyval_l, ann_l + +def remove_map_annotations(conn, obj, ann_l): + mapann_ids = [ann.id for ann in ann_l] if len(mapann_ids) == 0: return 0 @@ -110,10 +110,11 @@ def replace_namespace(conn, script_params): for target_obj in target_obj_l: ntarget_processed += 1 print("Processing object:", target_obj) - kv_list = get_existing_map_annotions(target_obj, old_namespace) - if len(kv_list) > 1: - remove_map_annotations(conn, target_obj, old_namespace) - annotate_object(conn, target_obj, kv_list, new_namespace) + keyval_l, ann_l = get_existing_map_annotions(target_obj, old_namespace) + + if len(keyval_l) > 1: + annotate_object(conn, target_obj, keyval_l, new_namespace) + remove_map_annotations(conn, target_obj, ann_l) ntarget_updated += 1 message = f"Updated kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" @@ -136,6 +137,7 @@ def run_script(): 'Convert_KeyVal_namespace', """ This script converts the namespace of key-value pair annotations. + TODO: add hyperlink to readthedocs \t Parameters: @@ -143,7 +145,7 @@ def run_script(): - Data Type: Type of the "parent objects" in which "target objects" are searched. - IDs: IDs of the "parent objects". - Target Data Type: Type of the "target objects" that will be changed. - - Old Namespace: Namespace of the annotations to change. + - Old Namespace: Namespace(s) of the annotations to group and change. - New Namespace: New namespace for the annotations. \t """, # Tabs are needed to add line breaks in the HTML @@ -162,9 +164,9 @@ def run_script(): description="The data type for which key-value pair annotations will be converted.", values=target_types, default="-- Image"), - scripts.String( + scripts.List( "Old Namespace (leave blank for default)", optional=True, grouping="1.4", - description="The namespace of the annotations to change"), + description="The namespace(s) of the annotations to group and change.").ofType(rstring("")), scripts.String( "New Namespace (leave blank for default)", optional=True, grouping="1.5", @@ -180,7 +182,7 @@ def run_script(): # process the list of args above. script_params = { # Param dict with defaults for optional parameters "File_Annotation": None, - "Old Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION, + "Old Namespace (leave blank for default)": [omero.constants.metadata.NSCLIENTMAPANNOTATION], "New Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION } for key in client.getInputKeys(): From 8c7c087e0095367e2a575973f8db53cf3e6f31ec Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 10:40:06 +0200 Subject: [PATCH 036/127] multi namespace support for delete and export --- .../Convert_KeyVal_namespace.py | 3 +- omero/annotation_scripts/KeyVal_to_csv.py | 72 +++++++++---------- omero/annotation_scripts/Remove_KeyVal.py | 37 +++++----- 3 files changed, 54 insertions(+), 58 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index ea342e259..3ebcfd40f 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -111,8 +111,7 @@ def replace_namespace(conn, script_params): ntarget_processed += 1 print("Processing object:", target_obj) keyval_l, ann_l = get_existing_map_annotions(target_obj, old_namespace) - - if len(keyval_l) > 1: + if len(keyval_l) > 0: annotate_object(conn, target_obj, keyval_l, new_namespace) remove_map_annotations(conn, target_obj, ann_l) ntarget_updated += 1 diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 36624c548..6c36a0e8d 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -45,15 +45,16 @@ ZERO_PADDING = 3 # To allow duplicated keys (3 means up to 1000 duplicate key on a single object) -def get_existing_map_annotions(obj, namespace, zero_padding): +def get_existing_map_annotions(obj, namespace_l, zero_padding): key_l = [] result = OrderedDict() - for ann in obj.listAnnotations(ns=namespace): - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - for (k,v) in ann.getValue(): - n_occurence = key_l.count(k) - result[f"{str(n_occurence).rjust(zero_padding, '0')}{k}"] = v - key_l.append(k) # To count the multiple occurence of keys + for namespace in namespace_l: + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + for (k,v) in ann.getValue(): + n_occurence = key_l.count(k) + result[f"{str(n_occurence).rjust(zero_padding, '0')}{k}"] = v + key_l.append(k) # To count the multiple occurence of keys return result def group_keyvalue_dictionaries(annotation_dicts, zero_padding): @@ -118,7 +119,7 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): else: # Same thing, but pad the 'well-name keys number' with zeros first sort_order = sorted(range(len(subseq)), key=lambda x: f"{subseq[x][0]}{int(subseq[x][1:]):03}") - # sorted(range(len(subseq)), key=lambda x: int("".join([i for i in subseq[x] if i.isdigit()]))) + for idx in sort_order: # result_name.append(obj_name_l[start_idx:stop_idx][idx]) result_ancestry.append(obj_ancestry_l[start_idx:stop_idx][idx]) @@ -131,12 +132,13 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): all_key, whole_values_l = group_keyvalue_dictionaries(annotation_dicts, ZERO_PADDING) - obj_name_l, obj_ancestry_l, whole_values_l = sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well) - counter = 0 + if len(obj_ancestry_l)>0: # If there's anything to add at all + obj_name_l, obj_ancestry_l, whole_values_l = sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well) #Sorting relies on parents for (parent_type, _) in obj_ancestry_l[0]: all_key.insert(counter, parent_type); counter += 1 + all_key.insert(counter, "target_id") all_key.insert(counter + 1, "target_name") for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, obj_name_l, whole_values_l)): @@ -181,7 +183,7 @@ def main_loop(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] - namespace = script_params["Namespace (leave blank for default)"] + namespace_l = script_params["Namespace (leave blank for default)"] separator = script_params["Separator"] include_parent = script_params["Include column(s) of parents name"] @@ -190,36 +192,32 @@ def main_loop(conn, script_params): # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): obj_ancestry_l = [] + annotation_dicts = [] + obj_id_l, obj_name_l = [], [] if source_type == target_type: - print("Processing object:", source_object) - annotation_dicts = [get_existing_map_annotions(source_object, namespace, ZERO_PADDING)] - obj_id_l = [source_object.getId()] - obj_name_l = [source_object.getWellPos() if source_object.OMERO_CLASS == "Well" else source_object.getName()] + target_obj_l = [source_object] else: - annotation_dicts = [] - obj_id_l, obj_name_l = [], [] - if source_type == "TagAnnotation": target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later source_object = target_obj_l[0] # Putting the csv file on the first child else: - print(source_object, target_type) target_obj_l = get_children_recursive(source_object, target_type) - # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - for target_obj in target_obj_l: - is_well = target_obj.OMERO_CLASS == "Well" - print("Processing object:", target_obj) - annotation_dicts.append(get_existing_map_annotions(target_obj, namespace, ZERO_PADDING)) - obj_id_l.append(target_obj.getId()) - obj_name_l.append(target_obj.getWellPos() if is_well else target_obj.getName()) - if include_parent: - ancestry = [(o.OMERO_CLASS, o.getWellPos() if o.OMERO_CLASS == "Well" else o.getName()) - for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"] - obj_ancestry_l.append(ancestry[::-1]) # Reverse the order to go from highest to lowest - - mess = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well) - print(mess) + # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) + for target_obj in target_obj_l: + is_well = target_obj.OMERO_CLASS == "Well" + print("Processing object:", target_obj) + annotation_dicts.append(get_existing_map_annotions(target_obj, namespace_l, ZERO_PADDING)) + obj_id_l.append(target_obj.getId()) + obj_name_l.append(target_obj.getWellPos() if is_well else target_obj.getName()) + if include_parent: + ancestry = [(o.OMERO_CLASS, o.getWellPos() if o.OMERO_CLASS == "Well" else o.getName()) + for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"] + obj_ancestry_l.append(ancestry[::-1]) # Reverse the order to go from highest to lowest + + message = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well) + print(message) + return message # for ds in datasets: # # name of the file @@ -283,7 +281,7 @@ def run_script(): - Data Type: Type of the "parent objects" in which "target objects" are searched. - IDs: IDs of the "parent objects". - Target Data Type: Type of the "target objects" of which KV-pairs are exported. - - Namespace: Only annotations with this namespace will be exported. + - Namespace: Only annotations having one of these namespace(s) will be exported. \t - Separator: Separator to be used in the .csv file. - Include column(s) of parents name: If checked, add columns for each object in the hierarchy of the target data. @@ -304,9 +302,9 @@ def run_script(): description="Choose the object type to delete annotation from.", values=target_types, default="-- Image"), - scripts.String( + scripts.List( "Namespace (leave blank for default)", optional=True, grouping="1.3", - description="Choose a namespace for the annotations."), + description="Namespace(s) to include for the export of key-value pairs annotations.").ofType(rstring("")), scripts.Bool( "Advanced parameters", optional=True, grouping="2", @@ -328,7 +326,7 @@ def run_script(): try: script_params = { - "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION + "Namespace (leave blank for default)": [omero.constants.metadata.NSCLIENTMAPANNOTATION] } for key in client.getInputKeys(): if client.getInput(key): diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 08f6e8be7..3142bd202 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -41,10 +41,12 @@ "WellSample": "Image" } -def remove_map_annotations(conn, obj, namespace): - anns = list(obj.listAnnotations(ns=namespace)) - mapann_ids = [ann.id for ann in anns - if isinstance(ann, omero.gateway.MapAnnotationWrapper)] +def remove_map_annotations(conn, obj, namespace_l): + mapann_ids = [] + for namespace in namespace_l: + anns = list(obj.listAnnotations(ns=namespace)) + mapann_ids.extend([ann.id for ann in anns + if isinstance(ann, omero.gateway.MapAnnotationWrapper)]) if len(mapann_ids) == 0: return 0 @@ -79,16 +81,13 @@ def remove_keyvalue(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] - namespace = script_params["Namespace (leave blank for default)"] + namespace_l = script_params["Namespace (leave blank for default)"] nsuccess = 0 ntotal = 0 for source_object in conn.getObjects(source_type, source_ids): if source_type == target_type: - print("Processing object:", source_object) - ret = remove_map_annotations(conn, source_object, namespace) - nsuccess += ret - ntotal += 1 + target_obj_l = [source_object] else: # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) if source_type == "TagAnnotation": @@ -96,11 +95,11 @@ def remove_keyvalue(conn, script_params): target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later else: target_obj_l = get_children_recursive(source_object, target_type) - for target_obj in target_obj_l: - print("Processing object:", target_obj) - ret = remove_map_annotations(conn, target_obj, namespace) - nsuccess += ret - ntotal += 1 + for target_obj in target_obj_l: + print("Processing object:", target_obj) + ret = remove_map_annotations(conn, target_obj, namespace_l) + nsuccess += ret + ntotal += 1 message = f"Key value data deleted from {nsuccess} of {ntotal} objects" @@ -142,7 +141,7 @@ def remove_keyvalue(conn, script_params): - Data Type: Type of the "parent objects" in which "target objects" are searched. - IDs: IDs of the "parent objects". - Target Data Type: Type of the "target objects" of which KV-pairs are deleted. - - Namespace: Only annotations with this namespace will be deleted. + - Namespace: Only annotations having one of these namespace(s) will be deleted. \t """, @@ -160,13 +159,13 @@ def remove_keyvalue(conn, script_params): description="Choose the object type to delete annotation from.", values=target_types, default="-- Image"), - scripts.String( + scripts.List( "Namespace (leave blank for default)", optional=True, grouping="1.3", default="NAMESPACE TO DELETE", - description="Choose a namespace for the annotations"), + description="Annotation with these namespace will be deleted.").ofType(rstring("")), scripts.Bool( - agreement, optional=False, grouping="3", + agreement, optional=False, grouping="2", description="Make sure that you understood the scope of what will be deleted."), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], @@ -176,7 +175,7 @@ def remove_keyvalue(conn, script_params): try: script_params = { - "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION + "Namespace (leave blank for default)": [omero.constants.metadata.NSCLIENTMAPANNOTATION] } for key in client.getInputKeys(): if client.getInput(key): From a557ed7d875600c4485b5d7fcf1d5567c81bef44 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 10:52:10 +0200 Subject: [PATCH 037/127] Return result object --- .../Convert_KeyVal_namespace.py | 12 +++++++++--- omero/annotation_scripts/KeyVal_from_csv.py | 13 ++++++++++--- omero/annotation_scripts/KeyVal_to_csv.py | 10 ++++++---- omero/annotation_scripts/Remove_KeyVal.py | 18 +++++++++++++----- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 3ebcfd40f..d137a400a 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -23,7 +23,7 @@ import omero from omero.gateway import BlitzGateway -from omero.rtypes import rstring, rlong +from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts CHILD_OBJECTS = { @@ -93,6 +93,7 @@ def replace_namespace(conn, script_params): ntarget_processed = 0 ntarget_updated = 0 + result_obj = None # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): @@ -115,9 +116,12 @@ def replace_namespace(conn, script_params): annotate_object(conn, target_obj, keyval_l, new_namespace) remove_map_annotations(conn, target_obj, ann_l) ntarget_updated += 1 + if result_obj is None: + result_obj = target_obj message = f"Updated kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" - return message + + return message, result_obj def run_script(): @@ -200,8 +204,10 @@ def run_script(): print("script params") for k, v in script_params.items(): print(k, v) - message = replace_namespace(conn, script_params) + message, robj = replace_namespace(conn, script_params) client.setOutput("Message", rstring(message)) + if robj is not None: + client.setOutput("Result", robject(robj._obj)) except AssertionError as err: #Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 7eda37f14..c14585fd9 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -32,7 +32,7 @@ import omero from omero.gateway import BlitzGateway -from omero.rtypes import rstring, rlong +from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts import sys @@ -159,6 +159,8 @@ def keyval_from_csv(conn, script_params): ntarget_updated = 0 missing_names = 0 + result_obj = None + # One file output per given ID for source_object, file_ann_id in zip(conn.getObjects(source_type, source_ids), file_ids): #if file_ann_id is not None: # If the file ID is not defined, only already linked file will be used @@ -219,12 +221,15 @@ def keyval_from_csv(conn, script_params): updated = annotate_object(conn, target_obj, header, row, cols_to_ignore, namespace) if updated: + if result_obj is None: + result_obj = target_obj ntarget_updated += 1 message = f"Added kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" if missing_names > 0: message += f". {missing_names} {target_type} not found (using {'ID' if use_id else 'name'} to identify them)." - return message + + return message, result_obj def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): @@ -377,8 +382,10 @@ def run_script(): print("script params") for k, v in script_params.items(): print(k, v) - message = keyval_from_csv(conn, script_params) + message, robj = keyval_from_csv(conn, script_params) client.setOutput("Message", rstring(message)) + if robj is not None: + client.setOutput("Result", robject(robj._obj)) except AssertionError as err: #Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 6c36a0e8d..56644918c 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -25,7 +25,7 @@ import omero from omero.gateway import BlitzGateway -from omero.rtypes import rstring, rlong +from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts from omero.cmd import Delete2 @@ -216,8 +216,8 @@ def main_loop(conn, script_params): obj_ancestry_l.append(ancestry[::-1]) # Reverse the order to go from highest to lowest message = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well) - print(message) - return message + + return message, source_object # for ds in datasets: # # name of the file @@ -346,8 +346,10 @@ def run_script(): conn = BlitzGateway(client_obj=client) # do the editing... - message = main_loop(conn, script_params) + message, robj = main_loop(conn, script_params) client.setOutput("Message", rstring(message)) + if robj is not None: + client.setOutput("Result", robject(robj._obj)) except AssertionError as err: #Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 3142bd202..09af13f79 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -28,7 +28,7 @@ from omero.gateway import BlitzGateway import omero -from omero.rtypes import rlong, rstring, wrap +from omero.rtypes import rlong, rstring, robject, wrap import omero.scripts as scripts CHILD_OBJECTS = { @@ -85,6 +85,8 @@ def remove_keyvalue(conn, script_params): nsuccess = 0 ntotal = 0 + result_obj = None + for source_object in conn.getObjects(source_type, source_ids): if source_type == target_type: target_obj_l = [source_object] @@ -97,13 +99,17 @@ def remove_keyvalue(conn, script_params): target_obj_l = get_children_recursive(source_object, target_type) for target_obj in target_obj_l: print("Processing object:", target_obj) - ret = remove_map_annotations(conn, target_obj, namespace_l) - nsuccess += ret + success = remove_map_annotations(conn, target_obj, namespace_l) + if success: + nsuccess += 1 + if result_obj is None: + result_obj = target_obj + ntotal += 1 message = f"Key value data deleted from {nsuccess} of {ntotal} objects" - return message + return message, result_obj if __name__ == "__main__": @@ -194,8 +200,10 @@ def remove_keyvalue(conn, script_params): conn = BlitzGateway(client_obj=client) # do the editing... - message = remove_keyvalue(conn, script_params) + message, robj = remove_keyvalue(conn, script_params) client.setOutput("Message", rstring(message)) + if robj is not None: + client.setOutput("Result", robject(robj._obj)) except AssertionError as err: #Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) raise AssertionError(str(err)) From d3859c32f885b91853307b162734fb69817e3ee5 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 15:12:05 +0200 Subject: [PATCH 038/127] flake8 and print output --- .../Convert_KeyVal_namespace.py | 155 +++++--- omero/annotation_scripts/KeyVal_from_csv.py | 337 +++++++++++------- omero/annotation_scripts/KeyVal_to_csv.py | 275 +++++++------- omero/annotation_scripts/Remove_KeyVal.py | 148 +++++--- 4 files changed, 550 insertions(+), 365 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index d137a400a..95600fc36 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -25,46 +25,52 @@ from omero.gateway import BlitzGateway from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts +from omero.constants.metadata import NSCLIENTMAPANNOTATION + +from collections import OrderedDict CHILD_OBJECTS = { "Project": "Dataset", "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - #"Run": ["Well", "Image"], + # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } + def get_children_recursive(source_object, target_type): - print(source_object, target_type) - if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: + # Stop condition, we return the source_obj children if source_object.OMERO_CLASS != "WellSample": return source_object.listChildren() else: return [source_object.getImage()] - else: + else: # Not yet the target result = [] for child_obj in source_object.listChildren(): - # Going down in the Hierarchy list for all childs that aren't yet the target + # Going down in the Hierarchy list result.extend(get_children_recursive(child_obj, target_type)) return result + def get_existing_map_annotions(obj, namespace_l): keyval_l, ann_l = [], [] for namespace in namespace_l: for ann in obj.listAnnotations(ns=namespace): if isinstance(ann, omero.gateway.MapAnnotationWrapper): - keyval_l.extend([(k,v) for (k,v) in ann.getValue()]) + keyval_l.extend([(k, v) for (k, v) in ann.getValue()]) ann_l.append(ann) return keyval_l, ann_l + def remove_map_annotations(conn, obj, ann_l): mapann_ids = [ann.id for ann in ann_l] if len(mapann_ids) == 0: return 0 - print("Map Annotation IDs to delete:", mapann_ids) + print(f"\tMap Annotation IDs to delete: {mapann_ids}\n") try: conn.deleteObjects("Annotation", mapann_ids) return 1 @@ -72,9 +78,8 @@ def remove_map_annotations(conn, obj, ann_l): print("Failed to delete links") return 0 -def annotate_object(conn, obj, kv_list, namespace): - print("Adding kv:") +def annotate_object(conn, obj, kv_list, namespace): map_ann = omero.gateway.MapAnnotationWrapper(conn) map_ann.setNs(namespace) @@ -84,6 +89,28 @@ def annotate_object(conn, obj, kv_list, namespace): print("Map Annotation created", map_ann.id) obj.linkAnnotation(map_ann) + +def target_iterator(conn, source_object, target_type): + source_type = source_object.OMERO_CLASS + if source_type == target_type: + target_obj_l = [source_object] + elif source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, + [source_object.getId()]) + # Need that to load objects + obj_ids = [o.getId() for o in target_obj_l] + target_obj_l = list(conn.getObjects(target_type, obj_ids)) + else: + target_obj_l = get_children_recursive(source_object, + target_type) + + print(f"Iterating objects from {source_object}:") + for target_obj in target_obj_l: + print(f"\t- {target_obj}") + yield target_obj + print() + + def replace_namespace(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] @@ -98,40 +125,36 @@ def replace_namespace(conn, script_params): # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): - if source_type == target_type: - target_obj_l = [source_object] - else: - if source_type == "TagAnnotation": - target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) - target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later - else: - target_obj_l = get_children_recursive(source_object, target_type) - # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - - for target_obj in target_obj_l: + for target_obj in target_iterator(conn, source_object, target_type): ntarget_processed += 1 - print("Processing object:", target_obj) - keyval_l, ann_l = get_existing_map_annotions(target_obj, old_namespace) + keyval_l, ann_l = get_existing_map_annotions(target_obj, + old_namespace) if len(keyval_l) > 0: annotate_object(conn, target_obj, keyval_l, new_namespace) remove_map_annotations(conn, target_obj, ann_l) ntarget_updated += 1 if result_obj is None: result_obj = target_obj - - message = f"Updated kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" + else: + print("\tNo MapAnnotation found with that namespace\n") + print("\n------------------------------------\n") + message = f"Updated kv pairs to \ + {ntarget_updated}/{ntarget_processed} {target_type}" return message, result_obj + def run_script(): + # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), rstring("Tag"), - rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + rstring("Image"), ] - target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script + # Duplicate Image for UI, but not a problem for script + target_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] @@ -145,34 +168,39 @@ def run_script(): \t Parameters: \t - - Data Type: Type of the "parent objects" in which "target objects" are searched. - - IDs: IDs of the "parent objects". - - Target Data Type: Type of the "target objects" that will be changed. + - Data Type: parent-objects type in which target-objects are searched. + - IDs: IDs of the parent-objects. + - Target Data Type: Type of the target-objects that will be changed. - Old Namespace: Namespace(s) of the annotations to group and change. - New Namespace: New namespace for the annotations. \t - """, # Tabs are needed to add line breaks in the HTML + """, # Tabs are needed to add line breaks in the HTML scripts.String( "Data_Type", optional=False, grouping="1", - description="Parent data type of the objects to annotate.", + description="Parent-data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent data IDs containing the objects to annotate.").ofType(rlong(0)), + description="List of parent-data IDs containing the objects \ + to annotate.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=False, grouping="1.2", - description="The data type for which key-value pair annotations will be converted.", + description="The data type for which key-value pair annotations \ + will be converted.", values=target_types, default="-- Image"), scripts.List( - "Old Namespace (leave blank for default)", optional=True, grouping="1.4", - description="The namespace(s) of the annotations to group and change.").ofType(rstring("")), + "Old Namespace (leave blank for default)", optional=True, + grouping="1.4", + description="The namespace(s) of the annotations to \ + group and change.").ofType(rstring("")), scripts.String( - "New Namespace (leave blank for default)", optional=True, grouping="1.5", + "New Namespace (leave blank for default)", optional=True, + grouping="1.5", description="The new namespace for the annotations."), authors=["Tom Boissonnet"], @@ -180,36 +208,24 @@ def run_script(): contact="https://forum.image.sc/tag/omero" ) - + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", + "Old Namespace (leave blank for default)", + "New Namespace (leave blank for default)"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") try: - # process the list of args above. - script_params = { # Param dict with defaults for optional parameters - "File_Annotation": None, - "Old Namespace (leave blank for default)": [omero.constants.metadata.NSCLIENTMAPANNOTATION], - "New Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION - } - for key in client.getInputKeys(): - if client.getInput(key): - script_params[key] = client.getInput(key, unwrap=True) - - # Getting rid of the trailing '---' added for the UI - tmp_trg = script_params["Target Data_Type"] - script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg - - if script_params["Data_Type"] == "Tag": - script_params["Data_Type"] = "TagAnnotation" - # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - print("script params") - for k, v in script_params.items(): - print(k, v) - message, robj = replace_namespace(conn, script_params) + message, robj = replace_namespace(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) - except AssertionError as err: #Display assertion errors in OMERO.web activities + except AssertionError as err: + # Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) raise AssertionError(str(err)) @@ -217,5 +233,26 @@ def run_script(): client.closeSession() +def parameters_parsing(client): + params = OrderedDict() + # Param dict with defaults for optional parameters + params["File_Annotation"] = None + params["Old Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] + params["New Namespace (leave blank for default)"] = NSCLIENTMAPANNOTATION + + for key in client.getInputKeys(): + if client.getInput(key): + params[key] = client.getInput(key, unwrap=True) + + # Getting rid of the trailing '---' added for the UI + if " " in params["Target Data_Type"]: + params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + + if params["Data_Type"] == "Tag": + params["Data_Type"] = "TagAnnotation" + + return params + + if __name__ == "__main__": run_script() diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index c14585fd9..b585a64f8 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -2,15 +2,17 @@ """ KeyVal_from_csv.py - Adds key-value pairs to TARGETS on OMERO from a CSV file attached to a SOURCE container. + Adds key-value pairs to TARGETS on OMERO from a CSV file attached + to a SOURCE container. SOURCES can be: [Project, Dataset, Screen, Plate, Well] TARGETS can be: [Dataset, Plate, Well, Image] - The targets are referenced in the CSV file either from their name (must then be unique, - and be called "target_name") or from their ID (must be called "target_id"). + The targets are referenced in the CSV file either from their name + (must then be unique, and be called "target_name") or from their + ID (must be called "target_id"). In the case both are given, the ID will be used. - Every row corresponds to a set of value to attach to the given TARGET with the key of the - correponding column. + Every row corresponds to a set of value to attach to the given + TARGET with the key of the correponding column. ----------------------------------------------------------------------------- Copyright (C) 2018 @@ -34,29 +36,37 @@ from omero.gateway import BlitzGateway from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts +from omero.constants.metadata import NSCLIENTMAPANNOTATION import sys import csv from math import floor +from collections import OrderedDict from omero.util.populate_roi import DownloadingOriginalFileProvider -from collections import OrderedDict - CHILD_OBJECTS = { "Project": "Dataset", "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - #"Run": ["Well", "Image"], + # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } -def get_original_file(omero_object): + +def get_obj_name(omero_obj): + if omero_obj.OMERO_CLASS == "Well": + return omero_obj.getWellPos() + else: + return omero_obj.getName() + + +def get_original_file(omero_obj): """Find last AnnotationFile linked to object""" file_ann = None - for ann in omero_object.listAnnotations(): + for ann in omero_obj.listAnnotations(): if ann.OMERO_TYPE == omero.model.FileAnnotationI: file_name = ann.getFile().getName() # Pick file by Ann ID (or name if ID is None) @@ -65,8 +75,8 @@ def get_original_file(omero_object): # Get the most recent file file_ann = ann - obj_name = omero_object.getWellPos() if omero_object.OMERO_CLASS == "Well" else omero_object.getName() - assert file_ann is not None, f"No .csv FileAnnotation was found on {omero_object.OMERO_CLASS}:{obj_name}:{omero_object.getId()}" + assert file_ann is not None, f"No .csv FileAnnotation was found on \ + {omero_obj.OMERO_CLASS}:{get_obj_name(omero_obj)}:{omero_obj.getId()}" return file_ann @@ -85,8 +95,11 @@ def link_file_ann(conn, object_type, object_id, file_ann_id): if len(links) == 0: omero_object.linkAnnotation(file_ann) -def read_csv(conn, original_file, delimiter): #Dedicated function to read the CSV file - print("Original File", original_file.id.val, original_file.name.val) + +def read_csv(conn, original_file, delimiter): + """ Dedicated function to read the CSV file """ + print("Using FileAnnotation", + f"{original_file.id.val}:{original_file.name.val}") provider = DownloadingOriginalFileProvider(conn) # read the csv temp_file = provider.get_original_file_data(original_file) @@ -98,27 +111,26 @@ def read_csv(conn, original_file, delimiter): #Dedicated function to read the CS try: delimiter = csv.Sniffer().sniff( file_handle.read(floor(file_length/4)), ",;\t").delimiter - print("Using delimiter: ", delimiter, - f" after reading {floor(file_length/4)} characters") + print(f"Using delimiter {delimiter}", + f"after reading {floor(file_length/4)} characters") except Exception: file_handle.seek(0) try: delimiter = csv.Sniffer().sniff( file_handle.read(floor(file_length/2)), ",;\t").delimiter - print("Using delimiter: ", delimiter, - f"after reading {floor(file_length/2)} characters") + print(f"Using delimiter {delimiter}", + f"after reading {floor(file_length/4)} characters") except Exception: file_handle.seek(0) try: delimiter = csv.Sniffer().sniff( file_handle.read(floor(file_length*0.75)), ",;\t").delimiter - print("Using delimiter: ", delimiter, - f" after reading {floor(file_length*0.75)}" - " characters") + print(f"Using delimiter {delimiter} after", + f"reading {floor(file_length*0.75)} characters") except Exception: - print("Failed to sniff delimiter, using ','") + print("Failed to sniff delimiter, use ','") delimiter = "," # reset to start and read whole file... @@ -127,23 +139,46 @@ def read_csv(conn, original_file, delimiter): #Dedicated function to read the CS # keys are in the header row header = [el.strip() for el in data[0]] - print("header", header) + print(f"Header: {header}\n") return data, header + def get_children_recursive(source_object, target_type): - print(source_object, target_type) - if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: + # Stop condition, we return the source_obj children if source_object.OMERO_CLASS != "WellSample": return source_object.listChildren() else: return [source_object.getImage()] - else: + else: # Not yet the target result = [] for child_obj in source_object.listChildren(): - # Going down in the Hierarchy list for all childs that aren't yet the target + # Going down in the Hierarchy list result.extend(get_children_recursive(child_obj, target_type)) return result + +def target_iterator(conn, source_object, target_type): + source_type = source_object.OMERO_CLASS + if source_type == target_type: + target_obj_l = [source_object] + elif source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, + [source_object.getId()]) + # Need that to load objects + obj_ids = [o.getId() for o in target_obj_l] + target_obj_l = list(conn.getObjects(target_type, obj_ids)) + else: + target_obj_l = get_children_recursive(source_object, + target_type) + + print(f"Iterating objects from {source_object}:") + for target_obj in target_obj_l: + print(f"\t- {target_obj}") + yield target_obj + print() + + def keyval_from_csv(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] @@ -162,79 +197,83 @@ def keyval_from_csv(conn, script_params): result_obj = None # One file output per given ID - for source_object, file_ann_id in zip(conn.getObjects(source_type, source_ids), file_ids): - #if file_ann_id is not None: # If the file ID is not defined, only already linked file will be used - # link_file_ann(conn, source_type, source_object.id, file_ann_id) # TODO do we want to keep that linking? + source_objects = conn.getObjects(source_type, source_ids) + for source_object, file_ann_id in zip(source_objects, file_ids): if file_ann_id is not None: file_ann = conn.getObject("Annotation", oid=file_ann_id) - assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, "The provided annotation ID must reference a FileAnnotation, not a {file_ann.OMERO_TYPE}" + assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, "The \ + provided annotation ID must reference a FileAnnotation, \ + not a {file_ann.OMERO_TYPE}" else: file_ann = get_original_file(source_object) original_file = file_ann.getFile()._obj - print("set ann id", file_ann.getId()) + data, header = read_csv(conn, original_file, separator) - if source_type == target_type: - print("Processing object:", source_object) - target_obj_l = [source_object] - else: - if source_type == "TagAnnotation": - target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) - target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later - else: - target_obj_l = get_children_recursive(source_object, target_type) - # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) + target_obj_l = target_iterator(conn, source_object, target_type) - # Finds the index of the column used to identify the targets. Try for IDs first - idx_id = header.index(target_id_colname) if target_id_colname in header else -1 - idx_name = header.index(target_name_colname) if target_name_colname in header else -1 - cols_to_ignore = [header.index(el) for el in to_exclude if el in header] + # Index of the column used to identify the targets. Try for IDs first + idx_id, idx_name = -1, -1 + if target_id_colname in header: + idx_id = header.index(target_id_colname) + if target_name_colname in header: + idx_name = header.index(target_name_colname) + cols_to_ignore = [header.index(el) for el in to_exclude + if el in header] - assert (idx_id != -1) or (idx_name != -1), "Neither the column for the objects name or the objects index were found" + assert (idx_id != -1) or (idx_name != -1), "Neither \ + the column for the objects name or the objects index were found" - use_id = idx_id != -1 # If the column for the object index exist, use it - if not use_id: # Identify images by name must fail if two images have identical names + use_id = idx_id != -1 # use the obj_idx column if exist + if not use_id: + # Identify images by name fail if two images have identical names idx_id = idx_name target_d = dict() for target_obj in target_obj_l: + name = get_obj_name(target_obj) if target_type == "Well": - assert target_obj.getWellPos().upper() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getWellPos()}" - target_d[target_obj.getWellPos().upper()] = target_obj - else: - assert target_obj.getName() not in target_d.keys(), f"Target objects identified by name have duplicate: {target_obj.getName()}" - target_d[target_obj.getName()] = target_obj - else: # Setting the dictionnary target_id:target_obj keys as string to match CSV reader output - target_d = {str(target_obj.getId()):target_obj for target_obj in target_obj_l} + name = name.upper() + assert name not in target_d.keys(), f"Target objects \ + identified by name have duplicate: {name}" + target_d[name] = target_obj + + else: + # Setting the dictionnary target_id:target_obj + # keys as string to match CSV reader output + target_d = {str(target_obj.getId()): target_obj + for target_obj in target_obj_l} ntarget_processed += len(target_d) rows = data[1:] - for row in rows: # Iterate the CSV rows and search for the matching target + for row in rows: + # Iterate the CSV rows and search for the matching target target_id = row[idx_id] if target_id in target_d.keys(): target_obj = target_d[target_id] - obj_name = target_obj.getWellPos() if target_obj.OMERO_CLASS == "Well" else target_obj.getName() - print("Annotating Target:", f"{obj_name+':' if use_id else ''}{target_id}") else: missing_names += 1 - print(f"Target not found: {target_id}") + print(f"Not found: {target_id}") continue - updated = annotate_object(conn, target_obj, header, row, cols_to_ignore, namespace) + updated = annotate_object(conn, target_obj, header, row, + cols_to_ignore, namespace) if updated: if result_obj is None: result_obj = target_obj ntarget_updated += 1 + print("\n------------------------------------\n") - message = f"Added kv pairs to {ntarget_updated}/{ntarget_processed} {target_type}" + message = f"Added KV-pairs to \ + {ntarget_updated}/{ntarget_processed} {target_type}" if missing_names > 0: - message += f". {missing_names} {target_type} not found (using {'ID' if use_id else 'name'} to identify them)." + message += f". {missing_names} {target_type} not found \ + (using {'ID' if use_id else 'name'} to identify them)." return message, result_obj def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): - print("Adding kv:") kv_list = [] for i in range(len(row)): @@ -249,38 +288,42 @@ def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): map_ann.setValue(kv_list) map_ann.save() - print("Map Annotation created", map_ann.id) + print(f"MapAnnotation:{map_ann.id} created on {obj}") obj.linkAnnotation(map_ann) return True def run_script(): - - source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), + # Cannot add fancy layout if we want auto fill and selct of object ID + source_types = [ + rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), rstring("Tag"), - rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + rstring("Image"), ] - target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script + # Duplicate Image for UI, but not a problem for script + target_types = [ + rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image")] + rstring("-- Well"), rstring("--- Image") + ] separators = ["guess", ";", ",", "TAB"] client = scripts.client( 'Add_Key_Val_from_csv', """ - This script reads a .csv file to annotate target objects with key-value pairs. + Reads a .csv file to annotate target objects with key-value pairs. TODO: add hyperlink to readthedocs \t Parameters: \t - - Data Type: Type of the "parent objects" in which "target objects" are searched. - - IDs: IDs of the "parent objects". - - Target Data Type: Type of the "target objects" that will be annotated. + - Data Type: parent-objects type in which target-objects are searched. + - IDs: IDs of the parent-objects. + - Target Data Type: Type of the target-objects that will be annotated. - File_Annotation: IDs of .csv FileAnnotation or input file. - Namespace: Namespace that will be given to the annotations. \t @@ -289,105 +332,89 @@ def run_script(): - Target ID colname: Column name in the .csv of the target IDs. - Target name colname: Column name in the .csv of the target names. \t - """, # Tabs are needed to add line breaks in the HTML + """, # Tabs are needed to add line breaks in the HTML scripts.String( "Data_Type", optional=False, grouping="1", - description="Parent data type of the objects to annotate.", + description="Parent-data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent data IDs containing the objects to annotate.").ofType(rlong(0)), + description="List of parent-data IDs containing \ + the objects to annotate.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=False, grouping="1.2", - description="The data type which will be annotated. Entries in the .csv correspond to these objects.", + description="The data type which will be annotated. \ + Entries in the .csv correspond to these objects.", values=target_types, default="-- Image"), scripts.String( "File_Annotation", optional=True, grouping="1.3", - description="If no file is provided, list of file IDs containing metadata to populate (must match length of 'IDs'). If neither, searches the most recently attached CSV file on each parent object."), + description="If no file is provided, list of file IDs containing \ + metadata to populate (must match length of 'IDs'). If \ + neither, searches the most recently attached CSV \ + file on each parent object."), scripts.String( - "Namespace (leave blank for default)", optional=True, grouping="1.4", - description="Namespace given to the created key-value pairs annotations."), + "Namespace (leave blank for default)", + optional=True, grouping="1.4", + description="Namespace given to the created key-value \ + pairs annotations."), scripts.Bool( - "Advanced parameters", optional=True, grouping="2", - description="Ticking or unticking this has no effect", default=False), + "Advanced parameters", optional=True, grouping="2", default=False, + description="Ticking or unticking this has no effect"), scripts.String( "Separator", optional=False, grouping="2.1", - description="The separator used in the .csv file. 'guess' will attempt to detetect automatically which of ,;\\t is used.", + description="The separator used in the .csv file. 'guess' will \ + attempt to detetect automatically which of ,;\\t is used.", values=separators, default="guess"), scripts.List( "Columns to exclude", optional=False, grouping="2.2", - description="List of columns in the .csv file to exclude from the key-value pair import. and correspond to the two following parameters.", default=",").ofType(rstring("")), + default=",", + description="List of columns in the .csv file to exclude from the \ + key-value pair import. and correspond to the \ + two following parameters.").ofType(rstring("")), scripts.String( "Target ID colname", optional=False, grouping="2.3", - description="The column name in the .csv containing the id of the objects to annotate. Correspond to in exclude parameter.", default="target_id"), + default="target_id", + description="The column name in the .csv containing the id of the \ + objects to annotate. Matches in exclude parameter."), scripts.String( "Target name colname", optional=False, grouping="2.4", - description="The column name in the .csv containing the name of the objects to annotate (used if no column ID is provided or found in the .csv). Correspond to in exclude parameter.", default="target_name"), + default="target_name", + description="The column name in the .csv containing the name of \ + the objects to annotate (used if no column ID is provided or \ + found in the .csv). Matches in exclude parameter."), authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" ) - - + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", + "Namespace (leave blank for default)", "Separator", + "Columns to exclude", "Target ID colname", "Target name colname"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") try: - # process the list of args above. - script_params = { # Param dict with defaults for optional parameters - "File_Annotation": None, - "Namespace (leave blank for default)": omero.constants.metadata.NSCLIENTMAPANNOTATION - } - for key in client.getInputKeys(): - if client.getInput(key): - script_params[key] = client.getInput(key, unwrap=True) - - # Getting rid of the trailing '---' added for the UI - tmp_trg = script_params["Target Data_Type"] - script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg - - if script_params["Data_Type"] == "Tag": - script_params["Data_Type"] = "TagAnnotation" - assert None not in script_params["File_Annotation"], "File annotation ID must be given when using Tag as source" - - if (script_params["File_Annotation"]) is not None and ("," in script_params["File_Annotation"]): # List of ID provided - script_params["File_Annotation"] = script_params["File_Annotation"].split(",") - else: - script_params["File_Annotation"] = [script_params["File_Annotation"]] - if len(script_params["File_Annotation"]) == 1: # Poulate the parameter with None or same ID for all source - script_params["File_Annotation"] = script_params["File_Annotation"] * len(script_params["IDs"]) - assert len(script_params["File_Annotation"]) == len(script_params["IDs"]), "Number of Source IDs and FileAnnotation IDs must match" - - # Replacing the placeholders and with the actual values from the parameters - to_exclude = list(map(lambda x: x.replace('', script_params["Target ID colname"]), - script_params["Columns to exclude"])) - script_params["Columns to exclude"] = list(map(lambda x: x.replace('', script_params["Target name colname"]), - to_exclude)) - - if script_params["Separator"] == "guess": - script_params["Separator"] = None - elif script_params["Separator"] == "TAB": - script_params["Separator"] = "\t" - # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - print("script params") - for k, v in script_params.items(): - print(k, v) - message, robj = keyval_from_csv(conn, script_params) + message, robj = keyval_from_csv(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) - except AssertionError as err: #Display assertion errors in OMERO.web activities + except AssertionError as err: + # Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) raise AssertionError(str(err)) @@ -395,5 +422,55 @@ def run_script(): client.closeSession() +def parameters_parsing(client): + params = OrderedDict() + # Param dict with defaults for optional parameters + params["File_Annotation"] = None + params["Namespace (leave blank for default)"] = NSCLIENTMAPANNOTATION + + for key in client.getInputKeys(): + if client.getInput(key): + params[key] = client.getInput(key, unwrap=True) + + # Getting rid of the trailing '---' added for the UI + if " " in params["Target Data_Type"]: + params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + + if params["Data_Type"] == "Tag": + params["Data_Type"] = "TagAnnotation" + assert None not in params["File_Annotation"], "File annotation \ + ID must be given when using Tag as source" + + if ((params["File_Annotation"]) is not None + and ("," in params["File_Annotation"])): + # List of ID provided, have to do the split + params["File_Annotation"] = params["File_Annotation"].split(",") + else: + params["File_Annotation"] = [int(params["File_Annotation"])] + if len(params["File_Annotation"]) == 1: + # Poulate the parameter with None or same ID for all source + params["File_Annotation"] *= len(params["IDs"]) + params["File_Annotation"] = list(map(int, params["File_Annotation"])) + + assert len(params["File_Annotation"]) == len(params["IDs"]), "Number of \ + Source IDs and FileAnnotation IDs must match" + + # Replacing the placeholders and with values from params + to_exclude = list(map(lambda x: x.replace('', + params["Target ID colname"]), + params["Columns to exclude"])) + to_exclude = list(map(lambda x: x.replace('', + params["Target name colname"]), + to_exclude)) + params["Columns to exclude"] = to_exclude + + if params["Separator"] == "guess": + params["Separator"] = None + elif params["Separator"] == "TAB": + params["Separator"] = "\t" + + return params + + if __name__ == "__main__": run_script() diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 56644918c..1dc8d84d8 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -26,8 +26,8 @@ import omero from omero.gateway import BlitzGateway from omero.rtypes import rstring, rlong, robject +from omero.constants.metadata import NSCLIENTMAPANNOTATION import omero.scripts as scripts -from omero.cmd import Delete2 import tempfile import os @@ -38,12 +38,22 @@ "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - #"Run": ["Well", "Image"], + # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } -ZERO_PADDING = 3 # To allow duplicated keys (3 means up to 1000 duplicate key on a single object) +# To allow duplicated keys +# (3 means up to 1000 same key on a single object) +ZERO_PADDING = 3 + + +def get_obj_name(omero_obj): + if omero_obj.OMERO_CLASS == "Well": + return omero_obj.getWellPos() + else: + return omero_obj.getName() + def get_existing_map_annotions(obj, namespace_l, zero_padding): key_l = [] @@ -51,45 +61,52 @@ def get_existing_map_annotions(obj, namespace_l, zero_padding): for namespace in namespace_l: for ann in obj.listAnnotations(ns=namespace): if isinstance(ann, omero.gateway.MapAnnotationWrapper): - for (k,v) in ann.getValue(): + for (k, v) in ann.getValue(): n_occurence = key_l.count(k) - result[f"{str(n_occurence).rjust(zero_padding, '0')}{k}"] = v - key_l.append(k) # To count the multiple occurence of keys + pad_key = f"{str(n_occurence).rjust(zero_padding, '0')}{k}" + result[pad_key] = v + key_l.append(k) # To count the multiple occurence of keys return result + def group_keyvalue_dictionaries(annotation_dicts, zero_padding): """ Groups the keys and values of each object into a single dictionary """ - all_key = OrderedDict() # To keep the keys in order, for what it's worth + all_key = OrderedDict() # To keep the keys in order, for what it's worth for annotation_dict in annotation_dicts: - all_key.update({k:None for k in annotation_dict.keys()}) + all_key.update({k: None for k in annotation_dict.keys()}) all_key = list(all_key.keys()) result = [] for annotation_dict in annotation_dicts: obj_dict = OrderedDict((k, "") for k in all_key) obj_dict.update(annotation_dict) - for k,v in obj_dict.items(): + for k, v in obj_dict.items(): if v is None: obj_dict[k] result.append(list(obj_dict.values())) - all_key = [key[zero_padding:] for key in all_key] # Removing temporary padding + # Removing temporary padding + all_key = [key[zero_padding:] for key in all_key] return all_key, result + def get_children_recursive(source_object, target_type): - if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: + # Stop condition, we return the source_obj children if source_object.OMERO_CLASS != "WellSample": return source_object.listChildren() else: return [source_object.getImage()] - else: + else: # Not yet the target result = [] for child_obj in source_object.listChildren(): - # Going down in the Hierarchy list for all childs that aren't yet the target + # Going down in the Hierarchy list result.extend(get_children_recursive(child_obj, target_type)) return result -def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well): + +def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, + obj_ancestry_l, annotation_dicts, separator, is_well): def to_csv(ll): """convience function to write a csv line""" nl = len(ll) @@ -101,26 +118,31 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): result_ancestry = [] result_value = [] - tmp_ancestry_l = [] # That's an imbricated list of list of tuples, making it simplier first + # That's an imbricated list of list of tuples, making it simplier first + tmp_ancestry_l = [] for ancestries in obj_ancestry_l: - tmp_ancestry_l.append(["".join(list(obj_name)) for obj_name in ancestries]) + tmp_ancestry_l.append(["".join(list(obj_name)) + for obj_name in ancestries]) start_idx = 0 stop_idx = 1 - while start_idx0: # If there's anything to add at all - obj_name_l, obj_ancestry_l, whole_values_l = sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well) #Sorting relies on parents + if len(obj_ancestry_l) > 0: # If there's anything to add at all + # Only sort when there are parents to group childs + obj_name_l, obj_ancestry_l, whole_values_l = sort_items(obj_name_l, + obj_ancestry_l, + whole_values_l, + is_well) for (parent_type, _) in obj_ancestry_l[0]: - all_key.insert(counter, parent_type); counter += 1 - + all_key.insert(counter, parent_type) + counter += 1 all_key.insert(counter, "target_id") all_key.insert(counter + 1, "target_name") - for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, obj_name_l, whole_values_l)): + print(f"\tColumn names: {all_key}", "\n") + + for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, + obj_name_l, + whole_values_l)): counter = 0 - if len(obj_ancestry_l)>0: # If there's anything to add at all + if len(obj_ancestry_l) > 0: # If there's anything to add at all for (_, parent_name) in obj_ancestry_l[k]: - whole_values.insert(counter, parent_name); counter += 1 + whole_values.insert(counter, parent_name) + counter += 1 whole_values.insert(counter, obj_id) whole_values.insert(counter + 1, obj_name) @@ -159,12 +191,11 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): tfile.write(to_csv(whole_values)) tfile.close() - source_name = source_object.getWellPos() if source_object.OMERO_CLASS == "Well" else source_object.getName() - name = "{}_metadata_out.csv".format(source_name) + name = "{}_metadata_out.csv".format(get_obj_name(source_object)) # link it to the object ann = conn.createFileAnnfromLocalFile( tmp_file, origFilePathAndName=name, - ns='MIF_test') + ns='KeyVal_export') ann = source_object.linkAnnotation(ann) # remove the tmp file @@ -173,6 +204,24 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): return "done" +def target_iterator(conn, source_object, target_type): + source_type = source_object.OMERO_CLASS + if source_type == target_type: + target_obj_l = [source_object] + elif source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, + [source_object.getId()]) + # Need that to load objects + obj_ids = [o.getId() for o in target_obj_l] + target_obj_l = list(conn.getObjects(target_type, obj_ids)) + else: + target_obj_l = get_children_recursive(source_object, + target_type) + + print(f"Iterating objects from {source_object}:") + for target_obj in target_obj_l: + print(f"\t- {target_obj}") + yield target_obj def main_loop(conn, script_params): @@ -194,58 +243,27 @@ def main_loop(conn, script_params): obj_ancestry_l = [] annotation_dicts = [] obj_id_l, obj_name_l = [], [] - if source_type == target_type: - target_obj_l = [source_object] - else: - if source_type == "TagAnnotation": - target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) - target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later - source_object = target_obj_l[0] # Putting the csv file on the first child - else: - target_obj_l = get_children_recursive(source_object, target_type) - # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - for target_obj in target_obj_l: + + for target_obj in target_iterator(conn, source_object, target_type): is_well = target_obj.OMERO_CLASS == "Well" - print("Processing object:", target_obj) - annotation_dicts.append(get_existing_map_annotions(target_obj, namespace_l, ZERO_PADDING)) + annotation_dicts.append(get_existing_map_annotions(target_obj, + namespace_l, + ZERO_PADDING)) obj_id_l.append(target_obj.getId()) - obj_name_l.append(target_obj.getWellPos() if is_well else target_obj.getName()) + obj_name_l.append(get_obj_name(target_obj)) if include_parent: - ancestry = [(o.OMERO_CLASS, o.getWellPos() if o.OMERO_CLASS == "Well" else o.getName()) - for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"] - obj_ancestry_l.append(ancestry[::-1]) # Reverse the order to go from highest to lowest - - message = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well) - - return message, source_object - - # for ds in datasets: - # # name of the file - # csv_name = "{}_metadata_out.csv".format(ds.getName()) - # print(csv_name) - - # # remove the csv if it exists - # for ann in ds.listAnnotations(): - # if(isinstance(ann, omero.gateway.FileAnnotationWrapper)): - # if(ann.getFileName() == csv_name): - # # if the name matches delete it - # try: - # delete = Delete2( - # targetObjects={'FileAnnotation': - # [ann.getId()]}) - # handle = conn.c.sf.submit(delete) - # conn.c.waitOnCmd( - # handle, loops=10, - # ms=500, failonerror=True, - # failontimeout=False, closehandle=False) - # print("Deleted existing csv") - # except Exception as ex: - # print("Failed to delete existing csv: {}".format( - # ex.message)) - # else: - # print("No exisiting file") - - # assemble the metadata into an OrderedDict + ancestry = [(o.OMERO_CLASS, get_obj_name(o)) + for o in target_obj.getAncestry() + if o.OMERO_CLASS != "WellSample"] + obj_ancestry_l.append(ancestry[::-1]) + + message = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, + obj_ancestry_l, annotation_dicts, separator, + is_well) + print("\n------------------------------------\n") + + return message, source_object + def run_script(): """ @@ -253,13 +271,15 @@ def run_script(): scripting service, passing the required parameters. """ + # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), rstring("Tag"), - rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + rstring("Image"), ] - target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script + # Duplicate Image for UI, but not a problem for script + target_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] @@ -278,15 +298,15 @@ def run_script(): \t Parameters: \t - - Data Type: Type of the "parent objects" in which "target objects" are searched. - - IDs: IDs of the "parent objects". - - Target Data Type: Type of the "target objects" of which KV-pairs are exported. - - Namespace: Only annotations having one of these namespace(s) will be exported. + - Data Type: parent-objects type in which target-objects are searched. + - IDs: IDs of the parent-objects. + - Target Data Type: target-objects type from which KV-pairs are exported. + - Namespace: Annotations having one of these namespace(s) will be exported. \t - Separator: Separator to be used in the .csv file. - - Include column(s) of parents name: If checked, add columns for each object in the hierarchy of the target data. + - Include column(s) of parents name: Add columns for target-data parents. \t - """, + """, # Tabs are needed to add line breaks in the HTML scripts.String( "Data_Type", optional=False, grouping="1", @@ -295,7 +315,8 @@ def run_script(): scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent data IDs containing the objects to delete annotation from.").ofType(rlong(0)), + description="List of parent data IDs containing the objects \ + to delete annotation from.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=True, grouping="1.2", @@ -303,12 +324,14 @@ def run_script(): values=target_types, default="-- Image"), scripts.List( - "Namespace (leave blank for default)", optional=True, grouping="1.3", - description="Namespace(s) to include for the export of key-value pairs annotations.").ofType(rstring("")), + "Namespace (leave blank for default)", optional=True, + grouping="1.3", + description="Namespace(s) to include for the export of key-value \ + pairs annotations.").ofType(rstring("")), scripts.Bool( - "Advanced parameters", optional=True, grouping="2", - description="Ticking or unticking this has no effect", default=False), + "Advanced parameters", optional=True, grouping="2", default=False, + description="Ticking or unticking this has no effect"), scripts.String( "Separator", optional=False, grouping="2.1", @@ -316,46 +339,58 @@ def run_script(): values=separators, default=";"), scripts.Bool( - "Include column(s) of parents name", optional=False, grouping="2.2", - description="Weather to include or not the name of the parent(s) objects as columns in the .csv.", default=False), + "Include column(s) of parents name", optional=False, + grouping="2.2", + description="Weather to include or not the name of the parent(s) \ + objects as columns in the .csv.", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", ) + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", + "Namespace (leave blank for default)", + "Separator", "Include column(s) of parents name"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") try: - script_params = { - "Namespace (leave blank for default)": [omero.constants.metadata.NSCLIENTMAPANNOTATION] - } - for key in client.getInputKeys(): - if client.getInput(key): - # unwrap rtypes to String, Integer etc - script_params[key] = client.getInput(key, unwrap=True) - - # Getting rid of the trailing '---' added for the UI - tmp_trg = script_params["Target Data_Type"] - script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg - - if script_params["Separator"] == "TAB": - script_params["Separator"] = "\t" - - print(script_params) # handy to have inputs in the std-out log - # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - - # do the editing... - message, robj = main_loop(conn, script_params) + message, robj = main_loop(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) - except AssertionError as err: #Display assertion errors in OMERO.web activities + except AssertionError as err: + # Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) raise AssertionError(str(err)) finally: client.closeSession() + +def parameters_parsing(client): + params = OrderedDict() + # Param dict with defaults for optional parameters + params["Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] + + for key in client.getInputKeys(): + if client.getInput(key): + # unwrap rtypes to String, Integer etc + params[key] = client.getInput(key, unwrap=True) + + # Getting rid of the trailing '---' added for the UI + if " " in params["Target Data_Type"]: + params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + + if params["Separator"] == "TAB": + params["Separator"] = "\t" + return params + + if __name__ == "__main__": - run_script() \ No newline at end of file + run_script() diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 09af13f79..7c22a5e5a 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -28,29 +28,37 @@ from omero.gateway import BlitzGateway import omero -from omero.rtypes import rlong, rstring, robject, wrap +from omero.rtypes import rlong, rstring, robject +from omero.constants.metadata import NSCLIENTMAPANNOTATION import omero.scripts as scripts +from collections import OrderedDict + CHILD_OBJECTS = { "Project": "Dataset", "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - #"Run": ["Well", "Image"], + # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } +AGREEMENT = "I understand what I am doing and that this will result \ + in a batch deletion of key-value pairs from the server" + + def remove_map_annotations(conn, obj, namespace_l): mapann_ids = [] for namespace in namespace_l: anns = list(obj.listAnnotations(ns=namespace)) mapann_ids.extend([ann.id for ann in anns - if isinstance(ann, omero.gateway.MapAnnotationWrapper)]) + if isinstance(ann, + omero.gateway.MapAnnotationWrapper)]) if len(mapann_ids) == 0: return 0 - print("Map Annotation IDs to delete:", mapann_ids) + print(f"\tMap Annotation IDs to delete: {mapann_ids}\n") try: conn.deleteObjects("Annotation", mapann_ids) return 1 @@ -60,18 +68,41 @@ def remove_map_annotations(conn, obj, namespace_l): def get_children_recursive(source_object, target_type): - if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children + if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: + # Stop condition, we return the source_obj children if source_object.OMERO_CLASS != "WellSample": return source_object.listChildren() else: return [source_object.getImage()] - else: + else: # Not yet the target result = [] for child_obj in source_object.listChildren(): - # Going down in the Hierarchy list for all childs that aren't yet the target + # Going down in the Hierarchy list result.extend(get_children_recursive(child_obj, target_type)) return result + +def target_iterator(conn, source_object, target_type): + source_type = source_object.OMERO_CLASS + if source_type == target_type: + target_obj_l = [source_object] + elif source_type == "TagAnnotation": + target_obj_l = conn.getObjectsByAnnotations(target_type, + [source_object.getId()]) + # Need that to load objects + obj_ids = [o.getId() for o in target_obj_l] + target_obj_l = list(conn.getObjects(target_type, obj_ids)) + else: + target_obj_l = get_children_recursive(source_object, + target_type) + + print(f"Iterating objects from {source_object}:") + for target_obj in target_obj_l: + print(f"\t- {target_obj}") + yield target_obj + print() + + def remove_keyvalue(conn, script_params): """ File the list of objects @@ -88,17 +119,7 @@ def remove_keyvalue(conn, script_params): result_obj = None for source_object in conn.getObjects(source_type, source_ids): - if source_type == target_type: - target_obj_l = [source_object] - else: - # Listing all target children to the source object (eg all images (target) in all datasets of the project (source)) - if source_type == "TagAnnotation": - target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) - target_obj_l = list(conn.getObjects(target_type, [o.getId() for o in target_obj_l])) # Need that to load annotations later - else: - target_obj_l = get_children_recursive(source_object, target_type) - for target_obj in target_obj_l: - print("Processing object:", target_obj) + for target_obj in target_iterator(conn, source_object, target_type): success = remove_map_annotations(conn, target_obj, namespace_l) if success: nsuccess += 1 @@ -106,31 +127,31 @@ def remove_keyvalue(conn, script_params): result_obj = target_obj ntotal += 1 - + print("\n------------------------------------\n") message = f"Key value data deleted from {nsuccess} of {ntotal} objects" return message, result_obj -if __name__ == "__main__": +def run_script(): """ The main entry point of the script, as called by the client via the scripting service, passing the required parameters. """ + # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), rstring("Tag"), - rstring("Image"), # Cannot add fancy layout if we want auto fill and selct of object ID + rstring("Image"), ] - target_types = [rstring("Project"), # Duplicate Image for UI, but not a problem for script + # Duplicate Image for UI, but not a problem for script + target_types = [rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] - agreement = "I understand what I am doing and that this will result in a batch deletion of key-value pairs from the server" - # Here we define the script name and description. # Good practice to put url here to give users more guidance on how to run # your script. @@ -144,21 +165,22 @@ def remove_keyvalue(conn, script_params): \t Parameters: \t - - Data Type: Type of the "parent objects" in which "target objects" are searched. - - IDs: IDs of the "parent objects". - - Target Data Type: Type of the "target objects" of which KV-pairs are deleted. - - Namespace: Only annotations having one of these namespace(s) will be deleted. + - Data Type: parent-objects type in which target-objects are searched. + - IDs: IDs of the parent-objects. + - Target Data Type: Target-objects type of which KV-pairs are deleted. + - Namespace: Annotations having one of these namespace(s) will be deleted. \t - """, + """, # Tabs are needed to add line breaks in the HTML scripts.String( "Data_Type", optional=False, grouping="1", - description="Parent data type of the objects to annotate.", + description="Parent-data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent data IDs containing the objects to delete annotation from.").ofType(rlong(0)), + description="List of parent-data IDs containing the objects \ + to delete annotation from.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=True, grouping="1.2", @@ -166,46 +188,60 @@ def remove_keyvalue(conn, script_params): values=target_types, default="-- Image"), scripts.List( - "Namespace (leave blank for default)", optional=True, grouping="1.3", - default="NAMESPACE TO DELETE", - description="Annotation with these namespace will be deleted.").ofType(rstring("")), + "Namespace (leave blank for default)", optional=True, + grouping="1.3", default="NAMESPACE TO DELETE", + description="Annotation with these namespace will \ + be deleted.").ofType(rstring("")), scripts.Bool( - agreement, optional=False, grouping="2", - description="Make sure that you understood the scope of what will be deleted."), + AGREEMENT, optional=False, grouping="2", + description="Make sure that you understood the scope of \ + what will be deleted."), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", ) + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", + "Namespace (leave blank for default)"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") try: - script_params = { - "Namespace (leave blank for default)": [omero.constants.metadata.NSCLIENTMAPANNOTATION] - } - for key in client.getInputKeys(): - if client.getInput(key): - # unwrap rtypes to String, Integer etc - script_params[key] = client.getInput(key, unwrap=True) - - assert script_params[agreement], "Please confirm that you understood the risks." - - # Getting rid of the trailing '---' added for the UI - tmp_trg = script_params["Target Data_Type"] - script_params["Target Data_Type"] = tmp_trg.split(" ")[1] if " " in tmp_trg else tmp_trg - - print(script_params) # handy to have inputs in the std-out log - # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - - # do the editing... - message, robj = remove_keyvalue(conn, script_params) + message, robj = remove_keyvalue(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) - except AssertionError as err: #Display assertion errors in OMERO.web activities + except AssertionError as err: + # Display assertion errors in OMERO.web activities client.setOutput("ERROR", rstring(err)) raise AssertionError(str(err)) finally: client.closeSession() + + +def parameters_parsing(client): + params = OrderedDict() + # Param dict with defaults for optional parameters + params["Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] + + for key in client.getInputKeys(): + if client.getInput(key): + # unwrap rtypes to String, Integer etc + params[key] = client.getInput(key, unwrap=True) + + assert params[AGREEMENT], "Please confirm that you understood the risks." + + # Getting rid of the trailing '---' added for the UI + if " " in params["Target Data_Type"]: + params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + return params + + +if __name__ == "__main__": + run_script() From 4f376e9dce4b69ffb3400736f225d2115555fec8 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 15:29:28 +0200 Subject: [PATCH 039/127] minor print change --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 3 +-- omero/annotation_scripts/Remove_KeyVal.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 95600fc36..d26e909d3 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -86,7 +86,7 @@ def annotate_object(conn, obj, kv_list, namespace): map_ann.setValue(kv_list) map_ann.save() - print("Map Annotation created", map_ann.id) + print("\tMap Annotation created", map_ann.id) obj.linkAnnotation(map_ann) @@ -108,7 +108,6 @@ def target_iterator(conn, source_object, target_type): for target_obj in target_obj_l: print(f"\t- {target_obj}") yield target_obj - print() def replace_namespace(conn, script_params): diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 7c22a5e5a..be4f1e537 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -100,7 +100,6 @@ def target_iterator(conn, source_object, target_type): for target_obj in target_obj_l: print(f"\t- {target_obj}") yield target_obj - print() def remove_keyvalue(conn, script_params): From a51a9e7ab0c2ad6a518c1d22c3d3488af66a3b24 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 15:42:36 +0200 Subject: [PATCH 040/127] minor output change --- omero/annotation_scripts/KeyVal_to_csv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index 1dc8d84d8..db8fe2604 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -198,10 +198,11 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): ns='KeyVal_export') ann = source_object.linkAnnotation(ann) + print(f"{ann} linked to {source_object}") + # remove the tmp file os.remove(tmp_file) os.rmdir(tmp_dir) - return "done" def target_iterator(conn, source_object, target_type): @@ -257,12 +258,11 @@ def main_loop(conn, script_params): if o.OMERO_CLASS != "WellSample"] obj_ancestry_l.append(ancestry[::-1]) - message = attach_csv_file(conn, source_object, obj_id_l, obj_name_l, - obj_ancestry_l, annotation_dicts, separator, - is_well) + attach_csv_file(conn, source_object, obj_id_l, obj_name_l, + obj_ancestry_l, annotation_dicts, separator, is_well) print("\n------------------------------------\n") - return message, source_object + return "Done", source_object def run_script(): From cec68b42afe13f00a9a245e4717d1593d946bc81 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 15:50:20 +0200 Subject: [PATCH 041/127] replaced object column names to OBJECT_ID and OBJECT_NAME --- omero/annotation_scripts/KeyVal_from_csv.py | 16 +++------------- omero/annotation_scripts/KeyVal_to_csv.py | 6 +++--- omero/annotation_scripts/Remove_KeyVal.py | 6 ++---- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index b585a64f8..0b629cc14 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -2,17 +2,7 @@ """ KeyVal_from_csv.py - Adds key-value pairs to TARGETS on OMERO from a CSV file attached - to a SOURCE container. - SOURCES can be: [Project, Dataset, Screen, Plate, Well] - TARGETS can be: [Dataset, Plate, Well, Image] - The targets are referenced in the CSV file either from their name - (must then be unique, and be called "target_name") or from their - ID (must be called "target_id"). - In the case both are given, the ID will be used. - - Every row corresponds to a set of value to attach to the given - TARGET with the key of the correponding column. + Adds key-value pairs to a target object on OMERO from a CSV file. ----------------------------------------------------------------------------- Copyright (C) 2018 @@ -382,13 +372,13 @@ def run_script(): scripts.String( "Target ID colname", optional=False, grouping="2.3", - default="target_id", + default="OBJECT_ID", description="The column name in the .csv containing the id of the \ objects to annotate. Matches in exclude parameter."), scripts.String( "Target name colname", optional=False, grouping="2.4", - default="target_name", + default="OBJECT_NAME", description="The column name in the .csv containing the name of \ the objects to annotate (used if no column ID is provided or \ found in the .csv). Matches in exclude parameter."), diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/KeyVal_to_csv.py index db8fe2604..7596f49b1 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/KeyVal_to_csv.py @@ -3,7 +3,7 @@ MIF/Key_Val_to_csv.py Reads the metadata associated with the images in a dataset - a creates a csv file attached to dataset + and creates a csv file attached to dataset ----------------------------------------------------------------------------- Copyright (C) 2018 @@ -166,8 +166,8 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): for (parent_type, _) in obj_ancestry_l[0]: all_key.insert(counter, parent_type) counter += 1 - all_key.insert(counter, "target_id") - all_key.insert(counter + 1, "target_name") + all_key.insert(counter, "OBJECT_ID") + all_key.insert(counter + 1, "OBJECT_NAME") print(f"\tColumn names: {all_key}", "\n") for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index be4f1e537..cc64fb21e 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -3,10 +3,8 @@ """ MIF/Key_Value_remove.py" - Remove all key-value pairs from: - * selected image(s) - * selected dataset(s) and the images contained in them - * selected screens(s) and the wells & images contained in them + Remove all key-value pairs associated with a namespace from + objects on OMERO. ----------------------------------------------------------------------------- Copyright (C) 2018 - 2022 From 16340520963b9920daa8d74ca66237063a94d07c Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 26 Oct 2023 16:05:11 +0200 Subject: [PATCH 042/127] permission check on annotation before copy/delete --- .../annotation_scripts/Convert_KeyVal_namespace.py | 10 ++++++++-- omero/annotation_scripts/Remove_KeyVal.py | 14 ++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index d26e909d3..1ca765d97 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -57,11 +57,17 @@ def get_children_recursive(source_object, target_type): def get_existing_map_annotions(obj, namespace_l): keyval_l, ann_l = [], [] + forbidden_deletion = [] for namespace in namespace_l: for ann in obj.listAnnotations(ns=namespace): if isinstance(ann, omero.gateway.MapAnnotationWrapper): - keyval_l.extend([(k, v) for (k, v) in ann.getValue()]) - ann_l.append(ann) + if ann.canEdit(): # If not, skipping it + keyval_l.extend([(k, v) for (k, v) in ann.getValue()]) + ann_l.append(ann) + else: + forbidden_deletion.append(ann.id) + print("\tMap Annotation IDs skipped (not permitted):", + f"{forbidden_deletion}") return keyval_l, ann_l diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index cc64fb21e..d726628f5 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -48,15 +48,21 @@ def remove_map_annotations(conn, obj, namespace_l): mapann_ids = [] + forbidden_deletion = [] for namespace in namespace_l: anns = list(obj.listAnnotations(ns=namespace)) - mapann_ids.extend([ann.id for ann in anns - if isinstance(ann, - omero.gateway.MapAnnotationWrapper)]) + for ann in anns: + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + if ann.canEdit(): # If not, skipping it + mapann_ids.append(ann.id) + else: + forbidden_deletion.append(ann.id) if len(mapann_ids) == 0: return 0 - print(f"\tMap Annotation IDs to delete: {mapann_ids}\n") + print(f"\tMap Annotation IDs to delete: {mapann_ids}") + print("\tMap Annotation IDs skipped (not permitted):", + f"{forbidden_deletion}\n") try: conn.deleteObjects("Annotation", mapann_ids) return 1 From 54dc1663b6c09149cbcb047adc45f82b7428cbe8 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 7 Nov 2023 09:44:24 +0100 Subject: [PATCH 043/127] Bug fix with tag and other improvements --- .../Convert_KeyVal_namespace.py | 37 ++++--- .../{KeyVal_to_csv.py => Export_to_csv.py} | 100 +++++++++++------- omero/annotation_scripts/KeyVal_from_csv.py | 99 +++++++++++------ omero/annotation_scripts/Remove_KeyVal.py | 48 +++++---- 4 files changed, 177 insertions(+), 107 deletions(-) rename omero/annotation_scripts/{KeyVal_to_csv.py => Export_to_csv.py} (82%) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 1ca765d97..4bebc3444 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -1,6 +1,6 @@ # coding=utf-8 """ - Convert_namespace_KeyVal.py + Convert_KeyVal_namespace.py Convert the namespace of objects key-value pairs. ----------------------------------------------------------------------------- @@ -34,7 +34,6 @@ "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } @@ -66,8 +65,9 @@ def get_existing_map_annotions(obj, namespace_l): ann_l.append(ann) else: forbidden_deletion.append(ann.id) - print("\tMap Annotation IDs skipped (not permitted):", - f"{forbidden_deletion}") + if len(forbidden_deletion) > 0: + print("\tMap Annotation IDs skipped (not permitted):", + f"{forbidden_deletion}") return keyval_l, ann_l @@ -96,11 +96,10 @@ def annotate_object(conn, obj, kv_list, namespace): obj.linkAnnotation(map_ann) -def target_iterator(conn, source_object, target_type): - source_type = source_object.OMERO_CLASS - if source_type == target_type: +def target_iterator(conn, source_object, target_type, is_tag): + if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] - elif source_type == "TagAnnotation": + elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) # Need that to load objects @@ -129,8 +128,9 @@ def replace_namespace(conn, script_params): # One file output per given ID for source_object in conn.getObjects(source_type, source_ids): - - for target_obj in target_iterator(conn, source_object, target_type): + is_tag = source_type == "TagAnnotation" + for target_obj in target_iterator(conn, source_object, + target_type, is_tag): ntarget_processed += 1 keyval_l, ann_l = get_existing_map_annotions(target_obj, old_namespace) @@ -154,18 +154,17 @@ def run_script(): # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Tag"), - rstring("Image"), + rstring("Well"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script - target_types = [rstring("Project"), + target_types = [rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] client = scripts.client( - 'Convert_KeyVal_namespace', + 'Convert_KV_namespace', """ This script converts the namespace of key-value pair annotations. @@ -195,7 +194,7 @@ def run_script(): "Target Data_Type", optional=False, grouping="1.2", description="The data type for which key-value pair annotations \ will be converted.", - values=target_types, default="-- Image"), + values=target_types, default=""), scripts.List( "Old Namespace (leave blank for default)", optional=True, @@ -239,7 +238,7 @@ def run_script(): def parameters_parsing(client): - params = OrderedDict() + params = {} # Param dict with defaults for optional parameters params["File_Annotation"] = None params["Old Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] @@ -250,7 +249,11 @@ def parameters_parsing(client): params[key] = client.getInput(key, unwrap=True) # Getting rid of the trailing '---' added for the UI - if " " in params["Target Data_Type"]: + if params["Target Data_Type"] == "": + assert params["Data_Type"] != "Tag", ("Choose a Target type " + + "with 'Tag' as Data Type ") + params["Target Data_Type"] = params["Data_Type"] + elif " " in params["Target Data_Type"]: params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] if params["Data_Type"] == "Tag": diff --git a/omero/annotation_scripts/KeyVal_to_csv.py b/omero/annotation_scripts/Export_to_csv.py similarity index 82% rename from omero/annotation_scripts/KeyVal_to_csv.py rename to omero/annotation_scripts/Export_to_csv.py index 7596f49b1..fcaf0b2fb 100644 --- a/omero/annotation_scripts/KeyVal_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -1,6 +1,6 @@ # coding=utf-8 """ - MIF/Key_Val_to_csv.py + Export_to_csv.py Reads the metadata associated with the images in a dataset and creates a csv file attached to dataset @@ -38,7 +38,6 @@ "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } @@ -46,6 +45,7 @@ # To allow duplicated keys # (3 means up to 1000 same key on a single object) ZERO_PADDING = 3 +WEBCLIENT_URL = "https://omero-cai-test.hhu.de/webclient" def get_obj_name(omero_obj): @@ -55,7 +55,7 @@ def get_obj_name(omero_obj): return omero_obj.getName() -def get_existing_map_annotions(obj, namespace_l, zero_padding): +def get_existing_map_annotions(obj, namespace_l): key_l = [] result = OrderedDict() for namespace in namespace_l: @@ -63,13 +63,13 @@ def get_existing_map_annotions(obj, namespace_l, zero_padding): if isinstance(ann, omero.gateway.MapAnnotationWrapper): for (k, v) in ann.getValue(): n_occurence = key_l.count(k) - pad_key = f"{str(n_occurence).rjust(zero_padding, '0')}{k}" + pad_key = f"{str(n_occurence).rjust(ZERO_PADDING, '0')}{k}" result[pad_key] = v key_l.append(k) # To count the multiple occurence of keys return result -def group_keyvalue_dictionaries(annotation_dicts, zero_padding): +def group_keyvalue_dictionaries(annotation_dicts): """ Groups the keys and values of each object into a single dictionary """ all_key = OrderedDict() # To keep the keys in order, for what it's worth for annotation_dict in annotation_dicts: @@ -86,7 +86,7 @@ def group_keyvalue_dictionaries(annotation_dicts, zero_padding): result.append(list(obj_dict.values())) # Removing temporary padding - all_key = [key[zero_padding:] for key in all_key] + all_key = [key[ZERO_PADDING:] for key in all_key] return all_key, result @@ -105,7 +105,7 @@ def get_children_recursive(source_object, target_type): return result -def attach_csv_file(conn, source_object, obj_id_l, obj_name_l, +def attach_csv_file(conn, obj_, csv_name, obj_id_l, obj_name_l, obj_ancestry_l, annotation_dicts, separator, is_well): def to_csv(ll): """convience function to write a csv line""" @@ -152,8 +152,7 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): return result_name, result_ancestry, result_value - all_key, whole_values_l = group_keyvalue_dictionaries(annotation_dicts, - ZERO_PADDING) + all_key, whole_values_l = group_keyvalue_dictionaries(annotation_dicts) counter = 0 @@ -191,25 +190,25 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): tfile.write(to_csv(whole_values)) tfile.close() - name = "{}_metadata_out.csv".format(get_obj_name(source_object)) # link it to the object - ann = conn.createFileAnnfromLocalFile( - tmp_file, origFilePathAndName=name, + file_ann = conn.createFileAnnfromLocalFile( + tmp_file, origFilePathAndName=csv_name, ns='KeyVal_export') - ann = source_object.linkAnnotation(ann) + obj_.linkAnnotation(file_ann) - print(f"{ann} linked to {source_object}") + print(f"{file_ann} linked to {obj_}") # remove the tmp file os.remove(tmp_file) os.rmdir(tmp_dir) + return file_ann.getFile() -def target_iterator(conn, source_object, target_type): - source_type = source_object.OMERO_CLASS - if source_type == target_type: + +def target_iterator(conn, source_object, target_type, is_tag): + if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] - elif source_type == "TagAnnotation": + elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) # Need that to load objects @@ -244,12 +243,16 @@ def main_loop(conn, script_params): obj_ancestry_l = [] annotation_dicts = [] obj_id_l, obj_name_l = [], [] + result_obj = source_object + if source_type == "TagAnnotation": + result_obj = None # Attach result csv on the first object + is_tag = source_type == "TagAnnotation" - for target_obj in target_iterator(conn, source_object, target_type): + for target_obj in target_iterator(conn, source_object, + target_type, is_tag): is_well = target_obj.OMERO_CLASS == "Well" annotation_dicts.append(get_existing_map_annotions(target_obj, - namespace_l, - ZERO_PADDING)) + namespace_l)) obj_id_l.append(target_obj.getId()) obj_name_l.append(get_obj_name(target_obj)) if include_parent: @@ -257,12 +260,21 @@ def main_loop(conn, script_params): for o in target_obj.getAncestry() if o.OMERO_CLASS != "WellSample"] obj_ancestry_l.append(ancestry[::-1]) + if result_obj is None: + result_obj = target_obj - attach_csv_file(conn, source_object, obj_id_l, obj_name_l, - obj_ancestry_l, annotation_dicts, separator, is_well) + csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) + file_ann = attach_csv_file(conn, result_obj, csv_name, obj_id_l, + obj_name_l, obj_ancestry_l, + annotation_dicts, separator, is_well) print("\n------------------------------------\n") - return "Done", source_object + if len(res_obj_l) == 1: + message = f"The csv is attached to {res_obj_l[0].OMERO_CLASS}:{res_obj_l[0].getId()}" + else: + message = ("The csv are attached to " + + ", ".join(map(lambda x: f"{x.OMERO_CLASS}:{x.getId()}", res_obj_l))) + return message, file_ann, res_obj_l[0] def run_script(): @@ -274,12 +286,11 @@ def run_script(): # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Tag"), - rstring("Image"), + rstring("Well"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script - target_types = [rstring("Project"), + target_types = [rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] @@ -289,7 +300,7 @@ def run_script(): # Good practice to put url here to give users more guidance on how to run # your script. client = scripts.client( - 'KeyVal_to_csv.py', + 'Export_KV_to_csv.py', """ This script exports key-value pairs of objects to a .csv file. Can also export a blank .csv with only of target objects' name and IDs. @@ -321,13 +332,15 @@ def run_script(): scripts.String( "Target Data_Type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", - values=target_types, default="-- Image"), + values=target_types, default=""), scripts.List( "Namespace (leave blank for default)", optional=True, grouping="1.3", - description="Namespace(s) to include for the export of key-value \ - pairs annotations.").ofType(rstring("")), + description="Namespace(s) to include for the export of key-" + + "value pairs annotations. Default is the client" + + "namespace, meaning editable in " + + "OMERO.web").ofType(rstring("")), scripts.Bool( "Advanced parameters", optional=True, grouping="2", default=False, @@ -360,10 +373,18 @@ def run_script(): try: # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - message, robj = main_loop(conn, params) + message, fileann, res_obj = main_loop(conn, params) client.setOutput("Message", rstring(message)) - if robj is not None: - client.setOutput("Result", robject(robj._obj)) + + if WEBCLIENT_URL != "": + url = omero.rtypes.wrap({ + "type": "URL", + "href": f"{WEBCLIENT_URL}/download_original_file/{fileann.getId()}", + "title": "CSV file of Key-Value pairs", + }) + client.setOutput("URL", url) + elif res_obj is not None: + client.setOutput("Result", robject(res_obj)) except AssertionError as err: # Display assertion errors in OMERO.web activities @@ -374,7 +395,7 @@ def run_script(): def parameters_parsing(client): - params = OrderedDict() + params = {} # Param dict with defaults for optional parameters params["Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] @@ -384,11 +405,18 @@ def parameters_parsing(client): params[key] = client.getInput(key, unwrap=True) # Getting rid of the trailing '---' added for the UI - if " " in params["Target Data_Type"]: + if params["Target Data_Type"] == "": + assert params["Data_Type"] != "Tag", ("Choose a Target type " + + "with 'Tag' as Data Type ") + params["Target Data_Type"] = params["Data_Type"] + elif " " in params["Target Data_Type"]: params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] if params["Separator"] == "TAB": params["Separator"] = "\t" + + if params["Data_Type"] == "Tag": + params["Data_Type"] = "TagAnnotation" return params diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 0b629cc14..06ecd54da 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -27,20 +27,41 @@ from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts from omero.constants.metadata import NSCLIENTMAPANNOTATION +from omero.util.populate_roi import DownloadingOriginalFileProvider import sys import csv from math import floor from collections import OrderedDict -from omero.util.populate_roi import DownloadingOriginalFileProvider +# source_object = conn.getObject("Acquisition", oid=51) +# target_type = "Image" +# if source_object.OMERO_CLASS == "PlateAcquisition": +# wellsamp_l = get_children_recursive(source_object.getParent(), "WellSample") +# wellsamp_l = list(filter(lambda x: x.getPlateAcquisition() == source_object, wellsamp_l)) +# image_l = [wellsamp.getImage() for wellsamp in wellsamp_l] +# def get_children_recursive(source_object, target_type): +# if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: +# # Stop condition, we return the source_obj children +# if source_object.OMERO_CLASS != "WellSample": +# return source_object.listChildren() +# elif target_type == "WellSample": +# return [source_object] +# else: +# return [source_object.getImage()] +# else: # Not yet the target +# result = [] +# for child_obj in source_object.listChildren(): +# # Going down in the Hierarchy list +# result.extend(get_children_recursive(child_obj, target_type)) +# return result + CHILD_OBJECTS = { "Project": "Dataset", "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } @@ -148,11 +169,10 @@ def get_children_recursive(source_object, target_type): return result -def target_iterator(conn, source_object, target_type): - source_type = source_object.OMERO_CLASS - if source_type == target_type: +def target_iterator(conn, source_object, target_type, is_tag): + if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] - elif source_type == "TagAnnotation": + elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) # Need that to load objects @@ -199,8 +219,9 @@ def keyval_from_csv(conn, script_params): original_file = file_ann.getFile()._obj data, header = read_csv(conn, original_file, separator) - - target_obj_l = target_iterator(conn, source_object, target_type) + is_tag = source_type == "TagAnnotation" + target_obj_l = target_iterator(conn, source_object, + target_type, is_tag) # Index of the column used to identify the targets. Try for IDs first idx_id, idx_name = -1, -1 @@ -289,13 +310,12 @@ def run_script(): source_types = [ rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Tag"), - rstring("Image"), + rstring("Well"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script target_types = [ - rstring("Project"), + rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image") @@ -304,7 +324,7 @@ def run_script(): separators = ["guess", ";", ",", "TAB"] client = scripts.client( - 'Add_Key_Val_from_csv', + 'Import_KV_from_csv', """ Reads a .csv file to annotate target objects with key-value pairs. TODO: add hyperlink to readthedocs @@ -331,27 +351,28 @@ def run_script(): scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent-data IDs containing \ - the objects to annotate.").ofType(rlong(0)), + description="List of parent-data IDs containing" + + " the objects to annotate.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=False, grouping="1.2", - description="The data type which will be annotated. \ - Entries in the .csv correspond to these objects.", - values=target_types, default="-- Image"), + description="The data type which will be annotated. " + + "Entries in the .csv correspond to these objects.", + values=target_types, default=""), scripts.String( "File_Annotation", optional=True, grouping="1.3", - description="If no file is provided, list of file IDs containing \ - metadata to populate (must match length of 'IDs'). If \ - neither, searches the most recently attached CSV \ - file on each parent object."), + description="If no file is provided, list of file IDs " + + "containing metadata to populate (must match length" + + " of 'IDs'). If neither, searches the most recently " + + "attached CSV file on each parent object."), scripts.String( "Namespace (leave blank for default)", optional=True, grouping="1.4", - description="Namespace given to the created key-value \ - pairs annotations."), + description="Namespace given to the created key-value " + + "pairs annotations. Default is the client" + + "namespace, meaning editable in OMERO.web"), scripts.Bool( "Advanced parameters", optional=True, grouping="2", default=False, @@ -359,29 +380,33 @@ def run_script(): scripts.String( "Separator", optional=False, grouping="2.1", - description="The separator used in the .csv file. 'guess' will \ - attempt to detetect automatically which of ,;\\t is used.", + description="The separator used in the .csv file. 'guess' will " + + "attempt to detetect automatically which of " + + ",;\\t is used.", values=separators, default="guess"), scripts.List( "Columns to exclude", optional=False, grouping="2.2", default=",", - description="List of columns in the .csv file to exclude from the \ - key-value pair import. and correspond to the \ - two following parameters.").ofType(rstring("")), + description="List of columns in the .csv file to exclude " + + "from the key-value pair import. " + + " and correspond to the two " + + "following parameters.").ofType(rstring("")), scripts.String( "Target ID colname", optional=False, grouping="2.3", default="OBJECT_ID", - description="The column name in the .csv containing the id of the \ - objects to annotate. Matches in exclude parameter."), + description="The column name in the .csv containing the id" + + " of the objects to annotate. " + + "Matches in exclude parameter."), scripts.String( "Target name colname", optional=False, grouping="2.4", default="OBJECT_NAME", - description="The column name in the .csv containing the name of \ - the objects to annotate (used if no column ID is provided or \ - found in the .csv). Matches in exclude parameter."), + description="The column name in the .csv containing the name of " + + "the objects to annotate (used if no column " + + "ID is provided or found in the .csv). Matches " + + " in exclude parameter."), authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], @@ -413,7 +438,7 @@ def run_script(): def parameters_parsing(client): - params = OrderedDict() + params = {} # Param dict with defaults for optional parameters params["File_Annotation"] = None params["Namespace (leave blank for default)"] = NSCLIENTMAPANNOTATION @@ -423,7 +448,11 @@ def parameters_parsing(client): params[key] = client.getInput(key, unwrap=True) # Getting rid of the trailing '---' added for the UI - if " " in params["Target Data_Type"]: + if params["Target Data_Type"] == "": + assert params["Data_Type"] != "Tag", ("Choose a Target type " + + "with 'Tag' as Data Type ") + params["Target Data_Type"] = params["Data_Type"] + elif " " in params["Target Data_Type"]: params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] if params["Data_Type"] == "Tag": diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index d726628f5..3d515cc73 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -29,6 +29,7 @@ from omero.rtypes import rlong, rstring, robject from omero.constants.metadata import NSCLIENTMAPANNOTATION import omero.scripts as scripts +from omero.model import TagAnnotationI from collections import OrderedDict @@ -37,7 +38,6 @@ "Dataset": "Image", "Screen": "Plate", "Plate": "Well", - # "Run": ["Well", "Image"], "Well": "WellSample", "WellSample": "Image" } @@ -61,8 +61,9 @@ def remove_map_annotations(conn, obj, namespace_l): if len(mapann_ids) == 0: return 0 print(f"\tMap Annotation IDs to delete: {mapann_ids}") - print("\tMap Annotation IDs skipped (not permitted):", - f"{forbidden_deletion}\n") + if len(forbidden_deletion) > 0: + print("\tMap Annotation IDs skipped (not permitted):", + f"{forbidden_deletion}\n") try: conn.deleteObjects("Annotation", mapann_ids) return 1 @@ -86,11 +87,10 @@ def get_children_recursive(source_object, target_type): return result -def target_iterator(conn, source_object, target_type): - source_type = source_object.OMERO_CLASS - if source_type == target_type: +def target_iterator(conn, source_object, target_type, is_tag): + if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] - elif source_type == "TagAnnotation": + elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) # Need that to load objects @@ -122,7 +122,9 @@ def remove_keyvalue(conn, script_params): result_obj = None for source_object in conn.getObjects(source_type, source_ids): - for target_obj in target_iterator(conn, source_object, target_type): + is_tag = source_type == "TagAnnotation" + for target_obj in target_iterator(conn, source_object, + target_type, is_tag): success = remove_map_annotations(conn, target_obj, namespace_l) if success: nsuccess += 1 @@ -145,12 +147,11 @@ def run_script(): # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Tag"), - rstring("Image"), + rstring("Well"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script - target_types = [rstring("Project"), + target_types = [rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), rstring("-- Well"), rstring("--- Image")] @@ -159,7 +160,7 @@ def run_script(): # Good practice to put url here to give users more guidance on how to run # your script. client = scripts.client( - 'Remove_Key_Value.py', + 'Remove_KV.py', """ This script deletes key-value pairs of all child objects founds. Only key-value pairs of the namespace are deleted. @@ -188,13 +189,15 @@ def run_script(): scripts.String( "Target Data_Type", optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", - values=target_types, default="-- Image"), + values=target_types, default=""), scripts.List( "Namespace (leave blank for default)", optional=True, - grouping="1.3", default="NAMESPACE TO DELETE", - description="Annotation with these namespace will \ - be deleted.").ofType(rstring("")), + grouping="1.3", + description="Annotation with these namespace will " + + "be deleted. Default is the client" + + " namespace, meaning editable in " + + "OMERO.web").ofType(rstring("")), scripts.Bool( AGREEMENT, optional=False, grouping="2", @@ -229,7 +232,7 @@ def run_script(): def parameters_parsing(client): - params = OrderedDict() + params = {} # Param dict with defaults for optional parameters params["Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] @@ -241,10 +244,17 @@ def parameters_parsing(client): assert params[AGREEMENT], "Please confirm that you understood the risks." # Getting rid of the trailing '---' added for the UI - if " " in params["Target Data_Type"]: + if params["Target Data_Type"] == "": + assert params["Data_Type"] != "Tag", ("Choose a Target type " + + "with 'Tag' as Data Type ") + params["Target Data_Type"] = params["Data_Type"] + elif " " in params["Target Data_Type"]: params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] - return params + if params["Data_Type"] == "Tag": + params["Data_Type"] = "TagAnnotation" + + return params if __name__ == "__main__": run_script() From b21c77888bc61ab0c0935f604993bfc98a501d94 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 7 Nov 2023 09:54:21 +0100 Subject: [PATCH 044/127] Export csv now always generate a single csv file --- omero/annotation_scripts/Export_to_csv.py | 26 +++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index fcaf0b2fb..953721e30 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -239,10 +239,11 @@ def main_loop(conn, script_params): is_well = False # One file output per given ID + obj_ancestry_l = [] + annotation_dicts = [] + obj_id_l, obj_name_l = [], [] for source_object in conn.getObjects(source_type, source_ids): - obj_ancestry_l = [] - annotation_dicts = [] - obj_id_l, obj_name_l = [], [] + result_obj = source_object if source_type == "TagAnnotation": result_obj = None # Attach result csv on the first object @@ -262,19 +263,16 @@ def main_loop(conn, script_params): obj_ancestry_l.append(ancestry[::-1]) if result_obj is None: result_obj = target_obj - - csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) - file_ann = attach_csv_file(conn, result_obj, csv_name, obj_id_l, - obj_name_l, obj_ancestry_l, - annotation_dicts, separator, is_well) print("\n------------------------------------\n") + csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) + file_ann = attach_csv_file(conn, result_obj, csv_name, obj_id_l, + obj_name_l, obj_ancestry_l, + annotation_dicts, separator, is_well) - if len(res_obj_l) == 1: - message = f"The csv is attached to {res_obj_l[0].OMERO_CLASS}:{res_obj_l[0].getId()}" - else: - message = ("The csv are attached to " + - ", ".join(map(lambda x: f"{x.OMERO_CLASS}:{x.getId()}", res_obj_l))) - return message, file_ann, res_obj_l[0] + message = ("The csv is attached to " + + f"{result_obj.OMERO_CLASS}:{result_obj.getId()}") + + return message, file_ann, result_obj def run_script(): From 18b827ab4b361670123a77b1277123baaa4925af Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 7 Nov 2023 10:07:06 +0100 Subject: [PATCH 045/127] fixed inline string comment --- .../annotation_scripts/Convert_KeyVal_namespace.py | 12 ++++++------ omero/annotation_scripts/Export_to_csv.py | 13 +++++++------ omero/annotation_scripts/Remove_KeyVal.py | 13 +++++++------ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 4bebc3444..b1171a819 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -187,20 +187,20 @@ def run_script(): scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent-data IDs containing the objects \ - to annotate.").ofType(rlong(0)), + description="List of parent-data IDs containing the objects " + + "to annotate.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=False, grouping="1.2", - description="The data type for which key-value pair annotations \ - will be converted.", + description="The data type for which key-value pair annotations " + + "will be converted.", values=target_types, default=""), scripts.List( "Old Namespace (leave blank for default)", optional=True, grouping="1.4", - description="The namespace(s) of the annotations to \ - group and change.").ofType(rstring("")), + description="The namespace(s) of the annotations to " + + "group and change.").ofType(rstring("")), scripts.String( "New Namespace (leave blank for default)", optional=True, diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 953721e30..f95159db2 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -324,8 +324,8 @@ def run_script(): scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent data IDs containing the objects \ - to delete annotation from.").ofType(rlong(0)), + description="List of parent data IDs containing the objects " + + "to delete annotation from.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=True, grouping="1.2", @@ -352,8 +352,8 @@ def run_script(): scripts.Bool( "Include column(s) of parents name", optional=False, grouping="2.2", - description="Weather to include or not the name of the parent(s) \ - objects as columns in the .csv.", default=False), + description="Weather to include or not the name of the parent(s)" + + " objects as columns in the .csv.", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], @@ -374,14 +374,15 @@ def run_script(): message, fileann, res_obj = main_loop(conn, params) client.setOutput("Message", rstring(message)) + href = f"{WEBCLIENT_URL}/download_original_file/{fileann.getId()}" if WEBCLIENT_URL != "": url = omero.rtypes.wrap({ "type": "URL", - "href": f"{WEBCLIENT_URL}/download_original_file/{fileann.getId()}", + "href": href, "title": "CSV file of Key-Value pairs", }) client.setOutput("URL", url) - elif res_obj is not None: + else: client.setOutput("Result", robject(res_obj)) except AssertionError as err: diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 3d515cc73..bf14d2cdd 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -42,8 +42,8 @@ "WellSample": "Image" } -AGREEMENT = "I understand what I am doing and that this will result \ - in a batch deletion of key-value pairs from the server" +AGREEMENT = ("I understand what I am doing and that this will result " + + "in a batch deletion of key-value pairs from the server") def remove_map_annotations(conn, obj, namespace_l): @@ -183,8 +183,8 @@ def run_script(): scripts.List( "IDs", optional=False, grouping="1.1", - description="List of parent-data IDs containing the objects \ - to delete annotation from.").ofType(rlong(0)), + description="List of parent-data IDs containing the objects " + + "to delete annotation from.").ofType(rlong(0)), scripts.String( "Target Data_Type", optional=True, grouping="1.2", @@ -201,8 +201,8 @@ def run_script(): scripts.Bool( AGREEMENT, optional=False, grouping="2", - description="Make sure that you understood the scope of \ - what will be deleted."), + description="Make sure that you understood the scope of " + + "what will be deleted."), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], @@ -256,5 +256,6 @@ def parameters_parsing(client): return params + if __name__ == "__main__": run_script() From 60ef4331d8f3ec15d2c19e11c065dd87e26385d0 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 7 Nov 2023 14:48:59 +0100 Subject: [PATCH 046/127] output condition on object not null --- omero/annotation_scripts/Export_to_csv.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index f95159db2..5dc19ec86 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -45,7 +45,7 @@ # To allow duplicated keys # (3 means up to 1000 same key on a single object) ZERO_PADDING = 3 -WEBCLIENT_URL = "https://omero-cai-test.hhu.de/webclient" +WEBCLIENT_URL = "" def get_obj_name(omero_obj): @@ -375,15 +375,16 @@ def run_script(): client.setOutput("Message", rstring(message)) href = f"{WEBCLIENT_URL}/download_original_file/{fileann.getId()}" - if WEBCLIENT_URL != "": - url = omero.rtypes.wrap({ - "type": "URL", - "href": href, - "title": "CSV file of Key-Value pairs", - }) - client.setOutput("URL", url) - else: - client.setOutput("Result", robject(res_obj)) + if res_obj is not None and fileann is not None: + if WEBCLIENT_URL != "": + url = omero.rtypes.wrap({ + "type": "URL", + "href": href, + "title": "CSV file of Key-Value pairs", + }) + client.setOutput("URL", url) + else: + client.setOutput("Result", robject(res_obj)) except AssertionError as err: # Display assertion errors in OMERO.web activities From 3118d63a2a5842ea1cefb102f2d7cfa7019e4b29 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 7 Nov 2023 15:29:56 +0100 Subject: [PATCH 047/127] Added parent target parameter validation --- .../Convert_KeyVal_namespace.py | 51 +++++++++++------- omero/annotation_scripts/Export_to_csv.py | 52 +++++++++++------- omero/annotation_scripts/KeyVal_from_csv.py | 53 ++++++++++++------- omero/annotation_scripts/Remove_KeyVal.py | 50 ++++++++++------- 4 files changed, 133 insertions(+), 73 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index b1171a819..a6af9eefa 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -30,13 +30,25 @@ from collections import OrderedDict CHILD_OBJECTS = { - "Project": "Dataset", - "Dataset": "Image", - "Screen": "Plate", - "Plate": "Well", - "Well": "WellSample", - "WellSample": "Image" - } + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + "Well": "WellSample", + "WellSample": "Image" +} + +ALLOWED_PARAM = { + "Project": ["Project", "Dataset", "Image"], + "Dataset": ["Dataset", "Image"], + "Image": ["Image"], + "Screen": ["Screen", "Plate", "Well", "Run", "Image"], + "Plate": ["Plate", "Well", "Run", "Image"], + "Well": ["Well", "Image"], + "Run": ["Run", "Image"], + "Tag": ["Project", "Dataset", "Image", + "Screen", "Plate", "Well", "Run"] +} def get_children_recursive(source_object, target_type): @@ -212,15 +224,16 @@ def run_script(): contact="https://forum.image.sc/tag/omero" ) - params = parameters_parsing(client) - print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", - "Old Namespace (leave blank for default)", - "New Namespace (leave blank for default)"] - for k in keys: - print(f"\t- {k}: {params[k]}") - print("\n####################################\n") try: + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", + "Old Namespace (leave blank for default)", + "New Namespace (leave blank for default)"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") + # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) message, robj = replace_namespace(conn, params) @@ -248,14 +261,16 @@ def parameters_parsing(client): if client.getInput(key): params[key] = client.getInput(key, unwrap=True) - # Getting rid of the trailing '---' added for the UI if params["Target Data_Type"] == "": - assert params["Data_Type"] != "Tag", ("Choose a Target type " + - "with 'Tag' as Data Type ") params["Target Data_Type"] = params["Data_Type"] elif " " in params["Target Data_Type"]: + # Getting rid of the trailing '---' added for the UI params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + (f"{params['Target Data_Type']} is not a valid target for " + + f"{params['Data_Type']}.") + if params["Data_Type"] == "Tag": params["Data_Type"] = "TagAnnotation" diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 5dc19ec86..b8801f575 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -34,13 +34,25 @@ from collections import OrderedDict CHILD_OBJECTS = { - "Project": "Dataset", - "Dataset": "Image", - "Screen": "Plate", - "Plate": "Well", - "Well": "WellSample", - "WellSample": "Image" - } + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + "Well": "WellSample", + "WellSample": "Image" +} + +ALLOWED_PARAM = { + "Project": ["Project", "Dataset", "Image"], + "Dataset": ["Dataset", "Image"], + "Image": ["Image"], + "Screen": ["Screen", "Plate", "Well", "Run", "Image"], + "Plate": ["Plate", "Well", "Run", "Image"], + "Well": ["Well", "Image"], + "Run": ["Run", "Image"], + "Tag": ["Project", "Dataset", "Image", + "Screen", "Plate", "Well", "Run"] +} # To allow duplicated keys # (3 means up to 1000 same key on a single object) @@ -359,16 +371,16 @@ def run_script(): institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", ) - - params = parameters_parsing(client) - print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", - "Namespace (leave blank for default)", - "Separator", "Include column(s) of parents name"] - for k in keys: - print(f"\t- {k}: {params[k]}") - print("\n####################################\n") try: + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", + "Namespace (leave blank for default)", + "Separator", "Include column(s) of parents name"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") + # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) message, fileann, res_obj = main_loop(conn, params) @@ -404,14 +416,16 @@ def parameters_parsing(client): # unwrap rtypes to String, Integer etc params[key] = client.getInput(key, unwrap=True) - # Getting rid of the trailing '---' added for the UI if params["Target Data_Type"] == "": - assert params["Data_Type"] != "Tag", ("Choose a Target type " + - "with 'Tag' as Data Type ") params["Target Data_Type"] = params["Data_Type"] elif " " in params["Target Data_Type"]: + # Getting rid of the trailing '---' added for the UI params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + (f"{params['Target Data_Type']} is not a valid target for " + + f"{params['Data_Type']}.") + if params["Separator"] == "TAB": params["Separator"] = "\t" diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 06ecd54da..2edfda909 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -58,13 +58,25 @@ CHILD_OBJECTS = { - "Project": "Dataset", - "Dataset": "Image", - "Screen": "Plate", - "Plate": "Well", - "Well": "WellSample", - "WellSample": "Image" - } + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + "Well": "WellSample", + "WellSample": "Image" +} + +ALLOWED_PARAM = { + "Project": ["Project", "Dataset", "Image"], + "Dataset": ["Dataset", "Image"], + "Image": ["Image"], + "Screen": ["Screen", "Plate", "Well", "Run", "Image"], + "Plate": ["Plate", "Well", "Run", "Image"], + "Well": ["Well", "Image"], + "Run": ["Run", "Image"], + "Tag": ["Project", "Dataset", "Image", + "Screen", "Plate", "Well", "Run"] +} def get_obj_name(omero_obj): @@ -412,15 +424,18 @@ def run_script(): institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" ) - params = parameters_parsing(client) - print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", - "Namespace (leave blank for default)", "Separator", - "Columns to exclude", "Target ID colname", "Target name colname"] - for k in keys: - print(f"\t- {k}: {params[k]}") - print("\n####################################\n") + try: + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", + "Namespace (leave blank for default)", "Separator", + "Columns to exclude", "Target ID colname", + "Target name colname"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") + # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) message, robj = keyval_from_csv(conn, params) @@ -447,14 +462,16 @@ def parameters_parsing(client): if client.getInput(key): params[key] = client.getInput(key, unwrap=True) - # Getting rid of the trailing '---' added for the UI if params["Target Data_Type"] == "": - assert params["Data_Type"] != "Tag", ("Choose a Target type " + - "with 'Tag' as Data Type ") params["Target Data_Type"] = params["Data_Type"] elif " " in params["Target Data_Type"]: + # Getting rid of the trailing '---' added for the UI params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + (f"{params['Target Data_Type']} is not a valid target for " + + f"{params['Data_Type']}.") + if params["Data_Type"] == "Tag": params["Data_Type"] = "TagAnnotation" assert None not in params["File_Annotation"], "File annotation \ diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index bf14d2cdd..9d141092c 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -34,13 +34,25 @@ from collections import OrderedDict CHILD_OBJECTS = { - "Project": "Dataset", - "Dataset": "Image", - "Screen": "Plate", - "Plate": "Well", - "Well": "WellSample", - "WellSample": "Image" - } + "Project": "Dataset", + "Dataset": "Image", + "Screen": "Plate", + "Plate": "Well", + "Well": "WellSample", + "WellSample": "Image" +} + +ALLOWED_PARAM = { + "Project": ["Project", "Dataset", "Image"], + "Dataset": ["Dataset", "Image"], + "Image": ["Image"], + "Screen": ["Screen", "Plate", "Well", "Run", "Image"], + "Plate": ["Plate", "Well", "Run", "Image"], + "Well": ["Well", "Image"], + "Run": ["Run", "Image"], + "Tag": ["Project", "Dataset", "Image", + "Screen", "Plate", "Well", "Run"] +} AGREEMENT = ("I understand what I am doing and that this will result " + "in a batch deletion of key-value pairs from the server") @@ -208,15 +220,15 @@ def run_script(): institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", ) - - params = parameters_parsing(client) - print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", - "Namespace (leave blank for default)"] - for k in keys: - print(f"\t- {k}: {params[k]}") - print("\n####################################\n") try: + params = parameters_parsing(client) + print("Input parameters:") + keys = ["Data_Type", "IDs", "Target Data_Type", + "Namespace (leave blank for default)"] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") + # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) message, robj = remove_keyvalue(conn, params) @@ -243,14 +255,16 @@ def parameters_parsing(client): assert params[AGREEMENT], "Please confirm that you understood the risks." - # Getting rid of the trailing '---' added for the UI if params["Target Data_Type"] == "": - assert params["Data_Type"] != "Tag", ("Choose a Target type " + - "with 'Tag' as Data Type ") params["Target Data_Type"] = params["Data_Type"] elif " " in params["Target Data_Type"]: + # Getting rid of the trailing '---' added for the UI params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + (f"{params['Target Data_Type']} is not a valid target for " + + f"{params['Data_Type']}.") + if params["Data_Type"] == "Tag": params["Data_Type"] = "TagAnnotation" From 70edfb3e797bc6da19fd94b11cf76ef327e2001a Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 7 Nov 2023 17:21:27 +0100 Subject: [PATCH 048/127] added Run support --- .../Convert_KeyVal_namespace.py | 41 ++++++++++--- omero/annotation_scripts/Export_to_csv.py | 42 +++++++++++--- omero/annotation_scripts/KeyVal_from_csv.py | 57 ++++++++++--------- omero/annotation_scripts/Remove_KeyVal.py | 41 ++++++++++--- 4 files changed, 130 insertions(+), 51 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index a6af9eefa..2f51b1a29 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -27,7 +27,6 @@ import omero.scripts as scripts from omero.constants.metadata import NSCLIENTMAPANNOTATION -from collections import OrderedDict CHILD_OBJECTS = { "Project": "Dataset", @@ -111,6 +110,24 @@ def annotate_object(conn, obj, kv_list, namespace): def target_iterator(conn, source_object, target_type, is_tag): if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] + elif source_object.OMERO_CLASS == "PlateAcquisition": + # Check if there is more than one Run, otherwise + # it's equivalent to start from a plate (and faster this way) + plate_o = source_object.getParent() + wellsamp_l = get_children_recursive(plate_o, "WellSample") + if len(list(plate_o.listPlateAcquisitions())) > 1: + # Only case where we need to filter on PlateAcquisition + run_id = source_object.getId() + wellsamp_l = filter(lambda x: x._obj.plateAcquisition._id._val + == run_id, wellsamp_l) + target_obj_l = [wellsamp.getImage() for wellsamp in wellsamp_l] + elif target_type == "PlateAcquisition": + # No direct children access from a plate + if source_object.OMERO_CLASS == "Screen": + plate_l = get_children_recursive(source_object, "Plate") + elif source_object.OMERO_CLASS == "Plate": + plate_l = [source_object] + target_obj_l = [r for p in plate_l for r in p.listPlateAcquisitions()] elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) @@ -162,18 +179,21 @@ def replace_namespace(conn, script_params): def run_script(): - # Cannot add fancy layout if we want auto fill and selct of object ID - source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), rstring("Tag"), - ] + source_types = [ + rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), rstring("Well"), + rstring("Run"), rstring("Image"), rstring("Tag"), + ] # Duplicate Image for UI, but not a problem for script - target_types = [rstring(""), rstring("Project"), + target_types = [ + rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image")] + rstring("-- Well"), rstring("-- Run"), + rstring("--- Image") + ] client = scripts.client( 'Convert_KV_namespace', @@ -274,6 +294,11 @@ def parameters_parsing(client): if params["Data_Type"] == "Tag": params["Data_Type"] = "TagAnnotation" + if params["Data_Type"] == "Run": + params["Data_Type"] = "Acquisition" + if params["Target Data_Type"] == "Run": + params["Target Data_Type"] = "PlateAcquisition" + return params diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index b8801f575..99940549e 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -220,6 +220,24 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): def target_iterator(conn, source_object, target_type, is_tag): if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] + elif source_object.OMERO_CLASS == "PlateAcquisition": + # Check if there is more than one Run, otherwise + # it's equivalent to start from a plate (and faster this way) + plate_o = source_object.getParent() + wellsamp_l = get_children_recursive(plate_o, "WellSample") + if len(list(plate_o.listPlateAcquisitions())) > 1: + # Only case where we need to filter on PlateAcquisition + run_id = source_object.getId() + wellsamp_l = filter(lambda x: x._obj.plateAcquisition._id._val + == run_id, wellsamp_l) + target_obj_l = [wellsamp.getImage() for wellsamp in wellsamp_l] + elif target_type == "PlateAcquisition": + # No direct children access from a plate + if source_object.OMERO_CLASS == "Screen": + plate_l = get_children_recursive(source_object, "Plate") + elif source_object.OMERO_CLASS == "Plate": + plate_l = [source_object] + target_obj_l = [r for p in plate_l for r in p.listPlateAcquisitions()] elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) @@ -294,16 +312,20 @@ def run_script(): """ # Cannot add fancy layout if we want auto fill and selct of object ID - source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), rstring("Tag"), - ] + source_types = [ + rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), rstring("Well"), + rstring("Run"), rstring("Image"), rstring("Tag"), + ] # Duplicate Image for UI, but not a problem for script - target_types = [rstring(""), rstring("Project"), + target_types = [ + rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image")] + rstring("-- Well"), rstring("-- Run"), + rstring("--- Image") + ] separators = [";", ",", "TAB"] # Here we define the script name and description. @@ -396,7 +418,7 @@ def run_script(): }) client.setOutput("URL", url) else: - client.setOutput("Result", robject(res_obj)) + client.setOutput("Result", robject(res_obj._obj)) except AssertionError as err: # Display assertion errors in OMERO.web activities @@ -431,6 +453,12 @@ def parameters_parsing(client): if params["Data_Type"] == "Tag": params["Data_Type"] = "TagAnnotation" + + if params["Data_Type"] == "Run": + params["Data_Type"] = "Acquisition" + if params["Target Data_Type"] == "Run": + params["Target Data_Type"] = "PlateAcquisition" + return params diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 2edfda909..dc8288a6f 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -32,29 +32,6 @@ import sys import csv from math import floor -from collections import OrderedDict - -# source_object = conn.getObject("Acquisition", oid=51) -# target_type = "Image" -# if source_object.OMERO_CLASS == "PlateAcquisition": -# wellsamp_l = get_children_recursive(source_object.getParent(), "WellSample") -# wellsamp_l = list(filter(lambda x: x.getPlateAcquisition() == source_object, wellsamp_l)) -# image_l = [wellsamp.getImage() for wellsamp in wellsamp_l] -# def get_children_recursive(source_object, target_type): -# if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: -# # Stop condition, we return the source_obj children -# if source_object.OMERO_CLASS != "WellSample": -# return source_object.listChildren() -# elif target_type == "WellSample": -# return [source_object] -# else: -# return [source_object.getImage()] -# else: # Not yet the target -# result = [] -# for child_obj in source_object.listChildren(): -# # Going down in the Hierarchy list -# result.extend(get_children_recursive(child_obj, target_type)) -# return result CHILD_OBJECTS = { @@ -184,6 +161,24 @@ def get_children_recursive(source_object, target_type): def target_iterator(conn, source_object, target_type, is_tag): if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] + elif source_object.OMERO_CLASS == "PlateAcquisition": + # Check if there is more than one Run, otherwise + # it's equivalent to start from a plate (and faster this way) + plate_o = source_object.getParent() + wellsamp_l = get_children_recursive(plate_o, "WellSample") + if len(list(plate_o.listPlateAcquisitions())) > 1: + # Only case where we need to filter on PlateAcquisition + run_id = source_object.getId() + wellsamp_l = filter(lambda x: x._obj.plateAcquisition._id._val + == run_id, wellsamp_l) + target_obj_l = [wellsamp.getImage() for wellsamp in wellsamp_l] + elif target_type == "PlateAcquisition": + # No direct children access from a plate + if source_object.OMERO_CLASS == "Screen": + plate_l = get_children_recursive(source_object, "Plate") + elif source_object.OMERO_CLASS == "Plate": + plate_l = [source_object] + target_obj_l = [r for p in plate_l for r in p.listPlateAcquisitions()] elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) @@ -321,17 +316,18 @@ def run_script(): # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [ rstring("Project"), rstring("Dataset"), rstring("Image"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), rstring("Tag"), - ] + rstring("Screen"), rstring("Plate"), rstring("Well"), + rstring("Run"), rstring("Image"), rstring("Tag"), + ] # Duplicate Image for UI, but not a problem for script target_types = [ rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image") - ] + rstring("-- Well"), rstring("-- Run"), + rstring("--- Image") + ] separators = ["guess", ";", ",", "TAB"] @@ -505,6 +501,11 @@ def parameters_parsing(client): elif params["Separator"] == "TAB": params["Separator"] = "\t" + if params["Data_Type"] == "Run": + params["Data_Type"] = "Acquisition" + if params["Target Data_Type"] == "Run": + params["Target Data_Type"] = "PlateAcquisition" + return params diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 9d141092c..c5af16178 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -29,9 +29,7 @@ from omero.rtypes import rlong, rstring, robject from omero.constants.metadata import NSCLIENTMAPANNOTATION import omero.scripts as scripts -from omero.model import TagAnnotationI -from collections import OrderedDict CHILD_OBJECTS = { "Project": "Dataset", @@ -102,6 +100,24 @@ def get_children_recursive(source_object, target_type): def target_iterator(conn, source_object, target_type, is_tag): if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] + elif source_object.OMERO_CLASS == "PlateAcquisition": + # Check if there is more than one Run, otherwise + # it's equivalent to start from a plate (and faster this way) + plate_o = source_object.getParent() + wellsamp_l = get_children_recursive(plate_o, "WellSample") + if len(list(plate_o.listPlateAcquisitions())) > 1: + # Only case where we need to filter on PlateAcquisition + run_id = source_object.getId() + wellsamp_l = filter(lambda x: x._obj.plateAcquisition._id._val + == run_id, wellsamp_l) + target_obj_l = [wellsamp.getImage() for wellsamp in wellsamp_l] + elif target_type == "PlateAcquisition": + # No direct children access from a plate + if source_object.OMERO_CLASS == "Screen": + plate_l = get_children_recursive(source_object, "Plate") + elif source_object.OMERO_CLASS == "Plate": + plate_l = [source_object] + target_obj_l = [r for p in plate_l for r in p.listPlateAcquisitions()] elif is_tag: target_obj_l = conn.getObjectsByAnnotations(target_type, [source_object.getId()]) @@ -157,16 +173,20 @@ def run_script(): """ # Cannot add fancy layout if we want auto fill and selct of object ID - source_types = [rstring("Project"), rstring("Dataset"), rstring("Image"), - rstring("Screen"), rstring("Plate"), - rstring("Well"), rstring("Image"), rstring("Tag"), - ] + source_types = [ + rstring("Project"), rstring("Dataset"), rstring("Image"), + rstring("Screen"), rstring("Plate"), rstring("Well"), + rstring("Run"), rstring("Image"), rstring("Tag"), + ] # Duplicate Image for UI, but not a problem for script - target_types = [rstring(""), rstring("Project"), + target_types = [ + rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("--- Image")] + rstring("-- Well"), rstring("-- Run"), + rstring("--- Image") + ] # Here we define the script name and description. # Good practice to put url here to give users more guidance on how to run @@ -268,6 +288,11 @@ def parameters_parsing(client): if params["Data_Type"] == "Tag": params["Data_Type"] = "TagAnnotation" + if params["Data_Type"] == "Run": + params["Data_Type"] = "Acquisition" + if params["Target Data_Type"] == "Run": + params["Target Data_Type"] = "PlateAcquisition" + return params From 3c62f1636f67e1d04827168dd89f4722a6c8c8e9 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 8 Nov 2023 10:48:07 +0100 Subject: [PATCH 049/127] added default namespace to script doc --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 2 ++ omero/annotation_scripts/Export_to_csv.py | 2 ++ omero/annotation_scripts/KeyVal_from_csv.py | 2 ++ omero/annotation_scripts/Remove_KeyVal.py | 3 +++ 4 files changed, 9 insertions(+) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 2f51b1a29..fc3a8f1ad 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -209,6 +209,8 @@ def run_script(): - Target Data Type: Type of the target-objects that will be changed. - Old Namespace: Namespace(s) of the annotations to group and change. - New Namespace: New namespace for the annotations. + \t + Default namespace: openmicroscopy.org/omero/client/mapAnnotation \t """, # Tabs are needed to add line breaks in the HTML diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 99940549e..9de90a060 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -348,6 +348,8 @@ def run_script(): \t - Separator: Separator to be used in the .csv file. - Include column(s) of parents name: Add columns for target-data parents. + \t + Default namespace: openmicroscopy.org/omero/client/mapAnnotation \t """, # Tabs are needed to add line breaks in the HTML diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index dc8288a6f..4bcdaac5d 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -349,6 +349,8 @@ def run_script(): - Columns to exclude: Columns name of the .csv file to exclude. - Target ID colname: Column name in the .csv of the target IDs. - Target name colname: Column name in the .csv of the target names. + \t + Default namespace: openmicroscopy.org/omero/client/mapAnnotation \t """, # Tabs are needed to add line breaks in the HTML diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index c5af16178..8823beaea 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -205,6 +205,8 @@ def run_script(): - IDs: IDs of the parent-objects. - Target Data Type: Target-objects type of which KV-pairs are deleted. - Namespace: Annotations having one of these namespace(s) will be deleted. + \t + Default namespace: openmicroscopy.org/omero/client/mapAnnotation \t """, # Tabs are needed to add line breaks in the HTML @@ -240,6 +242,7 @@ def run_script(): institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", ) + try: params = parameters_parsing(client) print("Input parameters:") From acc776d798b14185a799bad897f6378facbf4bc4 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 8 Nov 2023 11:26:15 +0100 Subject: [PATCH 050/127] Fail script when sniffer fails --- omero/annotation_scripts/KeyVal_from_csv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 4bcdaac5d..3bb277faf 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -120,7 +120,7 @@ def read_csv(conn, original_file, delimiter): file_handle.read(floor(file_length/2)), ",;\t").delimiter print(f"Using delimiter {delimiter}", - f"after reading {floor(file_length/4)} characters") + f"after reading {floor(file_length/2)} characters") except Exception: file_handle.seek(0) try: @@ -130,8 +130,8 @@ def read_csv(conn, original_file, delimiter): print(f"Using delimiter {delimiter} after", f"reading {floor(file_length*0.75)} characters") except Exception: - print("Failed to sniff delimiter, use ','") - delimiter = "," + assert False, ("Failed to sniff CSV delimiter, " + + "please specify the separator") # reset to start and read whole file... file_handle.seek(0) From 59eb5c5c401a4c948dd00df82c036cf31938e282 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:05:38 +0100 Subject: [PATCH 051/127] added functionality for Namespaces added failsafe for empty Values --- omero/annotation_scripts/KeyVal_from_csv.py | 84 +++++++++++++++++---- 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 3bb277faf..f743b4a0e 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -137,10 +137,20 @@ def read_csv(conn, original_file, delimiter): file_handle.seek(0) data = list(csv.reader(file_handle, delimiter=delimiter)) - # keys are in the header row - header = [el.strip() for el in data[0]] + # check if namespaces get declared + if data[0][0].lower() == "namespace": + different_namespaces = True + + # keys are in the header row (first row for no namespaces + # second row with namespaces declared) + if different_namespaces: + header = [el.strip() for el in data[1]] + namespaces = [el.strip() for el in data[0]] + else: + header = [el.strip() for el in data[0]] + namespaces = [] print(f"Header: {header}\n") - return data, header + return data, header, namespaces def get_children_recursive(source_object, target_type): @@ -202,6 +212,7 @@ def keyval_from_csv(conn, script_params): source_ids = script_params["IDs"] file_ids = script_params["File_Annotation"] namespace = script_params["Namespace (leave blank for default)"] + namespaces_in_csv = script_params["Namespaces defined in the .csv"] to_exclude = script_params["Columns to exclude"] target_id_colname = script_params["Target ID colname"] target_name_colname = script_params["Target name colname"] @@ -225,7 +236,7 @@ def keyval_from_csv(conn, script_params): file_ann = get_original_file(source_object) original_file = file_ann.getFile()._obj - data, header = read_csv(conn, original_file, separator) + data, header, namespaces = read_csv(conn, original_file, separator) is_tag = source_type == "TagAnnotation" target_obj_l = target_iterator(conn, source_object, target_type, is_tag) @@ -273,8 +284,32 @@ def keyval_from_csv(conn, script_params): print(f"Not found: {target_id}") continue - updated = annotate_object(conn, target_obj, header, row, - cols_to_ignore, namespace) + if namespaces_in_csv: + # get a dict of Namespaces with all occurring indizes + namespace_dict = get_namespace_dict(header, cols_to_ignore, + namespaces) + kv_list = [] + # loop over the namespaces + for ns in namespace_dict.keys: + # loop over all indizes and add to the list of KV-pairs + # if there is a value + for index in namespace_dict[ns]: + key = header[index].strip() + value = row[index].strip() + if len(value) > 0: + kv_list.append([key, value]) + updated = annotate_object(conn, target_obj, kv_list, ns) + kv_list.clear() + + else: + for i in range(len(row)): + if i not in cols_to_ignore: + key = header[index].strip() + value = row[index].strip() + if len(value) > 0: + kv_list.append([key, value]) + updated = annotate_object(conn, target_obj, kv_list, namespace) + if updated: if result_obj is None: result_obj = target_obj @@ -290,16 +325,25 @@ def keyval_from_csv(conn, script_params): return message, result_obj -def annotate_object(conn, obj, header, row, cols_to_ignore, namespace): +def get_namespace_dict(header, cols_to_ignore, namespaces): + # create a dictionary of namespaces with corresponding indizes + namespace_dict = {} + namespace_dict[NSCLIENTMAPANNOTATION] = [] + for i in range(len(header)): + if i in cols_to_ignore or i == 0: + continue + if len(namespaces[i]) > 0: + if namespaces[i] not in namespace_dict: + namespace_dict[namespaces[i]] = [] + namespace_dict[namespaces[i]].append(i) + # if no custom namespace is given fall back to default + else: + namespace_dict[NSCLIENTMAPANNOTATION].append(i) + + return namespace_dict - kv_list = [] - for i in range(len(row)): - if i in cols_to_ignore or i >= len(header): - continue - key = header[i].strip() - value = row[i].strip() - kv_list.append([key, value]) +def annotate_object(conn, obj, kv_list, namespace): map_ann = omero.gateway.MapAnnotationWrapper(conn) map_ann.setNs(namespace) @@ -384,6 +428,13 @@ def run_script(): "pairs annotations. Default is the client" + "namespace, meaning editable in OMERO.web"), + scripts.Bool( + "Namespaces defined in the .csv", grouping="1.5", default=False, + description="Check if you have defined the namespaces of the" + + " different keys already in the first row of your" + + " .csv.\nFor details check the" + + " documentation."), + scripts.Bool( "Advanced parameters", optional=True, grouping="2", default=False, description="Ticking or unticking this has no effect"), @@ -427,8 +478,9 @@ def run_script(): params = parameters_parsing(client) print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", - "Namespace (leave blank for default)", "Separator", - "Columns to exclude", "Target ID colname", + "Namespace (leave blank for default)", + "Namespaces defined in the .csv", + "Separator", "Columns to exclude", "Target ID colname", "Target name colname"] for k in keys: print(f"\t- {k}: {params[k]}") From e7f0fc30c6882cc1d46cdc6fc26b538f64a4260e Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:12:07 +0100 Subject: [PATCH 052/127] a bit more documentation comments --- omero/annotation_scripts/KeyVal_from_csv.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index f743b4a0e..2b8c47402 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -326,12 +326,15 @@ def keyval_from_csv(conn, script_params): def get_namespace_dict(header, cols_to_ignore, namespaces): - # create a dictionary of namespaces with corresponding indizes +# create a dictionary of namespaces with corresponding row-indizes namespace_dict = {} namespace_dict[NSCLIENTMAPANNOTATION] = [] + # assuming the header is the longest row for i in range(len(header)): if i in cols_to_ignore or i == 0: continue + # checking for a namespace and adding its index to a + # list ot integers if len(namespaces[i]) > 0: if namespaces[i] not in namespace_dict: namespace_dict[namespaces[i]] = [] @@ -344,7 +347,7 @@ def get_namespace_dict(header, cols_to_ignore, namespaces): def annotate_object(conn, obj, kv_list, namespace): - +# helper function for the creation and linking of a MapAnnotation map_ann = omero.gateway.MapAnnotationWrapper(conn) map_ann.setNs(namespace) map_ann.setValue(kv_list) From 1e40d7bf90297e7668e9c883f4f29ea19ad60b16 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:13:03 +0100 Subject: [PATCH 053/127] flake8 shit ... --- omero/annotation_scripts/KeyVal_from_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 2b8c47402..54750143e 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -326,7 +326,7 @@ def keyval_from_csv(conn, script_params): def get_namespace_dict(header, cols_to_ignore, namespaces): -# create a dictionary of namespaces with corresponding row-indizes + # create a dictionary of namespaces with corresponding row-indizes namespace_dict = {} namespace_dict[NSCLIENTMAPANNOTATION] = [] # assuming the header is the longest row @@ -347,7 +347,7 @@ def get_namespace_dict(header, cols_to_ignore, namespaces): def annotate_object(conn, obj, kv_list, namespace): -# helper function for the creation and linking of a MapAnnotation + # helper function for the creation and linking of a MapAnnotation map_ann = omero.gateway.MapAnnotationWrapper(conn) map_ann.setNs(namespace) map_ann.setValue(kv_list) From 67ad416297fb1bc91794e87ecf692075e5620cfd Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 11:51:44 +0100 Subject: [PATCH 054/127] sniffing on one line and error on rows len mismatch --- omero/annotation_scripts/KeyVal_from_csv.py | 41 +++++++++------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 54750143e..d789b4dc5 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -104,39 +104,32 @@ def read_csv(conn, original_file, delimiter): # read the csv temp_file = provider.get_original_file_data(original_file) # Needs omero-py 5.9.1 or later - temp_name = temp_file.name - file_length = original_file.size.val - with open(temp_name, 'rt', encoding='utf-8-sig') as file_handle: + with open(temp_file.name, 'rt', encoding='utf-8-sig') as file_handle: if delimiter is None: - try: + try: # Detecting csv delimiter from the first line delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length/4)), ",;\t").delimiter + file_handle.readline(), ",;\t").delimiter print(f"Using delimiter {delimiter}", - f"after reading {floor(file_length/4)} characters") + "after reading one line") except Exception: - file_handle.seek(0) - try: - delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length/2)), - ",;\t").delimiter - print(f"Using delimiter {delimiter}", - f"after reading {floor(file_length/2)} characters") - except Exception: - file_handle.seek(0) - try: - delimiter = csv.Sniffer().sniff( - file_handle.read(floor(file_length*0.75)), - ",;\t").delimiter - print(f"Using delimiter {delimiter} after", - f"reading {floor(file_length*0.75)} characters") - except Exception: - assert False, ("Failed to sniff CSV delimiter, " + - "please specify the separator") + # Send the error back to the UI + assert False, ("Failed to sniff CSV delimiter, " + + "please specify the separator") # reset to start and read whole file... file_handle.seek(0) data = list(csv.reader(file_handle, delimiter=delimiter)) + rowlen = len(data[0]) + error_msg = ( + "CSV rows lenght mismatch: Header has {} " + + "items, while line {} has {}" + ) + for i in range(1, len(data)): + assert len(data[i]) == rowlen, error_msg.format( + rowlen, i, len(data[i]) + ) + # check if namespaces get declared if data[0][0].lower() == "namespace": different_namespaces = True From e3d4c60c67bef7017fcabdc6e4588c4f600f6abf Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 12:24:24 +0100 Subject: [PATCH 055/127] option to attach files and debug --- omero/annotation_scripts/KeyVal_from_csv.py | 43 ++++++++++++--------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index d789b4dc5..4e88e3753 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -81,19 +81,18 @@ def get_original_file(omero_obj): return file_ann -def link_file_ann(conn, object_type, object_id, file_ann_id): +def link_file_ann(conn, object_type, object_, file_ann): """Link File Annotation to the Object, if not already linked.""" - file_ann = conn.getObject("Annotation", file_ann_id) - if file_ann is None: - sys.stderr.write("Error: File Annotation not found: %s.\n" - % file_ann_id) - sys.exit(1) - omero_object = conn.getObject(object_type, object_id) # Check for existing links - links = list(conn.getAnnotationLinks(object_type, parent_ids=[object_id], - ann_ids=[file_ann_id])) + if object_type == "TagAnnotation": + print("CSV file cannot be attached to the parent tag") + return + links = list(conn.getAnnotationLinks( + object_type, parent_ids=[object_.getId()], + ann_ids=[file_ann.getId()] + )) if len(links) == 0: - omero_object.linkAnnotation(file_ann) + object_.linkAnnotation(file_ann) def read_csv(conn, original_file, delimiter): @@ -130,13 +129,9 @@ def read_csv(conn, original_file, delimiter): rowlen, i, len(data[i]) ) - # check if namespaces get declared - if data[0][0].lower() == "namespace": - different_namespaces = True - # keys are in the header row (first row for no namespaces # second row with namespaces declared) - if different_namespaces: + if data[0][0].lower() == "namespace": header = [el.strip() for el in data[1]] namespaces = [el.strip() for el in data[0]] else: @@ -210,6 +205,7 @@ def keyval_from_csv(conn, script_params): target_id_colname = script_params["Target ID colname"] target_name_colname = script_params["Target name colname"] separator = script_params["Separator"] + attach_file = script_params["Attach csv to parents"] ntarget_processed = 0 ntarget_updated = 0 @@ -222,6 +218,7 @@ def keyval_from_csv(conn, script_params): for source_object, file_ann_id in zip(source_objects, file_ids): if file_ann_id is not None: file_ann = conn.getObject("Annotation", oid=file_ann_id) + assert file_ann is not None, f"Annotation {file_ann_id} not found" assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, "The \ provided annotation ID must reference a FileAnnotation, \ not a {file_ann.OMERO_TYPE}" @@ -230,6 +227,7 @@ def keyval_from_csv(conn, script_params): original_file = file_ann.getFile()._obj data, header, namespaces = read_csv(conn, original_file, separator) + is_tag = source_type == "TagAnnotation" target_obj_l = target_iterator(conn, source_object, target_type, is_tag) @@ -295,10 +293,11 @@ def keyval_from_csv(conn, script_params): kv_list.clear() else: + kv_list = [] for i in range(len(row)): if i not in cols_to_ignore: - key = header[index].strip() - value = row[index].strip() + key = header[i].strip() + value = row[i].strip() if len(value) > 0: kv_list.append([key, value]) updated = annotate_object(conn, target_obj, kv_list, namespace) @@ -307,6 +306,9 @@ def keyval_from_csv(conn, script_params): if result_obj is None: result_obj = target_obj ntarget_updated += 1 + if ntarget_updated > 0 and attach_file: + # Only attaching if this is successful + link_file_ann(conn, source_type, source_object, file_ann) print("\n------------------------------------\n") message = f"Added KV-pairs to \ @@ -465,6 +467,11 @@ def run_script(): "ID is provided or found in the .csv). Matches " + " in exclude parameter."), + scripts.Bool( + "Attach csv to parents", grouping="2.5", default=False, + description="Attach the given CSV to the parent-data objects" + + "when not already attached to it."), + authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" @@ -477,7 +484,7 @@ def run_script(): "Namespace (leave blank for default)", "Namespaces defined in the .csv", "Separator", "Columns to exclude", "Target ID colname", - "Target name colname"] + "Target name colname", "Attach csv to parents"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") From df326b10cee434ebd70056e8a54805d4007cf6ae Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 13:03:24 +0100 Subject: [PATCH 056/127] default separator to tab --- omero/annotation_scripts/Export_to_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 9de90a060..df8ab02a3 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -383,7 +383,7 @@ def run_script(): scripts.String( "Separator", optional=False, grouping="2.1", description="Choose the .csv separator.", - values=separators, default=";"), + values=separators, default="TAB"), scripts.Bool( "Include column(s) of parents name", optional=False, From 6bdce17c123e45dac109e10e823d024600e80a0d Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 13:05:38 +0100 Subject: [PATCH 057/127] allow .tsv file in search --- omero/annotation_scripts/KeyVal_from_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 4e88e3753..eadd9ca56 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -70,7 +70,7 @@ def get_original_file(omero_obj): if ann.OMERO_TYPE == omero.model.FileAnnotationI: file_name = ann.getFile().getName() # Pick file by Ann ID (or name if ID is None) - if file_name.endswith(".csv"): + if file_name.endswith(".csv") or file_name.endswith(".tsv"): if (file_ann is None) or (ann.getDate() > file_ann.getDate()): # Get the most recent file file_ann = ann From cc48eec8744172f1a62a79008696b4c69c1f8fc8 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 17:49:05 +0100 Subject: [PATCH 058/127] export namespace option --- omero/annotation_scripts/Export_to_csv.py | 133 ++++++++++++-------- omero/annotation_scripts/KeyVal_from_csv.py | 2 +- 2 files changed, 85 insertions(+), 50 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index df8ab02a3..a6c796867 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -31,7 +31,7 @@ import tempfile import os -from collections import OrderedDict +from collections import OrderedDict, defaultdict CHILD_OBJECTS = { "Project": "Dataset", @@ -54,9 +54,8 @@ "Screen", "Plate", "Well", "Run"] } -# To allow duplicated keys -# (3 means up to 1000 same key on a single object) -ZERO_PADDING = 3 +# Add your OMERO.web URL for direct download from link: +# eg https://omero-adress.org/webclient WEBCLIENT_URL = "" @@ -68,38 +67,57 @@ def get_obj_name(omero_obj): def get_existing_map_annotions(obj, namespace_l): - key_l = [] - result = OrderedDict() - for namespace in namespace_l: - for ann in obj.listAnnotations(ns=namespace): + "Return list of KV with updated keys with NS and occurences" + annotation_dict_l = defaultdict(list) + for ns in namespace_l: + for ann in obj.listAnnotations(ns=ns): if isinstance(ann, omero.gateway.MapAnnotationWrapper): - for (k, v) in ann.getValue(): - n_occurence = key_l.count(k) - pad_key = f"{str(n_occurence).rjust(ZERO_PADDING, '0')}{k}" - result[pad_key] = v - key_l.append(k) # To count the multiple occurence of keys - return result + annotation_dict_l[ns].append(ann.getValue()) + return annotation_dict_l -def group_keyvalue_dictionaries(annotation_dicts): +def group_keyvalue_dicts(annotation_dict_l, include_namespace): """ Groups the keys and values of each object into a single dictionary """ - all_key = OrderedDict() # To keep the keys in order, for what it's worth - for annotation_dict in annotation_dicts: - all_key.update({k: None for k in annotation_dict.keys()}) - all_key = list(all_key.keys()) - - result = [] - for annotation_dict in annotation_dicts: - obj_dict = OrderedDict((k, "") for k in all_key) + + # STEP 1: finding all namespace + set_ns = set() + for ann_dict in annotation_dict_l: + set_ns.update(list(ann_dict.keys())) + set_ns = list(set_ns) + + # STEP 2: Changing keys for every object according to occurence + # if use namespace, the occurence are given a namespace + header_row = [] + ns_row = [] + n_obj = len(annotation_dict_l) + count_k_l = [[] for i in range(n_obj)] + annotation_dict_l_upd = [OrderedDict() for i in range(n_obj)] + for idx_ns, ns in enumerate(set_ns): + # Iterating by namespace to group the keys + if not include_namespace: + idx_ns = 0 + for i, ann_dict in enumerate(annotation_dict_l): + for keyval in ann_dict[ns]: + for (k, v) in keyval: + # Count key occurence per namespace for that object + n_iden = count_k_l[i].count(f"{idx_ns}#{k}") + count_k_l[i].append(k) + pad_key = f"{idx_ns}#{n_iden}${k}" + annotation_dict_l_upd[i][pad_key] = v + if pad_key not in header_row: + header_row.append(pad_key) + ns_row.append(ns) + + # STEP 3: Populating objects values + object_rows = [] + for annotation_dict in annotation_dict_l_upd: + obj_dict = OrderedDict((k, "") for k in header_row) obj_dict.update(annotation_dict) - for k, v in obj_dict.items(): - if v is None: - obj_dict[k] - result.append(list(obj_dict.values())) + object_rows.append(list(obj_dict.values())) # Removing temporary padding - all_key = [key[ZERO_PADDING:] for key in all_key] - return all_key, result + header_row = list(map(lambda x: x[x.index("$")+1:], header_row)) + return ns_row, header_row, object_rows def get_children_recursive(source_object, target_type): @@ -118,7 +136,8 @@ def get_children_recursive(source_object, target_type): def attach_csv_file(conn, obj_, csv_name, obj_id_l, obj_name_l, - obj_ancestry_l, annotation_dicts, separator, is_well): + obj_ancestry_l, annotation_dict_l, separator, + is_well, include_namespace): def to_csv(ll): """convience function to write a csv line""" nl = len(ll) @@ -164,26 +183,31 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): return result_name, result_ancestry, result_value - all_key, whole_values_l = group_keyvalue_dictionaries(annotation_dicts) + ns_row, header_row, object_rows = group_keyvalue_dicts(annotation_dict_l, + include_namespace) counter = 0 - if len(obj_ancestry_l) > 0: # If there's anything to add at all + # Sorting rows # Only sort when there are parents to group childs - obj_name_l, obj_ancestry_l, whole_values_l = sort_items(obj_name_l, - obj_ancestry_l, - whole_values_l, - is_well) + obj_name_l, obj_ancestry_l, object_rows = sort_items(obj_name_l, + obj_ancestry_l, + object_rows, + is_well) for (parent_type, _) in obj_ancestry_l[0]: - all_key.insert(counter, parent_type) + header_row.insert(counter, parent_type) + ns_row.insert(counter, "") # padding namespace like all keys counter += 1 - all_key.insert(counter, "OBJECT_ID") - all_key.insert(counter + 1, "OBJECT_NAME") - print(f"\tColumn names: {all_key}", "\n") + header_row.insert(counter, "OBJECT_ID") + header_row.insert(counter + 1, "OBJECT_NAME") + ns_row.insert(counter, "") + ns_row.insert(counter + 1, "") + ns_row[0] = "namespace" # Finalizing the namespace row + print(f"\tColumn names: {header_row}", "\n") for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, obj_name_l, - whole_values_l)): + object_rows)): counter = 0 if len(obj_ancestry_l) > 0: # If there's anything to add at all for (_, parent_name) in obj_ancestry_l[k]: @@ -196,10 +220,12 @@ def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) tfile = os.fdopen(fd, 'w') - tfile.write(to_csv(all_key)) + if include_namespace: + tfile.write(to_csv(ns_row)) + tfile.write(to_csv(header_row)) # write the keys values for each file - for whole_values in whole_values_l: - tfile.write(to_csv(whole_values)) + for object_row in object_rows: + tfile.write(to_csv(object_row)) tfile.close() # link it to the object @@ -265,12 +291,13 @@ def main_loop(conn, script_params): namespace_l = script_params["Namespace (leave blank for default)"] separator = script_params["Separator"] include_parent = script_params["Include column(s) of parents name"] + include_namespace = script_params["Include namespace"] is_well = False # One file output per given ID obj_ancestry_l = [] - annotation_dicts = [] + annotation_dict_l = [] obj_id_l, obj_name_l = [], [] for source_object in conn.getObjects(source_type, source_ids): @@ -282,8 +309,8 @@ def main_loop(conn, script_params): for target_obj in target_iterator(conn, source_object, target_type, is_tag): is_well = target_obj.OMERO_CLASS == "Well" - annotation_dicts.append(get_existing_map_annotions(target_obj, - namespace_l)) + annotation_dict_l.append(get_existing_map_annotions(target_obj, + namespace_l)) obj_id_l.append(target_obj.getId()) obj_name_l.append(get_obj_name(target_obj)) if include_parent: @@ -297,7 +324,8 @@ def main_loop(conn, script_params): csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) file_ann = attach_csv_file(conn, result_obj, csv_name, obj_id_l, obj_name_l, obj_ancestry_l, - annotation_dicts, separator, is_well) + annotation_dict_l, separator, is_well, + include_namespace) message = ("The csv is attached to " + f"{result_obj.OMERO_CLASS}:{result_obj.getId()}") @@ -391,6 +419,12 @@ def run_script(): description="Weather to include or not the name of the parent(s)" + " objects as columns in the .csv.", default=False), + scripts.Bool( + "Include namespace", optional=False, + grouping="2.3", + description="Weather to include or not the namespace" + + " of the annotations in the .csv.", default=False), + authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", @@ -400,7 +434,8 @@ def run_script(): print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", "Namespace (leave blank for default)", - "Separator", "Include column(s) of parents name"] + "Separator", "Include column(s) of parents name", + "Include namespace"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index eadd9ca56..84b7a1ecd 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -281,7 +281,7 @@ def keyval_from_csv(conn, script_params): namespaces) kv_list = [] # loop over the namespaces - for ns in namespace_dict.keys: + for ns in namespace_dict.keys(): # loop over all indizes and add to the list of KV-pairs # if there is a value for index in namespace_dict[ns]: From f1c9b375587b3f0237382ab2bad490f5358bfffc Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 18:30:19 +0100 Subject: [PATCH 059/127] Fixed NS import and add exclude-empty-val parameter --- omero/annotation_scripts/KeyVal_from_csv.py | 142 ++++++++------------ 1 file changed, 56 insertions(+), 86 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 84b7a1ecd..78589417a 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -29,9 +29,7 @@ from omero.constants.metadata import NSCLIENTMAPANNOTATION from omero.util.populate_roi import DownloadingOriginalFileProvider -import sys import csv -from math import floor CHILD_OBJECTS = { @@ -117,28 +115,30 @@ def read_csv(conn, original_file, delimiter): # reset to start and read whole file... file_handle.seek(0) - data = list(csv.reader(file_handle, delimiter=delimiter)) + rows = list(csv.reader(file_handle, delimiter=delimiter)) - rowlen = len(data[0]) + rowlen = len(rows[0]) error_msg = ( "CSV rows lenght mismatch: Header has {} " + "items, while line {} has {}" ) - for i in range(1, len(data)): - assert len(data[i]) == rowlen, error_msg.format( - rowlen, i, len(data[i]) + for i in range(1, len(rows)): + assert len(rows[i]) == rowlen, error_msg.format( + rowlen, i, len(rows[i]) ) # keys are in the header row (first row for no namespaces # second row with namespaces declared) - if data[0][0].lower() == "namespace": - header = [el.strip() for el in data[1]] - namespaces = [el.strip() for el in data[0]] - else: - header = [el.strip() for el in data[0]] - namespaces = [] + namespaces = [] + if rows[0][0].lower() == "namespace": + namespaces = [el.strip() for el in rows[0]] + namespaces = [ns if ns else NSCLIENTMAPANNOTATION for ns in namespaces] + rows = rows[1:] + header = [el.strip() for el in rows[0]] + rows = rows[1:] + print(f"Header: {header}\n") - return data, header, namespaces + return rows, header, namespaces def get_children_recursive(source_object, target_type): @@ -200,12 +200,12 @@ def keyval_from_csv(conn, script_params): source_ids = script_params["IDs"] file_ids = script_params["File_Annotation"] namespace = script_params["Namespace (leave blank for default)"] - namespaces_in_csv = script_params["Namespaces defined in the .csv"] to_exclude = script_params["Columns to exclude"] target_id_colname = script_params["Target ID colname"] target_name_colname = script_params["Target name colname"] separator = script_params["Separator"] attach_file = script_params["Attach csv to parents"] + exclude_empty_value = script_params["Exclude empty values"] ntarget_processed = 0 ntarget_updated = 0 @@ -226,7 +226,9 @@ def keyval_from_csv(conn, script_params): file_ann = get_original_file(source_object) original_file = file_ann.getFile()._obj - data, header, namespaces = read_csv(conn, original_file, separator) + rows, header, namespaces = read_csv(conn, original_file, separator) + if len(namespaces) == 0: + namespaces = [namespace] * len(header) is_tag = source_type == "TagAnnotation" target_obj_l = target_iterator(conn, source_object, @@ -256,7 +258,6 @@ def keyval_from_csv(conn, script_params): assert name not in target_d.keys(), f"Target objects \ identified by name have duplicate: {name}" target_d[name] = target_obj - else: # Setting the dictionnary target_id:target_obj # keys as string to match CSV reader output @@ -264,7 +265,9 @@ def keyval_from_csv(conn, script_params): for target_obj in target_obj_l} ntarget_processed += len(target_d) - rows = data[1:] + ok_idxs = [i for i in range(len(header)) if i not in cols_to_ignore] + tmp_ns = [namespaces[i] for i in ok_idxs] + tmp_head = [header[i] for i in ok_idxs] for row in rows: # Iterate the CSV rows and search for the matching target target_id = row[idx_id] @@ -275,37 +278,17 @@ def keyval_from_csv(conn, script_params): print(f"Not found: {target_id}") continue - if namespaces_in_csv: - # get a dict of Namespaces with all occurring indizes - namespace_dict = get_namespace_dict(header, cols_to_ignore, - namespaces) - kv_list = [] - # loop over the namespaces - for ns in namespace_dict.keys(): - # loop over all indizes and add to the list of KV-pairs - # if there is a value - for index in namespace_dict[ns]: - key = header[index].strip() - value = row[index].strip() - if len(value) > 0: - kv_list.append([key, value]) - updated = annotate_object(conn, target_obj, kv_list, ns) - kv_list.clear() - - else: - kv_list = [] - for i in range(len(row)): - if i not in cols_to_ignore: - key = header[i].strip() - value = row[i].strip() - if len(value) > 0: - kv_list.append([key, value]) - updated = annotate_object(conn, target_obj, kv_list, namespace) + row = [row[i].strip() for i in ok_idxs] + updated = annotate_object( + conn, target_obj, row, tmp_head, tmp_ns, + cols_to_ignore, exclude_empty_value + ) if updated: if result_obj is None: result_obj = target_obj ntarget_updated += 1 + if ntarget_updated > 0 and attach_file: # Only attaching if this is successful link_file_ann(conn, source_type, source_object, file_ann) @@ -320,38 +303,26 @@ def keyval_from_csv(conn, script_params): return message, result_obj -def get_namespace_dict(header, cols_to_ignore, namespaces): - # create a dictionary of namespaces with corresponding row-indizes - namespace_dict = {} - namespace_dict[NSCLIENTMAPANNOTATION] = [] - # assuming the header is the longest row - for i in range(len(header)): - if i in cols_to_ignore or i == 0: - continue - # checking for a namespace and adding its index to a - # list ot integers - if len(namespaces[i]) > 0: - if namespaces[i] not in namespace_dict: - namespace_dict[namespaces[i]] = [] - namespace_dict[namespaces[i]].append(i) - # if no custom namespace is given fall back to default - else: - namespace_dict[NSCLIENTMAPANNOTATION].append(i) - - return namespace_dict - - -def annotate_object(conn, obj, kv_list, namespace): - # helper function for the creation and linking of a MapAnnotation - map_ann = omero.gateway.MapAnnotationWrapper(conn) - map_ann.setNs(namespace) - map_ann.setValue(kv_list) - map_ann.save() - - print(f"MapAnnotation:{map_ann.id} created on {obj}") - obj.linkAnnotation(map_ann) - - return True +def annotate_object( + conn, obj, row, header, namespaces, + cols_to_ignore, exclude_empty_value +): + updated = False + for curr_ns in set(namespaces): + kv_list = [] + for ns, h, r in zip(namespaces, header, row): + if ns == curr_ns and (len(r) > 0 or not exclude_empty_value): + kv_list.append([h, r]) + if len(kv_list) > 0: # Always exclude empty KV pairs + # creation and linking of a MapAnnotation + map_ann = omero.gateway.MapAnnotationWrapper(conn) + map_ann.setNs(curr_ns) + map_ann.setValue(kv_list) + map_ann.save() + obj.linkAnnotation(map_ann) + print(f"MapAnnotation:{map_ann.id} created on {obj}") + updated = True + return updated def run_script(): @@ -426,13 +397,6 @@ def run_script(): "pairs annotations. Default is the client" + "namespace, meaning editable in OMERO.web"), - scripts.Bool( - "Namespaces defined in the .csv", grouping="1.5", default=False, - description="Check if you have defined the namespaces of the" + - " different keys already in the first row of your" + - " .csv.\nFor details check the" + - " documentation."), - scripts.Bool( "Advanced parameters", optional=True, grouping="2", default=False, description="Ticking or unticking this has no effect"), @@ -468,10 +432,16 @@ def run_script(): " in exclude parameter."), scripts.Bool( - "Attach csv to parents", grouping="2.5", default=False, + "Exclude empty values", grouping="2.5", default=False, + description="Exclude a key-value if the value is empty"), + + scripts.Bool( + "Attach csv to parents", grouping="2.6", default=False, description="Attach the given CSV to the parent-data objects" + "when not already attached to it."), + + authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" @@ -482,9 +452,9 @@ def run_script(): print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", "Namespace (leave blank for default)", - "Namespaces defined in the .csv", "Separator", "Columns to exclude", "Target ID colname", - "Target name colname", "Attach csv to parents"] + "Target name colname", "Exclude empty values", + "Attach csv to parents"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") From 2842a0e6631beb43aad7da0fcd201b4ea087be33 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 17 Nov 2023 20:40:43 +0100 Subject: [PATCH 060/127] Added parameter to split value for key duplications --- omero/annotation_scripts/KeyVal_from_csv.py | 32 +++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 78589417a..6e082ee85 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -206,6 +206,7 @@ def keyval_from_csv(conn, script_params): separator = script_params["Separator"] attach_file = script_params["Attach csv to parents"] exclude_empty_value = script_params["Exclude empty values"] + split_on = script_params["Split value on"] ntarget_processed = 0 ntarget_updated = 0 @@ -266,8 +267,6 @@ def keyval_from_csv(conn, script_params): ntarget_processed += len(target_d) ok_idxs = [i for i in range(len(header)) if i not in cols_to_ignore] - tmp_ns = [namespaces[i] for i in ok_idxs] - tmp_head = [header[i] for i in ok_idxs] for row in rows: # Iterate the CSV rows and search for the matching target target_id = row[idx_id] @@ -278,9 +277,20 @@ def keyval_from_csv(conn, script_params): print(f"Not found: {target_id}") continue - row = [row[i].strip() for i in ok_idxs] + if split_on != "": + parsed_row, parsed_ns, parsed_head = [], [], [] + for i in ok_idxs: + curr_vals = row[i].strip().split(split_on) + parsed_row.extend(curr_vals) + parsed_ns.extend([namespaces[i]] * len(curr_vals)) + parsed_head.extend([header[i]] * len(curr_vals)) + else: + parsed_row = [row[i] for i in ok_idxs] + parsed_ns = [namespaces[i] for i in ok_idxs] + parsed_head = [header[i] for i in ok_idxs] + updated = annotate_object( - conn, target_obj, row, tmp_head, tmp_ns, + conn, target_obj, parsed_row, parsed_head, parsed_ns, cols_to_ignore, exclude_empty_value ) @@ -440,7 +450,11 @@ def run_script(): description="Attach the given CSV to the parent-data objects" + "when not already attached to it."), - + scripts.String( + "Split value on", optional=True, grouping="2.7", + default="", + description="Split values according to that input to " + + "create key duplicates."), authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], @@ -454,7 +468,7 @@ def run_script(): "Namespace (leave blank for default)", "Separator", "Columns to exclude", "Target ID colname", "Target name colname", "Exclude empty values", - "Attach csv to parents"] + "Attach csv to parents", "Split value on"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") @@ -480,6 +494,7 @@ def parameters_parsing(client): # Param dict with defaults for optional parameters params["File_Annotation"] = None params["Namespace (leave blank for default)"] = NSCLIENTMAPANNOTATION + params["Split value on"] = "" for key in client.getInputKeys(): if client.getInput(key): @@ -533,6 +548,11 @@ def parameters_parsing(client): if params["Target Data_Type"] == "Run": params["Target Data_Type"] = "PlateAcquisition" + assert (params["Separator"] is None + or params["Separator"] not in params["Split value on"]), ( + "Cannot split cells with a character used as CSV separator" + ) + return params From 87a10854b29f772631e7a443e9fbeffe26ac7843 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:17:56 +0100 Subject: [PATCH 061/127] removed unnecessary parameter --- omero/annotation_scripts/KeyVal_from_csv.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 6e082ee85..d7eb4cd29 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -313,10 +313,7 @@ def keyval_from_csv(conn, script_params): return message, result_obj -def annotate_object( - conn, obj, row, header, namespaces, - cols_to_ignore, exclude_empty_value -): +def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): updated = False for curr_ns in set(namespaces): kv_list = [] From 3de4ac1c89b0984fb692f752cc76765811315b02 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:08:04 +0100 Subject: [PATCH 062/127] implemented annotating Tags --- omero/annotation_scripts/KeyVal_from_csv.py | 54 ++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index d7eb4cd29..3dde6edbf 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -93,6 +93,46 @@ def link_file_ann(conn, object_type, object_, file_ann): object_.linkAnnotation(file_ann) +def get_tag_dict(conn): + """Gets a dict of all existing Tag Names with their + respective OMERO IDs as values + + Parameters: + -------------- + conn : ``omero.gateway.BlitzGateway`` object + OMERO connection. + + Returns: + ------------- + tag_dict: dict + Dictionary in the format {tag1.name:tag1.id, tag2.name:tag2.id, ...} + """ + meta = conn.getMetadataService() + taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) + tag_dict = {} + for tag in taglist: + name = tag.getTextValue().getValue() + tag_id = tag.getId().getValue() + if name not in tag_dict: + tag_dict[name] = tag_id + return tag_dict + + +def tag_annotation(conn, obj, tag_value, tag_dict): + """Create a TagAnnotation on an Object. + If the Tag already exists use it.""" + if tag_value not in tag_dict: + tag_ann = omero.gateway.TagAnnotationWrapper(conn) + tag_ann.setValue(tag_value) + tag_ann.save() + obj.linkAnnotation(tag_ann) + print(f"created new Tag '{tag_value}'.") + else: + tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value]) + obj.linkAnnotation(tag_ann) + print(f"TagAnnotation:{tag_ann.id} created on {obj}") + + def read_csv(conn, original_file, delimiter): """ Dedicated function to read the CSV file """ print("Using FileAnnotation", @@ -315,11 +355,20 @@ def keyval_from_csv(conn, script_params): def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): updated = False + tag_dict = {} for curr_ns in set(namespaces): kv_list = [] for ns, h, r in zip(namespaces, header, row): if ns == curr_ns and (len(r) > 0 or not exclude_empty_value): - kv_list.append([h, r]) + # check for "tag" in header and create&link a TagAnnotation + if h.lower() == "tag": + # create a dict of existing tags, once + if len(tag_dict) == 0: + tag_dict = get_tag_dict(conn) + tag_annotation(conn, obj, r, tag_dict) + updated = True + else: + kv_list.append([h, r]) if len(kv_list) > 0: # Always exclude empty KV pairs # creation and linking of a MapAnnotation map_ann = omero.gateway.MapAnnotationWrapper(conn) @@ -533,7 +582,8 @@ def parameters_parsing(client): to_exclude = list(map(lambda x: x.replace('', params["Target name colname"]), to_exclude)) - params["Columns to exclude"] = to_exclude + # add Tag to excluded columns, as it is processed separately + params["Columns to exclude"] = to_exclude.append("tag") if params["Separator"] == "guess": params["Separator"] = None From 99d41c50c3a00e9a834444f686493fb20c10fc33 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 23 Nov 2023 15:14:21 +0100 Subject: [PATCH 063/127] simplify code and fix row sorting --- omero/annotation_scripts/Export_to_csv.py | 267 +++++++++++----------- 1 file changed, 128 insertions(+), 139 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index a6c796867..11f34af7e 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -31,6 +31,7 @@ import tempfile import os +import re from collections import OrderedDict, defaultdict CHILD_OBJECTS = { @@ -66,58 +67,42 @@ def get_obj_name(omero_obj): return omero_obj.getName() -def get_existing_map_annotions(obj, namespace_l): +def get_existing_map_annotions(obj, namespace): "Return list of KV with updated keys with NS and occurences" - annotation_dict_l = defaultdict(list) - for ns in namespace_l: - for ann in obj.listAnnotations(ns=ns): - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - annotation_dict_l[ns].append(ann.getValue()) - return annotation_dict_l + annotation_l = [] + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + annotation_l.append(ann) + return annotation_l -def group_keyvalue_dicts(annotation_dict_l, include_namespace): +def group_keyvalues(objannotation_l): """ Groups the keys and values of each object into a single dictionary """ - - # STEP 1: finding all namespace - set_ns = set() - for ann_dict in annotation_dict_l: - set_ns.update(list(ann_dict.keys())) - set_ns = list(set_ns) - - # STEP 2: Changing keys for every object according to occurence - # if use namespace, the occurence are given a namespace - header_row = [] - ns_row = [] - n_obj = len(annotation_dict_l) - count_k_l = [[] for i in range(n_obj)] - annotation_dict_l_upd = [OrderedDict() for i in range(n_obj)] - for idx_ns, ns in enumerate(set_ns): - # Iterating by namespace to group the keys - if not include_namespace: - idx_ns = 0 - for i, ann_dict in enumerate(annotation_dict_l): - for keyval in ann_dict[ns]: - for (k, v) in keyval: - # Count key occurence per namespace for that object - n_iden = count_k_l[i].count(f"{idx_ns}#{k}") - count_k_l[i].append(k) - pad_key = f"{idx_ns}#{n_iden}${k}" - annotation_dict_l_upd[i][pad_key] = v - if pad_key not in header_row: - header_row.append(pad_key) - ns_row.append(ns) - - # STEP 3: Populating objects values - object_rows = [] - for annotation_dict in annotation_dict_l_upd: + header_row = OrderedDict() # To keep the keys in order + keyval_obj_l = [] + for ann_l in objannotation_l: + count_k_l = [] + keyval_obj_l.append({}) + for ann in ann_l: + for (k, v) in ann.getValue(): + n_occurence = count_k_l.count(k) + pad_k = f"{n_occurence}#{k}" + keyval_obj_l[-1][pad_k] = v + header_row[pad_k] = None + count_k_l.append(k) + header_row = list(header_row.keys()) + # TODO find how to sort columns when multiple exist + # or similar + + rows = [] + for keyval_obj in keyval_obj_l: obj_dict = OrderedDict((k, "") for k in header_row) - obj_dict.update(annotation_dict) - object_rows.append(list(obj_dict.values())) + obj_dict.update(keyval_obj) + rows.append(list(obj_dict.values())) # Removing temporary padding - header_row = list(map(lambda x: x[x.index("$")+1:], header_row)) - return ns_row, header_row, object_rows + header_row = [k[k.find("#")+1:] for k in header_row] + return header_row, rows def get_children_recursive(source_object, target_type): @@ -135,97 +120,84 @@ def get_children_recursive(source_object, target_type): return result -def attach_csv_file(conn, obj_, csv_name, obj_id_l, obj_name_l, - obj_ancestry_l, annotation_dict_l, separator, - is_well, include_namespace): - def to_csv(ll): - """convience function to write a csv line""" - nl = len(ll) - fmstr = ("{}"+separator)*(nl-1)+"{}\n" - return fmstr.format(*ll) - - def sort_items(obj_name_l, obj_ancestry_l, whole_values_l, is_well): - result_name = [] - result_ancestry = [] - result_value = [] - - # That's an imbricated list of list of tuples, making it simplier first - tmp_ancestry_l = [] - for ancestries in obj_ancestry_l: - tmp_ancestry_l.append(["".join(list(obj_name)) - for obj_name in ancestries]) - - start_idx = 0 - stop_idx = 1 - while start_idx < len(tmp_ancestry_l): - while (stop_idx < len(tmp_ancestry_l) - and ("".join(tmp_ancestry_l[stop_idx]) - == "".join(tmp_ancestry_l[start_idx]))): - stop_idx += 1 - - subseq = obj_name_l[start_idx:stop_idx] - if not is_well: - # Get the sort index from the range object (argsort) - sort_order = sorted(range(len(subseq)), key=subseq.__getitem__) - else: - # Same but pad the 'well-name keys number' with zeros first - sort_order = sorted(range(len(subseq)), - key=lambda x: f"{subseq[x][0]}\ - {int(subseq[x][1:]):03}") - - for idx in sort_order: - result_name.append(obj_name_l[start_idx:stop_idx][idx]) - result_ancestry.append(obj_ancestry_l[start_idx:stop_idx][idx]) - result_value.append(whole_values_l[start_idx:stop_idx][idx]) - - start_idx = stop_idx - stop_idx += 1 - - return result_name, result_ancestry, result_value - - ns_row, header_row, object_rows = group_keyvalue_dicts(annotation_dict_l, - include_namespace) - - counter = 0 - if len(obj_ancestry_l) > 0: # If there's anything to add at all - # Sorting rows - # Only sort when there are parents to group childs - obj_name_l, obj_ancestry_l, object_rows = sort_items(obj_name_l, - obj_ancestry_l, - object_rows, - is_well) - for (parent_type, _) in obj_ancestry_l[0]: - header_row.insert(counter, parent_type) - ns_row.insert(counter, "") # padding namespace like all keys - counter += 1 - header_row.insert(counter, "OBJECT_ID") - header_row.insert(counter + 1, "OBJECT_NAME") - ns_row.insert(counter, "") - ns_row.insert(counter + 1, "") - ns_row[0] = "namespace" # Finalizing the namespace row +def sort_concat_rows(ns_row, header_row, rows, obj_id_l, + obj_name_l, obj_ancestry_l): + def convert(text): + return int(text) if text.isdigit() else text.lower() + + def alphanum_key(key): + return [convert(c) for c in re.split('([0-9]+)', key)] + + def natural_sort(names): + # kudos to https://stackoverflow.com/a/4836734/10712860 + names = list(map(alphanum_key, names)) + return sorted(range(len(names)), key=names.__getitem__) + + with_parents = len(obj_ancestry_l) > 0 + + prefixes = [""] * len(obj_name_l) + if with_parents: + for i in range(len(obj_ancestry_l[0])): + curr_name_list = [prf+names[i][1] for prf, names + in zip(prefixes, obj_ancestry_l)] + curr_name_set = list(set(curr_name_list)) + indexes = natural_sort(curr_name_set) + prefix_d = {curr_name_set[idx]: j for j, idx in enumerate(indexes)} + prefixes = [f"{prefix_d[name]}_" for name in curr_name_list] + curr_name_list = [prf+name for prf, name in zip(prefixes, obj_name_l)] + indexes = natural_sort(curr_name_list) + + # End sorting, start concatenation + + res_rows = [] + for idx in indexes: + curr_row = [str(obj_id_l[idx])] + [obj_name_l[idx]] + rows[idx] + if with_parents: + curr_row = [e[1] for e in obj_ancestry_l[idx]] + curr_row + res_rows.append(curr_row) + header_row.insert(0, "OBJECT_ID") + header_row.insert(1, "OBJECT_NAME") + ns_row.insert(0, "") + ns_row.insert(1, "") + + if with_parents: + i = 0 + while "" in [e[0] for e in obj_ancestry_l[i]]: + i += 1 # Find the row with complete parent names + for j in range(len(obj_ancestry_l[i])): + header_row.insert(j, obj_ancestry_l[i][j][0].upper()) + ns_row.insert(j, "") + ns_row[0] = "namespace" + print(f"\tColumn names: {header_row}", "\n") - for k, (obj_id, obj_name, whole_values) in enumerate(zip(obj_id_l, - obj_name_l, - object_rows)): - counter = 0 - if len(obj_ancestry_l) > 0: # If there's anything to add at all - for (_, parent_name) in obj_ancestry_l[k]: - whole_values.insert(counter, parent_name) - counter += 1 - whole_values.insert(counter, obj_id) - whole_values.insert(counter + 1, obj_name) + return ns_row, header_row, res_rows + +def build_rows(annotation_dict_l, include_namespace): + ns_row = [] + if include_namespace: + header_row, rows = [], [[] for i in range(len(annotation_dict_l[0]))] + for ns, annotation_l in annotation_dict_l.items(): + if ns == 0: + continue + next_header, next_rows = group_keyvalues(annotation_l) + ns_row.extend([ns]*len(next_header)) + header_row.extend(next_header) + for i, next_row in enumerate(next_rows): + rows[i].extend(next_row) + else: + header_row, rows = group_keyvalues(annotation_dict_l[0]) + return ns_row, header_row, rows + + +def attach_csv(conn, obj_, rows, separator, csv_name): # create the tmp directory tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) tfile = os.fdopen(fd, 'w') - if include_namespace: - tfile.write(to_csv(ns_row)) - tfile.write(to_csv(header_row)) - # write the keys values for each file - for object_row in object_rows: - tfile.write(to_csv(object_row)) + for row in rows: + tfile.write(f"{separator.join(row)}\n") tfile.close() # link it to the object @@ -293,11 +265,9 @@ def main_loop(conn, script_params): include_parent = script_params["Include column(s) of parents name"] include_namespace = script_params["Include namespace"] - is_well = False - # One file output per given ID obj_ancestry_l = [] - annotation_dict_l = [] + annotations_d = defaultdict(list) obj_id_l, obj_name_l = [], [] for source_object in conn.getObjects(source_type, source_ids): @@ -308,9 +278,13 @@ def main_loop(conn, script_params): for target_obj in target_iterator(conn, source_object, target_type, is_tag): - is_well = target_obj.OMERO_CLASS == "Well" - annotation_dict_l.append(get_existing_map_annotions(target_obj, - namespace_l)) + annotations_d[0].append([]) # (when no ns exported, all ann in 0) + for ns in namespace_l: + next_ann_l = get_existing_map_annotions(target_obj, + ns) + annotations_d[ns].append(next_ann_l) + annotations_d[0][-1].extend(next_ann_l) + obj_id_l.append(target_obj.getId()) obj_name_l.append(get_obj_name(target_obj)) if include_parent: @@ -321,11 +295,26 @@ def main_loop(conn, script_params): if result_obj is None: result_obj = target_obj print("\n------------------------------------\n") + csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) - file_ann = attach_csv_file(conn, result_obj, csv_name, obj_id_l, - obj_name_l, obj_ancestry_l, - annotation_dict_l, separator, is_well, - include_namespace) + + # Complete ancestry for image/dataset/plate without parents + norm_ancestry_l = [] + if len(obj_ancestry_l) > 0: + max_level = max(map(lambda x: len(x), obj_ancestry_l)) + for ancestry in obj_ancestry_l: + norm_ancestry_l.append([("", "")] * + (max_level - len(ancestry)) + + ancestry) + + ns_row, header_row, rows = build_rows(annotations_d, include_namespace) + ns_row, header_row, rows = sort_concat_rows(ns_row, header_row, rows, + obj_id_l, obj_name_l, + norm_ancestry_l) + rows.insert(0, header_row) + if include_namespace: + rows.insert(0, ns_row) + file_ann = attach_csv(conn, result_obj, rows, separator, csv_name) message = ("The csv is attached to " + f"{result_obj.OMERO_CLASS}:{result_obj.getId()}") From e3c4519859129396562d7797d66977362c71555c Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 24 Nov 2023 11:48:21 +0100 Subject: [PATCH 064/127] Reorganized the code --- .../Convert_KeyVal_namespace.py | 94 +++--- omero/annotation_scripts/Export_to_csv.py | 280 +++++++++--------- omero/annotation_scripts/KeyVal_from_csv.py | 264 +++++++++-------- omero/annotation_scripts/Remove_KeyVal.py | 61 ++-- 4 files changed, 362 insertions(+), 337 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index fc3a8f1ad..c93a95ad2 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -65,48 +65,6 @@ def get_children_recursive(source_object, target_type): return result -def get_existing_map_annotions(obj, namespace_l): - keyval_l, ann_l = [], [] - forbidden_deletion = [] - for namespace in namespace_l: - for ann in obj.listAnnotations(ns=namespace): - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - if ann.canEdit(): # If not, skipping it - keyval_l.extend([(k, v) for (k, v) in ann.getValue()]) - ann_l.append(ann) - else: - forbidden_deletion.append(ann.id) - if len(forbidden_deletion) > 0: - print("\tMap Annotation IDs skipped (not permitted):", - f"{forbidden_deletion}") - return keyval_l, ann_l - - -def remove_map_annotations(conn, obj, ann_l): - mapann_ids = [ann.id for ann in ann_l] - - if len(mapann_ids) == 0: - return 0 - print(f"\tMap Annotation IDs to delete: {mapann_ids}\n") - try: - conn.deleteObjects("Annotation", mapann_ids) - return 1 - except Exception: - print("Failed to delete links") - return 0 - - -def annotate_object(conn, obj, kv_list, namespace): - - map_ann = omero.gateway.MapAnnotationWrapper(conn) - map_ann.setNs(namespace) - map_ann.setValue(kv_list) - map_ann.save() - - print("\tMap Annotation created", map_ann.id) - obj.linkAnnotation(map_ann) - - def target_iterator(conn, source_object, target_type, is_tag): if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] @@ -144,7 +102,13 @@ def target_iterator(conn, source_object, target_type, is_tag): yield target_obj -def replace_namespace(conn, script_params): +def main_loop(conn, script_params): + """ + For every object: + - Find annotations in the namespace + - Remove annotations with old namespace + - Create annotations with new namespace + """ source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] @@ -178,6 +142,48 @@ def replace_namespace(conn, script_params): return message, result_obj +def get_existing_map_annotions(obj, namespace_l): + keyval_l, ann_l = [], [] + forbidden_deletion = [] + for namespace in namespace_l: + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + if ann.canEdit(): # If not, skipping it + keyval_l.extend([(k, v) for (k, v) in ann.getValue()]) + ann_l.append(ann) + else: + forbidden_deletion.append(ann.id) + if len(forbidden_deletion) > 0: + print("\tMap Annotation IDs skipped (not permitted):", + f"{forbidden_deletion}") + return keyval_l, ann_l + + +def remove_map_annotations(conn, obj, ann_l): + mapann_ids = [ann.id for ann in ann_l] + + if len(mapann_ids) == 0: + return 0 + print(f"\tMap Annotation IDs to delete: {mapann_ids}\n") + try: + conn.deleteObjects("Annotation", mapann_ids) + return 1 + except Exception: + print("Failed to delete links") + return 0 + + +def annotate_object(conn, obj, kv_list, namespace): + + map_ann = omero.gateway.MapAnnotationWrapper(conn) + map_ann.setNs(namespace) + map_ann.setValue(kv_list) + map_ann.save() + + print("\tMap Annotation created", map_ann.id) + obj.linkAnnotation(map_ann) + + def run_script(): # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [ @@ -258,7 +264,7 @@ def run_script(): # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - message, robj = replace_namespace(conn, params) + message, robj = main_loop(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 11f34af7e..c0d13b77b 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -61,50 +61,13 @@ def get_obj_name(omero_obj): + """ Helper function """ if omero_obj.OMERO_CLASS == "Well": return omero_obj.getWellPos() else: return omero_obj.getName() -def get_existing_map_annotions(obj, namespace): - "Return list of KV with updated keys with NS and occurences" - annotation_l = [] - for ann in obj.listAnnotations(ns=namespace): - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - annotation_l.append(ann) - return annotation_l - - -def group_keyvalues(objannotation_l): - """ Groups the keys and values of each object into a single dictionary """ - header_row = OrderedDict() # To keep the keys in order - keyval_obj_l = [] - for ann_l in objannotation_l: - count_k_l = [] - keyval_obj_l.append({}) - for ann in ann_l: - for (k, v) in ann.getValue(): - n_occurence = count_k_l.count(k) - pad_k = f"{n_occurence}#{k}" - keyval_obj_l[-1][pad_k] = v - header_row[pad_k] = None - count_k_l.append(k) - header_row = list(header_row.keys()) - # TODO find how to sort columns when multiple exist - # or similar - - rows = [] - for keyval_obj in keyval_obj_l: - obj_dict = OrderedDict((k, "") for k in header_row) - obj_dict.update(keyval_obj) - rows.append(list(obj_dict.values())) - - # Removing temporary padding - header_row = [k[k.find("#")+1:] for k in header_row] - return header_row, rows - - def get_children_recursive(source_object, target_type): if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children @@ -120,101 +83,6 @@ def get_children_recursive(source_object, target_type): return result -def sort_concat_rows(ns_row, header_row, rows, obj_id_l, - obj_name_l, obj_ancestry_l): - def convert(text): - return int(text) if text.isdigit() else text.lower() - - def alphanum_key(key): - return [convert(c) for c in re.split('([0-9]+)', key)] - - def natural_sort(names): - # kudos to https://stackoverflow.com/a/4836734/10712860 - names = list(map(alphanum_key, names)) - return sorted(range(len(names)), key=names.__getitem__) - - with_parents = len(obj_ancestry_l) > 0 - - prefixes = [""] * len(obj_name_l) - if with_parents: - for i in range(len(obj_ancestry_l[0])): - curr_name_list = [prf+names[i][1] for prf, names - in zip(prefixes, obj_ancestry_l)] - curr_name_set = list(set(curr_name_list)) - indexes = natural_sort(curr_name_set) - prefix_d = {curr_name_set[idx]: j for j, idx in enumerate(indexes)} - prefixes = [f"{prefix_d[name]}_" for name in curr_name_list] - curr_name_list = [prf+name for prf, name in zip(prefixes, obj_name_l)] - indexes = natural_sort(curr_name_list) - - # End sorting, start concatenation - - res_rows = [] - for idx in indexes: - curr_row = [str(obj_id_l[idx])] + [obj_name_l[idx]] + rows[idx] - if with_parents: - curr_row = [e[1] for e in obj_ancestry_l[idx]] + curr_row - res_rows.append(curr_row) - header_row.insert(0, "OBJECT_ID") - header_row.insert(1, "OBJECT_NAME") - ns_row.insert(0, "") - ns_row.insert(1, "") - - if with_parents: - i = 0 - while "" in [e[0] for e in obj_ancestry_l[i]]: - i += 1 # Find the row with complete parent names - for j in range(len(obj_ancestry_l[i])): - header_row.insert(j, obj_ancestry_l[i][j][0].upper()) - ns_row.insert(j, "") - ns_row[0] = "namespace" - - print(f"\tColumn names: {header_row}", "\n") - - return ns_row, header_row, res_rows - - -def build_rows(annotation_dict_l, include_namespace): - ns_row = [] - if include_namespace: - header_row, rows = [], [[] for i in range(len(annotation_dict_l[0]))] - for ns, annotation_l in annotation_dict_l.items(): - if ns == 0: - continue - next_header, next_rows = group_keyvalues(annotation_l) - ns_row.extend([ns]*len(next_header)) - header_row.extend(next_header) - for i, next_row in enumerate(next_rows): - rows[i].extend(next_row) - else: - header_row, rows = group_keyvalues(annotation_dict_l[0]) - return ns_row, header_row, rows - - -def attach_csv(conn, obj_, rows, separator, csv_name): - # create the tmp directory - tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') - (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) - tfile = os.fdopen(fd, 'w') - for row in rows: - tfile.write(f"{separator.join(row)}\n") - tfile.close() - - # link it to the object - file_ann = conn.createFileAnnfromLocalFile( - tmp_file, origFilePathAndName=csv_name, - ns='KeyVal_export') - obj_.linkAnnotation(file_ann) - - print(f"{file_ann} linked to {obj_}") - - # remove the tmp file - os.remove(tmp_file) - os.rmdir(tmp_dir) - - return file_ann.getFile() - - def target_iterator(conn, source_object, target_type, is_tag): if target_type == source_object.OMERO_CLASS: target_obj_l = [source_object] @@ -253,10 +121,15 @@ def target_iterator(conn, source_object, target_type, is_tag): def main_loop(conn, script_params): - ''' writes the data (list of dicts) to a file - @param conn: Blitz Gateway connection wrapper - @param script_params: A map of the input parameters - ''' + """ + For every object: + - Find annotations in the namespace and gather in a dict + - (opt) Gather ancestry + Finalize: + - Group all annotations together + - Sort rows (useful for wells) + - Write a single CSV file + """ source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] @@ -322,6 +195,139 @@ def main_loop(conn, script_params): return message, file_ann, result_obj +def get_existing_map_annotions(obj, namespace): + "Return list of KV with updated keys with NS and occurences" + annotation_l = [] + for ann in obj.listAnnotations(ns=namespace): + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + annotation_l.append(ann) + return annotation_l + + +def build_rows(annotation_dict_l, include_namespace): + ns_row = [] + if include_namespace: + header_row, rows = [], [[] for i in range(len(annotation_dict_l[0]))] + for ns, annotation_l in annotation_dict_l.items(): + if ns == 0: + continue + next_header, next_rows = group_keyvalues(annotation_l) + ns_row.extend([ns]*len(next_header)) + header_row.extend(next_header) + for i, next_row in enumerate(next_rows): + rows[i].extend(next_row) + else: + header_row, rows = group_keyvalues(annotation_dict_l[0]) + return ns_row, header_row, rows + + +def group_keyvalues(objannotation_l): + """ Groups the keys and values of each object into a single dictionary """ + header_row = OrderedDict() # To keep the keys in order + keyval_obj_l = [] + for ann_l in objannotation_l: + count_k_l = [] + keyval_obj_l.append({}) + for ann in ann_l: + for (k, v) in ann.getValue(): + n_occurence = count_k_l.count(k) + pad_k = f"{n_occurence}#{k}" + keyval_obj_l[-1][pad_k] = v + header_row[pad_k] = None + count_k_l.append(k) + header_row = list(header_row.keys()) + # TODO find how to sort columns when multiple exist + # or similar + + rows = [] + for keyval_obj in keyval_obj_l: + obj_dict = OrderedDict((k, "") for k in header_row) + obj_dict.update(keyval_obj) + rows.append(list(obj_dict.values())) + + # Removing temporary padding + header_row = [k[k.find("#")+1:] for k in header_row] + return header_row, rows + + +def sort_concat_rows(ns_row, header_row, rows, obj_id_l, + obj_name_l, obj_ancestry_l): + def convert(text): + return int(text) if text.isdigit() else text.lower() + + def alphanum_key(key): + return [convert(c) for c in re.split('([0-9]+)', key)] + + def natural_sort(names): + # kudos to https://stackoverflow.com/a/4836734/10712860 + names = list(map(alphanum_key, names)) + return sorted(range(len(names)), key=names.__getitem__) + + with_parents = len(obj_ancestry_l) > 0 + + prefixes = [""] * len(obj_name_l) + if with_parents: + for i in range(len(obj_ancestry_l[0])): + curr_name_list = [prf+names[i][1] for prf, names + in zip(prefixes, obj_ancestry_l)] + curr_name_set = list(set(curr_name_list)) + indexes = natural_sort(curr_name_set) + prefix_d = {curr_name_set[idx]: j for j, idx in enumerate(indexes)} + prefixes = [f"{prefix_d[name]}_" for name in curr_name_list] + curr_name_list = [prf+name for prf, name in zip(prefixes, obj_name_l)] + indexes = natural_sort(curr_name_list) + + # End sorting, start concatenation + + res_rows = [] + for idx in indexes: + curr_row = [str(obj_id_l[idx])] + [obj_name_l[idx]] + rows[idx] + if with_parents: + curr_row = [e[1] for e in obj_ancestry_l[idx]] + curr_row + res_rows.append(curr_row) + header_row.insert(0, "OBJECT_ID") + header_row.insert(1, "OBJECT_NAME") + ns_row.insert(0, "") + ns_row.insert(1, "") + + if with_parents: + i = 0 + while "" in [e[0] for e in obj_ancestry_l[i]]: + i += 1 # Find the row with complete parent names + for j in range(len(obj_ancestry_l[i])): + header_row.insert(j, obj_ancestry_l[i][j][0].upper()) + ns_row.insert(j, "") + ns_row[0] = "namespace" + + print(f"\tColumn names: {header_row}", "\n") + + return ns_row, header_row, res_rows + + +def attach_csv(conn, obj_, rows, separator, csv_name): + # create the tmp directory + tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') + (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) + tfile = os.fdopen(fd, 'w') + for row in rows: + tfile.write(f"{separator.join(row)}\n") + tfile.close() + + # link it to the object + file_ann = conn.createFileAnnfromLocalFile( + tmp_file, origFilePathAndName=csv_name, + ns='KeyVal_export') + obj_.linkAnnotation(file_ann) + + print(f"{file_ann} linked to {obj_}") + + # remove the tmp file + os.remove(tmp_file) + os.rmdir(tmp_dir) + + return file_ann.getFile() + + def run_script(): """ The main entry point of the script, as called by the client via the diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 3dde6edbf..dfaec517c 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -55,132 +55,13 @@ def get_obj_name(omero_obj): + """ Helper function """ if omero_obj.OMERO_CLASS == "Well": return omero_obj.getWellPos() else: return omero_obj.getName() -def get_original_file(omero_obj): - """Find last AnnotationFile linked to object""" - file_ann = None - for ann in omero_obj.listAnnotations(): - if ann.OMERO_TYPE == omero.model.FileAnnotationI: - file_name = ann.getFile().getName() - # Pick file by Ann ID (or name if ID is None) - if file_name.endswith(".csv") or file_name.endswith(".tsv"): - if (file_ann is None) or (ann.getDate() > file_ann.getDate()): - # Get the most recent file - file_ann = ann - - assert file_ann is not None, f"No .csv FileAnnotation was found on \ - {omero_obj.OMERO_CLASS}:{get_obj_name(omero_obj)}:{omero_obj.getId()}" - - return file_ann - - -def link_file_ann(conn, object_type, object_, file_ann): - """Link File Annotation to the Object, if not already linked.""" - # Check for existing links - if object_type == "TagAnnotation": - print("CSV file cannot be attached to the parent tag") - return - links = list(conn.getAnnotationLinks( - object_type, parent_ids=[object_.getId()], - ann_ids=[file_ann.getId()] - )) - if len(links) == 0: - object_.linkAnnotation(file_ann) - - -def get_tag_dict(conn): - """Gets a dict of all existing Tag Names with their - respective OMERO IDs as values - - Parameters: - -------------- - conn : ``omero.gateway.BlitzGateway`` object - OMERO connection. - - Returns: - ------------- - tag_dict: dict - Dictionary in the format {tag1.name:tag1.id, tag2.name:tag2.id, ...} - """ - meta = conn.getMetadataService() - taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) - tag_dict = {} - for tag in taglist: - name = tag.getTextValue().getValue() - tag_id = tag.getId().getValue() - if name not in tag_dict: - tag_dict[name] = tag_id - return tag_dict - - -def tag_annotation(conn, obj, tag_value, tag_dict): - """Create a TagAnnotation on an Object. - If the Tag already exists use it.""" - if tag_value not in tag_dict: - tag_ann = omero.gateway.TagAnnotationWrapper(conn) - tag_ann.setValue(tag_value) - tag_ann.save() - obj.linkAnnotation(tag_ann) - print(f"created new Tag '{tag_value}'.") - else: - tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value]) - obj.linkAnnotation(tag_ann) - print(f"TagAnnotation:{tag_ann.id} created on {obj}") - - -def read_csv(conn, original_file, delimiter): - """ Dedicated function to read the CSV file """ - print("Using FileAnnotation", - f"{original_file.id.val}:{original_file.name.val}") - provider = DownloadingOriginalFileProvider(conn) - # read the csv - temp_file = provider.get_original_file_data(original_file) - # Needs omero-py 5.9.1 or later - with open(temp_file.name, 'rt', encoding='utf-8-sig') as file_handle: - if delimiter is None: - try: # Detecting csv delimiter from the first line - delimiter = csv.Sniffer().sniff( - file_handle.readline(), ",;\t").delimiter - print(f"Using delimiter {delimiter}", - "after reading one line") - except Exception: - # Send the error back to the UI - assert False, ("Failed to sniff CSV delimiter, " + - "please specify the separator") - - # reset to start and read whole file... - file_handle.seek(0) - rows = list(csv.reader(file_handle, delimiter=delimiter)) - - rowlen = len(rows[0]) - error_msg = ( - "CSV rows lenght mismatch: Header has {} " + - "items, while line {} has {}" - ) - for i in range(1, len(rows)): - assert len(rows[i]) == rowlen, error_msg.format( - rowlen, i, len(rows[i]) - ) - - # keys are in the header row (first row for no namespaces - # second row with namespaces declared) - namespaces = [] - if rows[0][0].lower() == "namespace": - namespaces = [el.strip() for el in rows[0]] - namespaces = [ns if ns else NSCLIENTMAPANNOTATION for ns in namespaces] - rows = rows[1:] - header = [el.strip() for el in rows[0]] - rows = rows[1:] - - print(f"Header: {header}\n") - return rows, header, namespaces - - def get_children_recursive(source_object, target_type): if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children @@ -234,7 +115,17 @@ def target_iterator(conn, source_object, target_type, is_tag): print() -def keyval_from_csv(conn, script_params): +def main_loop(conn, script_params): + """ + Startup: + - Find CSV and read + For every object: + - Gather name and ID + Finalize: + - Find a match between CSV rows and objects + - Annotate the objects + - (opt) attach the CSV to the source object + """ source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] @@ -257,6 +148,7 @@ def keyval_from_csv(conn, script_params): # One file output per given ID source_objects = conn.getObjects(source_type, source_ids) for source_object, file_ann_id in zip(source_objects, file_ids): + ntarget_updated_curr = 0 if file_ann_id is not None: file_ann = conn.getObject("Annotation", oid=file_ann_id) assert file_ann is not None, f"Annotation {file_ann_id} not found" @@ -331,15 +223,16 @@ def keyval_from_csv(conn, script_params): updated = annotate_object( conn, target_obj, parsed_row, parsed_head, parsed_ns, - cols_to_ignore, exclude_empty_value + exclude_empty_value ) if updated: if result_obj is None: result_obj = target_obj ntarget_updated += 1 + ntarget_updated_curr += 1 - if ntarget_updated > 0 and attach_file: + if ntarget_updated_curr > 0 and attach_file: # Only attaching if this is successful link_file_ann(conn, source_type, source_object, file_ann) print("\n------------------------------------\n") @@ -353,6 +246,72 @@ def keyval_from_csv(conn, script_params): return message, result_obj +def get_original_file(omero_obj): + """Find last AnnotationFile linked to object if no annotation is given""" + file_ann = None + for ann in omero_obj.listAnnotations(): + if ann.OMERO_TYPE == omero.model.FileAnnotationI: + file_name = ann.getFile().getName() + # Pick file by Ann ID (or name if ID is None) + if file_name.endswith(".csv") or file_name.endswith(".tsv"): + if (file_ann is None) or (ann.getDate() > file_ann.getDate()): + # Get the most recent file + file_ann = ann + + assert file_ann is not None, f"No .csv FileAnnotation was found on \ + {omero_obj.OMERO_CLASS}:{get_obj_name(omero_obj)}:{omero_obj.getId()}" + + return file_ann + + +def read_csv(conn, original_file, delimiter): + """ Dedicated function to read the CSV file """ + print("Using FileAnnotation", + f"{original_file.id.val}:{original_file.name.val}") + provider = DownloadingOriginalFileProvider(conn) + # read the csv + temp_file = provider.get_original_file_data(original_file) + # Needs omero-py 5.9.1 or later + with open(temp_file.name, 'rt', encoding='utf-8-sig') as file_handle: + if delimiter is None: + try: # Detecting csv delimiter from the first line + delimiter = csv.Sniffer().sniff( + file_handle.readline(), ",;\t").delimiter + print(f"Using delimiter {delimiter}", + "after reading one line") + except Exception: + # Send the error back to the UI + assert False, ("Failed to sniff CSV delimiter, " + + "please specify the separator") + + # reset to start and read whole file... + file_handle.seek(0) + rows = list(csv.reader(file_handle, delimiter=delimiter)) + + rowlen = len(rows[0]) + error_msg = ( + "CSV rows lenght mismatch: Header has {} " + + "items, while line {} has {}" + ) + for i in range(1, len(rows)): + assert len(rows[i]) == rowlen, error_msg.format( + rowlen, i, len(rows[i]) + ) + + # keys are in the header row (first row for no namespaces + # second row with namespaces declared) + namespaces = [] + if rows[0][0].lower() == "namespace": + namespaces = [el.strip() for el in rows[0]] + namespaces = [ns if ns else NSCLIENTMAPANNOTATION for ns in namespaces] + rows = rows[1:] + header = [el.strip() for el in rows[0]] + rows = rows[1:] + + print(f"Header: {header}\n") + return rows, header, namespaces + + def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): updated = False tag_dict = {} @@ -381,6 +340,60 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): return updated +def get_tag_dict(conn): + """Gets a dict of all existing Tag Names with their + respective OMERO IDs as values + + Parameters: + -------------- + conn : ``omero.gateway.BlitzGateway`` object + OMERO connection. + + Returns: + ------------- + tag_dict: dict + Dictionary in the format {tag1.name:tag1.id, tag2.name:tag2.id, ...} + """ + meta = conn.getMetadataService() + taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) + tag_dict = {} + for tag in taglist: + name = tag.getTextValue().getValue() + tag_id = tag.getId().getValue() + if name not in tag_dict: + tag_dict[name] = tag_id + return tag_dict + + +def tag_annotation(conn, obj, tag_value, tag_dict): + """Create a TagAnnotation on an Object. + If the Tag already exists use it.""" + if tag_value not in tag_dict: + tag_ann = omero.gateway.TagAnnotationWrapper(conn) + tag_ann.setValue(tag_value) + tag_ann.save() + obj.linkAnnotation(tag_ann) + print(f"created new Tag '{tag_value}'.") + else: + tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value]) + obj.linkAnnotation(tag_ann) + print(f"TagAnnotation:{tag_ann.id} created on {obj}") + + +def link_file_ann(conn, object_type, object_, file_ann): + """Link File Annotation to the Object, if not already linked.""" + # Check for existing links + if object_type == "TagAnnotation": + print("CSV file cannot be attached to the parent tag") + return + links = list(conn.getAnnotationLinks( + object_type, parent_ids=[object_.getId()], + ann_ids=[file_ann.getId()] + )) + if len(links) == 0: + object_.linkAnnotation(file_ann) + + def run_script(): # Cannot add fancy layout if we want auto fill and selct of object ID source_types = [ @@ -521,7 +534,7 @@ def run_script(): # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - message, robj = keyval_from_csv(conn, params) + message, robj = main_loop(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) @@ -583,7 +596,8 @@ def parameters_parsing(client): params["Target name colname"]), to_exclude)) # add Tag to excluded columns, as it is processed separately - params["Columns to exclude"] = to_exclude.append("tag") + to_exclude.append("tag") + params["Columns to exclude"] = to_exclude if params["Separator"] == "guess": params["Separator"] = None diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 8823beaea..3037b5ca1 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -56,32 +56,6 @@ "in a batch deletion of key-value pairs from the server") -def remove_map_annotations(conn, obj, namespace_l): - mapann_ids = [] - forbidden_deletion = [] - for namespace in namespace_l: - anns = list(obj.listAnnotations(ns=namespace)) - for ann in anns: - if isinstance(ann, omero.gateway.MapAnnotationWrapper): - if ann.canEdit(): # If not, skipping it - mapann_ids.append(ann.id) - else: - forbidden_deletion.append(ann.id) - - if len(mapann_ids) == 0: - return 0 - print(f"\tMap Annotation IDs to delete: {mapann_ids}") - if len(forbidden_deletion) > 0: - print("\tMap Annotation IDs skipped (not permitted):", - f"{forbidden_deletion}\n") - try: - conn.deleteObjects("Annotation", mapann_ids) - return 1 - except Exception: - print("Failed to delete links") - return 0 - - def get_children_recursive(source_object, target_type): if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: # Stop condition, we return the source_obj children @@ -134,11 +108,10 @@ def target_iterator(conn, source_object, target_type, is_tag): yield target_obj -def remove_keyvalue(conn, script_params): +def main_loop(conn, script_params): """ - File the list of objects - @param conn: Blitz Gateway connection wrapper - @param script_params: A map of the input parameters + For every object: + - Find annotations in the namespace and remove """ source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] @@ -166,6 +139,32 @@ def remove_keyvalue(conn, script_params): return message, result_obj +def remove_map_annotations(conn, obj, namespace_l): + mapann_ids = [] + forbidden_deletion = [] + for namespace in namespace_l: + anns = list(obj.listAnnotations(ns=namespace)) + for ann in anns: + if isinstance(ann, omero.gateway.MapAnnotationWrapper): + if ann.canEdit(): # If not, skipping it + mapann_ids.append(ann.id) + else: + forbidden_deletion.append(ann.id) + + if len(mapann_ids) == 0: + return 0 + print(f"\tMap Annotation IDs to delete: {mapann_ids}") + if len(forbidden_deletion) > 0: + print("\tMap Annotation IDs skipped (not permitted):", + f"{forbidden_deletion}\n") + try: + conn.deleteObjects("Annotation", mapann_ids) + return 1 + except Exception: + print("Failed to delete links") + return 0 + + def run_script(): """ The main entry point of the script, as called by the client via the @@ -254,7 +253,7 @@ def run_script(): # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) - message, robj = remove_keyvalue(conn, params) + message, robj = main_loop(conn, params) client.setOutput("Message", rstring(message)) if robj is not None: client.setOutput("Result", robject(robj._obj)) From e63adf4c08a08ef3f905ad73d94c2d30ebc4686a Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:04:52 +0100 Subject: [PATCH 065/127] implemented Boolean to use only ones own tags --- omero/annotation_scripts/KeyVal_from_csv.py | 27 ++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index dfaec517c..1107461ab 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -138,6 +138,7 @@ def main_loop(conn, script_params): attach_file = script_params["Attach csv to parents"] exclude_empty_value = script_params["Exclude empty values"] split_on = script_params["Split value on"] + use_personal_tags = script_params["Use only personal Tags"] ntarget_processed = 0 ntarget_updated = 0 @@ -223,7 +224,7 @@ def main_loop(conn, script_params): updated = annotate_object( conn, target_obj, parsed_row, parsed_head, parsed_ns, - exclude_empty_value + exclude_empty_value, use_personal_tags ) if updated: @@ -312,7 +313,8 @@ def read_csv(conn, original_file, delimiter): return rows, header, namespaces -def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): +def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, + use_personal_tags): updated = False tag_dict = {} for curr_ns in set(namespaces): @@ -323,7 +325,7 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): if h.lower() == "tag": # create a dict of existing tags, once if len(tag_dict) == 0: - tag_dict = get_tag_dict(conn) + tag_dict = get_tag_dict(conn, use_personal_tags) tag_annotation(conn, obj, r, tag_dict) updated = True else: @@ -340,7 +342,7 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value): return updated -def get_tag_dict(conn): +def get_tag_dict(conn, use_personal_tags): """Gets a dict of all existing Tag Names with their respective OMERO IDs as values @@ -348,6 +350,8 @@ def get_tag_dict(conn): -------------- conn : ``omero.gateway.BlitzGateway`` object OMERO connection. + use_personal_tags: ``Boolean``, indicates the use of only tags + owned by the user. Returns: ------------- @@ -358,6 +362,8 @@ def get_tag_dict(conn): taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) tag_dict = {} for tag in taglist: + if use_personal_tags and tag.getOwner().id == conn.getUserId(): + continue name = tag.getTextValue().getValue() tag_id = tag.getId().getValue() if name not in tag_dict: @@ -367,13 +373,16 @@ def get_tag_dict(conn): def tag_annotation(conn, obj, tag_value, tag_dict): """Create a TagAnnotation on an Object. - If the Tag already exists use it.""" + If the Tag already exists use it. + """ + if tag_value not in tag_dict: tag_ann = omero.gateway.TagAnnotationWrapper(conn) tag_ann.setValue(tag_value) tag_ann.save() obj.linkAnnotation(tag_ann) print(f"created new Tag '{tag_value}'.") + tag_dict[tag_value] = tag_ann.id else: tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value]) obj.linkAnnotation(tag_ann) @@ -502,7 +511,7 @@ def run_script(): scripts.Bool( "Exclude empty values", grouping="2.5", default=False, - description="Exclude a key-value if the value is empty"), + description="Exclude a key-value if the value is empty."), scripts.Bool( "Attach csv to parents", grouping="2.6", default=False, @@ -515,6 +524,12 @@ def run_script(): description="Split values according to that input to " + "create key duplicates."), + scripts.Bool( + "Use only personal Tags", grouping="2.8", default=False, + description="Determines if Tags of other users in the group" + + "can be used on objects.\n Using only personal Tags might" + + "lead to multiple Tags with the same name in one OMERO-group."), + authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" From 6a8878c8a88c5f0ce6bca6dcebfdfbfc3d189a34 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:41:38 +0100 Subject: [PATCH 066/127] Implemented Tag sets --- omero/annotation_scripts/KeyVal_from_csv.py | 92 +++++++++++++++++---- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 1107461ab..dd33dc345 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -27,6 +27,7 @@ from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts from omero.constants.metadata import NSCLIENTMAPANNOTATION +from omero.model import AnnotationAnnotationLinkI from omero.util.populate_roi import DownloadingOriginalFileProvider import csv @@ -326,7 +327,14 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, # create a dict of existing tags, once if len(tag_dict) == 0: tag_dict = get_tag_dict(conn, use_personal_tags) - tag_annotation(conn, obj, r, tag_dict) + # create a list of tags + tags_raw = r.split(",").strip() + tags = [] + # separate the TagSet from the Tag --> [Tag, TagSet] + for tag in tags_raw: + tags.append([x.replace("]", "") for x in + tag.split("[")]) + tag_annotation(conn, obj, tags, tag_dict) updated = True else: kv_list.append([h, r]) @@ -355,38 +363,86 @@ def get_tag_dict(conn, use_personal_tags): Returns: ------------- - tag_dict: dict - Dictionary in the format {tag1.name:tag1.id, tag2.name:tag2.id, ...} + tag_dict: ``dictionary`` {tag_name : {tagSet_name : tag_id}} """ meta = conn.getMetadataService() taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) tag_dict = {} for tag in taglist: - if use_personal_tags and tag.getOwner().id == conn.getUserId(): + if use_personal_tags and tag.getOwner().id != conn.getUserId(): continue name = tag.getTextValue().getValue() tag_id = tag.getId().getValue() + # check if it has parents, assert the namespace is correct and + # then add them as a dict + parents = {"", tag_id} + raw_links = list(conn.getAnnotationLinks("TagAnnotation", + ann_ids=[tag_id])) + if raw_links: + parents = {link.parent.textValue.val: tag_id for link + in raw_links if link.parent.ns.val == + "openmicroscopy.org/omero/insight/tagset"} if name not in tag_dict: - tag_dict[name] = tag_id + tag_dict[name] = parents + return tag_dict -def tag_annotation(conn, obj, tag_value, tag_dict): +def tag_annotation(conn, obj, tags, tag_dict): """Create a TagAnnotation on an Object. If the Tag already exists use it. """ - - if tag_value not in tag_dict: - tag_ann = omero.gateway.TagAnnotationWrapper(conn) - tag_ann.setValue(tag_value) - tag_ann.save() - obj.linkAnnotation(tag_ann) - print(f"created new Tag '{tag_value}'.") - tag_dict[tag_value] = tag_ann.id - else: - tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value]) - obj.linkAnnotation(tag_ann) - print(f"TagAnnotation:{tag_ann.id} created on {obj}") + update = conn.getUpdateService() + for tag in tags: + tag_value = tag[0] + if len(tag) > 1: + tagSet = tag[1] + else: + tagSet = "" + + # if the Tag does not exist + if tag_value not in tag_dict: + # create TagAnnotation + tag_ann = omero.gateway.TagAnnotationWrapper(conn) + tag_ann.setValue(tag_value) + tag_ann.save() + obj.linkAnnotation(tag_ann) + print(f"created new Tag '{tag_value}'.") + # update tag dictionary + tag_dict[tag_value] = {tagSet: tag_ann.id} + if tagSet: + if tagSet not in tag_dict: + # create new TagSet + parent_tag = omero.gateway.TagAnnotationWrapper(conn) + parent_tag.setValue(tagSet) + parent_tag.ns = "openmicroscopy.org/omero/insight/tagset" + parent_tag.save() + # update tag dictionary + tag_dict[tagSet] = {"": parent_tag.id} + else: + # or get existing + parent_tag = conn.getObject("TagAnnotation", + tag_dict[tagSet][""]) + # create a Link and link Tag and TagSet + link = AnnotationAnnotationLinkI() + link.parent = parent_tag._obj + link.child = tag_ann._obj + update.saveObject(link) + + # if the Tag does exist + else: + if tagSet and tagSet not in tag_dict: + # create new TagSet + parent_tag = omero.gateway.TagAnnotationWrapper(conn) + parent_tag.setValue(tagSet) + parent_tag.ns = "openmicroscopy.org/omero/insight/tagset" + parent_tag.save() + # update tag dictionary + tag_dict[tagSet] = {"": parent_tag.id} + tag_ann = conn.getObject("TagAnnotation", + tag_dict[tag_value][tagSet]) + obj.linkAnnotation(tag_ann) + print(f"TagAnnotation:{tag_ann.id} created on {obj}") def link_file_ann(conn, object_type, object_, file_ann): From a7c33791b6218d0060e3968ce710dae3576b5956 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:20:44 +0100 Subject: [PATCH 067/127] a lot of bugfixes and fixing the tag-tagSet interaction; still not done though. --- omero/annotation_scripts/KeyVal_from_csv.py | 133 +++++++++++++++----- 1 file changed, 101 insertions(+), 32 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index dd33dc345..7cf4ba433 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -27,7 +27,8 @@ from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts from omero.constants.metadata import NSCLIENTMAPANNOTATION -from omero.model import AnnotationAnnotationLinkI +from omero.constants.metadata import NSINSIGHTTAGSET +from omero.model import AnnotationAnnotationLinkI, TagAnnotationI from omero.util.populate_roi import DownloadingOriginalFileProvider import csv @@ -140,10 +141,13 @@ def main_loop(conn, script_params): exclude_empty_value = script_params["Exclude empty values"] split_on = script_params["Split value on"] use_personal_tags = script_params["Use only personal Tags"] + file_ann_multiplied = script_params["File_Annotation_multiplied"] ntarget_processed = 0 ntarget_updated = 0 - missing_names = 0 + missing_names = set() + processed_names = set() + total_missing_names = 0 result_obj = None @@ -204,11 +208,21 @@ def main_loop(conn, script_params): for row in rows: # Iterate the CSV rows and search for the matching target target_id = row[idx_id] + # skip empty rows + if target_id == "": + continue if target_id in target_d.keys(): target_obj = target_d[target_id] + # add name/id to processed set + if file_ann_multiplied: + processed_names.add(target_id) else: - missing_names += 1 - print(f"Not found: {target_id}") + # add name/id to missing set + if file_ann_multiplied: + missing_names.add(target_id) + else: + total_missing_names += 1 + print(f"Not found: {target_id}") continue if split_on != "": @@ -241,8 +255,17 @@ def main_loop(conn, script_params): message = f"Added KV-pairs to \ {ntarget_updated}/{ntarget_processed} {target_type}" - if missing_names > 0: - message += f". {missing_names} {target_type} not found \ + + if file_ann_multiplied and len(missing_names) > 0: + # subtract the processed names/ids from the + # missing ones and print the missing names/ids + missing_names = missing_names - processed_names + if len(missing_names) > 0: + print(f"Not found: {missing_names}") + total_missing_names = len(missing_names) + + if total_missing_names > 0: + message += f". {total_missing_names} {target_type} not found \ (using {'ID' if use_id else 'name'} to identify them)." return message, result_obj @@ -328,9 +351,10 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, if len(tag_dict) == 0: tag_dict = get_tag_dict(conn, use_personal_tags) # create a list of tags - tags_raw = r.split(",").strip() + tags_raw = [tag.strip() for tag in r.split(",")] tags = [] - # separate the TagSet from the Tag --> [Tag, TagSet] + # separate the TagSet from the Tag --> + # [[Tag1, TagSet], [Tag2]] for tag in tags_raw: tags.append([x.replace("]", "") for x in tag.split("[")]) @@ -352,7 +376,7 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, def get_tag_dict(conn, use_personal_tags): """Gets a dict of all existing Tag Names with their - respective OMERO IDs as values + respective OMERO IDs as values. Parameters: -------------- @@ -364,6 +388,10 @@ def get_tag_dict(conn, use_personal_tags): Returns: ------------- tag_dict: ``dictionary`` {tag_name : {tagSet_name : tag_id}} + tagSet_name: + "" for a TagAnnotation w/o TagSet parent\n + "*" for a TagSet\n + "" for a TagAnnotation with a TagSet parent """ meta = conn.getMetadataService() taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) @@ -373,9 +401,14 @@ def get_tag_dict(conn, use_personal_tags): continue name = tag.getTextValue().getValue() tag_id = tag.getId().getValue() + namespace = tag.getNs() + if namespace is not None and\ + namespace._val == "openmicroscopy.org/omero/insight/tagset": + parents = {"*": tag_id} + else: + parents = {"": tag_id} # check if it has parents, assert the namespace is correct and # then add them as a dict - parents = {"", tag_id} raw_links = list(conn.getAnnotationLinks("TagAnnotation", ann_ids=[tag_id])) if raw_links: @@ -383,7 +416,8 @@ def get_tag_dict(conn, use_personal_tags): in raw_links if link.parent.ns.val == "openmicroscopy.org/omero/insight/tagset"} if name not in tag_dict: - tag_dict[name] = parents + tag_dict[name] = {} + tag_dict[name].update(parents) return tag_dict @@ -401,7 +435,9 @@ def tag_annotation(conn, obj, tags, tag_dict): tagSet = "" # if the Tag does not exist - if tag_value not in tag_dict: + # also check for maybe existing TagSet with same name + if tag_value not in tag_dict or tag_value in tag_dict\ + and "" not in tag_dict[tag_value]: # create TagAnnotation tag_ann = omero.gateway.TagAnnotationWrapper(conn) tag_ann.setValue(tag_value) @@ -410,38 +446,72 @@ def tag_annotation(conn, obj, tags, tag_dict): print(f"created new Tag '{tag_value}'.") # update tag dictionary tag_dict[tag_value] = {tagSet: tag_ann.id} + if tagSet: - if tagSet not in tag_dict: + # check if the TagSet exists, + # respect potential Tag with the same name + if tagSet not in tag_dict or tagSet in tag_dict\ + and "*" not in tag_dict[tagSet]: # create new TagSet - parent_tag = omero.gateway.TagAnnotationWrapper(conn) - parent_tag.setValue(tagSet) - parent_tag.ns = "openmicroscopy.org/omero/insight/tagset" - parent_tag.save() + parent_tag = TagAnnotationI() + parent_tag.textValue = rstring(tagSet) + parent_tag.ns = rstring(NSINSIGHTTAGSET) + parent_tag = update.saveAndReturnObject(parent_tag) + print(f"Created new TagSet {tagSet} {parent_tag.id.val}") # update tag dictionary - tag_dict[tagSet] = {"": parent_tag.id} + tag_dict[tagSet] = {"*": parent_tag.id} else: # or get existing parent_tag = conn.getObject("TagAnnotation", - tag_dict[tagSet][""]) + tag_dict[tagSet]["*"]) # create a Link and link Tag and TagSet link = AnnotationAnnotationLinkI() - link.parent = parent_tag._obj + link.parent = parent_tag link.child = tag_ann._obj update.saveObject(link) # if the Tag does exist else: - if tagSet and tagSet not in tag_dict: - # create new TagSet - parent_tag = omero.gateway.TagAnnotationWrapper(conn) - parent_tag.setValue(tagSet) - parent_tag.ns = "openmicroscopy.org/omero/insight/tagset" - parent_tag.save() + # if the correct Tag-TagSet combo does not exist + if tagSet and tagSet not in tag_dict[tag_value]: + # if the TagSet does not exist + if tagSet not in tag_dict: + # create new TagSet + parent_tag = TagAnnotationI() + parent_tag.textValue = rstring(tagSet) + parent_tag.ns = rstring(NSINSIGHTTAGSET) + parent_tag = update.saveAndReturnObject(parent_tag) + print(f"Created new TagSet {tagSet} {parent_tag.id.val}") + # update tag dictionary + tag_dict[tagSet] = {"*": parent_tag.id} + # if the TagSet does exist but does not contain the Tag + else: + parent_tag = conn.getObject("TagAnnotation", + tag_dict[tagSet]["*"]) + # create TagAnnotation + print(f"creating new TagAnnotation for '{tag_value}'" + + f"in the TagSet '{tagSet}'") + tag_ann = omero.gateway.TagAnnotationWrapper(conn) + tag_ann.setValue(tag_value) + tag_ann.save() # update tag dictionary - tag_dict[tagSet] = {"": parent_tag.id} - tag_ann = conn.getObject("TagAnnotation", - tag_dict[tag_value][tagSet]) - obj.linkAnnotation(tag_ann) + tag_dict[tag_value] = {tagSet: parent_tag.id} + # create a Link and link Tag and TagSet + link = AnnotationAnnotationLinkI() + link.parent = parent_tag + link.child = tag_ann._obj + update.saveObject(link) + obj.linkAnnotation(tag_ann) + # if the correct Tag-TagSet combo exists + elif tagSet and tagSet in tag_dict[tag_value]: + tag_ann = conn.getObject("TagAnnotation", + tag_dict[tag_value][tagSet]) + obj.linkAnnotation(tag_ann) + elif not tagSet: + tag_ann = conn.getObject("TagAnnotation", + tag_dict[tag_value][""]) + obj.linkAnnotation(tag_ann) + print(f"TagAnnotation:{tag_ann.id} created on {obj}") @@ -654,6 +724,7 @@ def parameters_parsing(client): if len(params["File_Annotation"]) == 1: # Poulate the parameter with None or same ID for all source params["File_Annotation"] *= len(params["IDs"]) + params["File_Annotation_multiplied"] = True params["File_Annotation"] = list(map(int, params["File_Annotation"])) assert len(params["File_Annotation"]) == len(params["IDs"]), "Number of \ @@ -666,8 +737,6 @@ def parameters_parsing(client): to_exclude = list(map(lambda x: x.replace('', params["Target name colname"]), to_exclude)) - # add Tag to excluded columns, as it is processed separately - to_exclude.append("tag") params["Columns to exclude"] = to_exclude if params["Separator"] == "guess": From be2d54a9cbbc6eb6ccf3051722dd02f3663e0d7b Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:45:06 +0100 Subject: [PATCH 068/127] implemented check for creation of new Tag; implemented Tag[TagId] feature --- omero/annotation_scripts/KeyVal_from_csv.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 7cf4ba433..5a213c2d9 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -653,9 +653,14 @@ def run_script(): scripts.Bool( "Use only personal Tags", grouping="2.8", default=False, description="Determines if Tags of other users in the group" + - "can be used on objects.\n Using only personal Tags might" + + " can be used on objects.\n Using only personal Tags might" + "lead to multiple Tags with the same name in one OMERO-group."), + scripts.Bool( + "Create new Tags", grouping="2.9", default=False, + description="Creates new Tags and Tag Sets if the ones" + + " specified in the .csv do not exist."), + authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" From d7377ae423d61f4a15fa36d09932335fc47e2f15 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Tue, 2 Jan 2024 16:02:16 +0100 Subject: [PATCH 069/127] NOW REALLY implemented check for creation of new Tag; implemented Tag[TagId] feature --- omero/annotation_scripts/KeyVal_from_csv.py | 55 ++++++++++++++++----- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 5a213c2d9..7d0ab8c5b 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -141,6 +141,7 @@ def main_loop(conn, script_params): exclude_empty_value = script_params["Exclude empty values"] split_on = script_params["Split value on"] use_personal_tags = script_params["Use only personal Tags"] + create_new_tags = script_params["Create new Tags"] file_ann_multiplied = script_params["File_Annotation_multiplied"] ntarget_processed = 0 @@ -239,7 +240,7 @@ def main_loop(conn, script_params): updated = annotate_object( conn, target_obj, parsed_row, parsed_head, parsed_ns, - exclude_empty_value, use_personal_tags + exclude_empty_value, use_personal_tags, create_new_tags ) if updated: @@ -338,9 +339,10 @@ def read_csv(conn, original_file, delimiter): def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, - use_personal_tags): + use_personal_tags, create_new_tags): updated = False tag_dict = {} + print(f"-->processing {obj}") for curr_ns in set(namespaces): kv_list = [] for ns, h, r in zip(namespaces, header, row): @@ -354,11 +356,11 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, tags_raw = [tag.strip() for tag in r.split(",")] tags = [] # separate the TagSet from the Tag --> - # [[Tag1, TagSet], [Tag2]] + # [[Tag1, TagSet(or TagId)], [Tag2]] for tag in tags_raw: tags.append([x.replace("]", "") for x in tag.split("[")]) - tag_annotation(conn, obj, tags, tag_dict) + tag_annotation(conn, obj, tags, tag_dict, create_new_tags) updated = True else: kv_list.append([h, r]) @@ -397,7 +399,8 @@ def get_tag_dict(conn, use_personal_tags): taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) tag_dict = {} for tag in taglist: - if use_personal_tags and tag.getOwner().id != conn.getUserId(): + if use_personal_tags and tag.getDetails()._owner._id._val !=\ + conn.getUserId(): continue name = tag.getTextValue().getValue() tag_id = tag.getId().getValue() @@ -422,7 +425,7 @@ def get_tag_dict(conn, use_personal_tags): return tag_dict -def tag_annotation(conn, obj, tags, tag_dict): +def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): """Create a TagAnnotation on an Object. If the Tag already exists use it. """ @@ -430,14 +433,26 @@ def tag_annotation(conn, obj, tags, tag_dict): for tag in tags: tag_value = tag[0] if len(tag) > 1: - tagSet = tag[1] + # check if it is an Id or a TagSet name + if tag[1].strip().isnumeric(): + tagId = int(tag[1].strip()) + tagSet = "" + else: + tagSet = tag[1] else: tagSet = "" + tagId = int() + # check if a Tag Set exists with the same name as the tag + condition = tag_value in tag_dict and len(tag_dict[tag_value]) == 1\ + and "*" in tag_dict[tag_value] # if the Tag does not exist # also check for maybe existing TagSet with same name - if tag_value not in tag_dict or tag_value in tag_dict\ - and "" not in tag_dict[tag_value]: + if tag_value not in tag_dict or condition: + assert create_new_tags is True, (f"Tag '{tag_value}'" + + " does not exist but" + + " creation of new Tags" + + " is not permitted") # create TagAnnotation tag_ann = omero.gateway.TagAnnotationWrapper(conn) tag_ann.setValue(tag_value) @@ -463,7 +478,7 @@ def tag_annotation(conn, obj, tags, tag_dict): else: # or get existing parent_tag = conn.getObject("TagAnnotation", - tag_dict[tagSet]["*"]) + tag_dict[tagSet]["*"])._obj # create a Link and link Tag and TagSet link = AnnotationAnnotationLinkI() link.parent = parent_tag @@ -476,6 +491,10 @@ def tag_annotation(conn, obj, tags, tag_dict): if tagSet and tagSet not in tag_dict[tag_value]: # if the TagSet does not exist if tagSet not in tag_dict: + assert create_new_tags is True, (f"Tag Set '{tagSet}'" + + " does not exist but" + + " creation of new Tags" + + " is not permitted") # create new TagSet parent_tag = TagAnnotationI() parent_tag.textValue = rstring(tagSet) @@ -487,7 +506,7 @@ def tag_annotation(conn, obj, tags, tag_dict): # if the TagSet does exist but does not contain the Tag else: parent_tag = conn.getObject("TagAnnotation", - tag_dict[tagSet]["*"]) + tag_dict[tagSet]["*"])._obj # create TagAnnotation print(f"creating new TagAnnotation for '{tag_value}'" + f"in the TagSet '{tagSet}'") @@ -507,10 +526,22 @@ def tag_annotation(conn, obj, tags, tag_dict): tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value][tagSet]) obj.linkAnnotation(tag_ann) - elif not tagSet: + # if there is just a normal Tag without Tag Set + elif not tagSet and not tagId: tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value][""]) obj.linkAnnotation(tag_ann) + # if there is a TagId given + elif tagId: + # check if the Tag-Id exists + keys = [] + for key in tag_dict.values(): + for k in key.values(): + keys.append(k) + assert tagId in keys, (f"The Tag-Id '{tagId}' is not" + + " in the permitted selection of Tags") + tag_ann = conn.getObject("TagAnnotation", tagId) + obj.linkAnnotation(tag_ann) print(f"TagAnnotation:{tag_ann.id} created on {obj}") From 821654d0dccf72388e3903350ee740db58865e81 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Wed, 3 Jan 2024 11:45:17 +0100 Subject: [PATCH 070/127] fixed bugs which showed up in testing --- omero/annotation_scripts/KeyVal_from_csv.py | 35 ++++++++++++++------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 7d0ab8c5b..c86627461 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -360,7 +360,9 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, for tag in tags_raw: tags.append([x.replace("]", "") for x in tag.split("[")]) - tag_annotation(conn, obj, tags, tag_dict, create_new_tags) + # annotate the Tags and return the updated tag dictionary + tag_dict = tag_annotation(conn, obj, tags, tag_dict, + create_new_tags) updated = True else: kv_list.append([h, r]) @@ -443,12 +445,10 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): tagSet = "" tagId = int() - # check if a Tag Set exists with the same name as the tag - condition = tag_value in tag_dict and len(tag_dict[tag_value]) == 1\ - and "*" in tag_dict[tag_value] # if the Tag does not exist - # also check for maybe existing TagSet with same name - if tag_value not in tag_dict or condition: + # check if a Tag Set exists with the same name as the tag + if tag_value not in tag_dict or tag_value in tag_dict and "" not\ + in tag_dict[tag_value]: assert create_new_tags is True, (f"Tag '{tag_value}'" + " does not exist but" + " creation of new Tags" + @@ -460,7 +460,10 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): obj.linkAnnotation(tag_ann) print(f"created new Tag '{tag_value}'.") # update tag dictionary - tag_dict[tag_value] = {tagSet: tag_ann.id} + if tag_value not in tag_dict: + tag_dict[tag_value] = {"": tag_ann.id} + else: + tag_dict[tag_value][""] = tag_ann.id if tagSet: # check if the TagSet exists, @@ -474,7 +477,10 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): parent_tag = update.saveAndReturnObject(parent_tag) print(f"Created new TagSet {tagSet} {parent_tag.id.val}") # update tag dictionary - tag_dict[tagSet] = {"*": parent_tag.id} + if tagSet in tag_dict: + tag_dict[tagSet]["*"] = parent_tag.id + else: + tag_dict[tagSet] = {"*": parent_tag.id} else: # or get existing parent_tag = conn.getObject("TagAnnotation", @@ -490,7 +496,7 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): # if the correct Tag-TagSet combo does not exist if tagSet and tagSet not in tag_dict[tag_value]: # if the TagSet does not exist - if tagSet not in tag_dict: + if tagSet not in tag_dict or "*" not in tag_dict[tagSet]: assert create_new_tags is True, (f"Tag Set '{tagSet}'" + " does not exist but" + " creation of new Tags" + @@ -514,7 +520,10 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): tag_ann.setValue(tag_value) tag_ann.save() # update tag dictionary - tag_dict[tag_value] = {tagSet: parent_tag.id} + if tag_value in tag_dict: + tag_dict[tag_value][tagSet] = tag_ann.id + else: + tag_dict[tag_value] = {tagSet: tag_ann.id} # create a Link and link Tag and TagSet link = AnnotationAnnotationLinkI() link.parent = parent_tag @@ -528,6 +537,7 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): obj.linkAnnotation(tag_ann) # if there is just a normal Tag without Tag Set elif not tagSet and not tagId: + # just get the existing normal Tag tag_ann = conn.getObject("TagAnnotation", tag_dict[tag_value][""]) obj.linkAnnotation(tag_ann) @@ -544,6 +554,8 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): obj.linkAnnotation(tag_ann) print(f"TagAnnotation:{tag_ann.id} created on {obj}") + # return the updated tag dictionary + return tag_dict def link_file_ann(conn, object_type, object_, file_ann): @@ -704,7 +716,8 @@ def run_script(): "Namespace (leave blank for default)", "Separator", "Columns to exclude", "Target ID colname", "Target name colname", "Exclude empty values", - "Attach csv to parents", "Split value on"] + "Attach csv to parents", "Split value on", + "Use only personal Tags", "Create new Tags"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") From f45336787d9a4e7465f842a84288fde4b923c990 Mon Sep 17 00:00:00 2001 From: JensWendt <92271654+JensWendt@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:25:12 +0100 Subject: [PATCH 071/127] various bug fixes --- omero/annotation_scripts/KeyVal_from_csv.py | 74 ++++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index c86627461..163ba6fc9 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -23,7 +23,7 @@ """ import omero -from omero.gateway import BlitzGateway +from omero.gateway import BlitzGateway, TagAnnotationWrapper from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts from omero.constants.metadata import NSCLIENTMAPANNOTATION @@ -59,7 +59,7 @@ def get_obj_name(omero_obj): """ Helper function """ if omero_obj.OMERO_CLASS == "Well": - return omero_obj.getWellPos() + return omero_obj.getWellPos().upper() else: return omero_obj.getName() @@ -188,15 +188,23 @@ def main_loop(conn, script_params): use_id = idx_id != -1 # use the obj_idx column if exist if not use_id: - # Identify images by name fail if two images have identical names idx_id = idx_name + # check if the names in the .csv contain duplicates + name_list = [row[idx_id] for row in rows] + duplicates = {name for name in name_list + if name_list.count(name) > 1} + print("duplicates:", duplicates) + assert not len(duplicates) > 0, ("The .csv contains" + + f"duplicates {duplicates} which" + + " makes it impossible" + + " to correctly allocate the" + + " annotations.") + # Identify target-objects by name fail if two have identical names target_d = dict() for target_obj in target_obj_l: name = get_obj_name(target_obj) - if target_type == "Well": - name = name.upper() assert name not in target_d.keys(), f"Target objects \ - identified by name have duplicate: {name}" + identified by name have at least one duplicate: {name}" target_d[name] = target_obj else: # Setting the dictionnary target_id:target_obj @@ -254,8 +262,8 @@ def main_loop(conn, script_params): link_file_ann(conn, source_type, source_object, file_ann) print("\n------------------------------------\n") - message = f"Added KV-pairs to \ - {ntarget_updated}/{ntarget_processed} {target_type}" + message = f"Added Annotations to \ + {ntarget_updated}/{ntarget_processed} {target_type}(s)" if file_ann_multiplied and len(missing_names) > 0: # subtract the processed names/ids from the @@ -266,7 +274,7 @@ def main_loop(conn, script_params): total_missing_names = len(missing_names) if total_missing_names > 0: - message += f". {total_missing_names} {target_type} not found \ + message += f". {total_missing_names} {target_type}(s) not found \ (using {'ID' if use_id else 'name'} to identify them)." return message, result_obj @@ -344,6 +352,7 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, tag_dict = {} print(f"-->processing {obj}") for curr_ns in set(namespaces): + updated = False kv_list = [] for ns, h, r in zip(namespaces, header, row): if ns == curr_ns and (len(r) > 0 or not exclude_empty_value): @@ -427,6 +436,21 @@ def get_tag_dict(conn, use_personal_tags): return tag_dict +def check_tag(id, obj): + """check if the Tag already exists on the object. + """ + tag_ids = [] + # get a list of all Annotations of the object + annotations = list(obj.listAnnotations()) + for ann in annotations: + if type(ann) is TagAnnotationWrapper: + tag_ids.append(ann.id) + if id in tag_ids: + return True + else: + return False + + def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): """Create a TagAnnotation on an Object. If the Tag already exists use it. @@ -532,15 +556,25 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): obj.linkAnnotation(tag_ann) # if the correct Tag-TagSet combo exists elif tagSet and tagSet in tag_dict[tag_value]: - tag_ann = conn.getObject("TagAnnotation", - tag_dict[tag_value][tagSet]) - obj.linkAnnotation(tag_ann) + id = tag_dict[tag_value][tagSet] + # check if the Tag already exists on the object + if check_tag(id, obj): + continue + else: + tag_ann = conn.getObject("TagAnnotation", + tag_dict[tag_value][tagSet]) + obj.linkAnnotation(tag_ann) # if there is just a normal Tag without Tag Set elif not tagSet and not tagId: - # just get the existing normal Tag - tag_ann = conn.getObject("TagAnnotation", - tag_dict[tag_value][""]) - obj.linkAnnotation(tag_ann) + id = tag_dict[tag_value][""] + # check if the Tag already exists on the object + if check_tag(id, obj): + continue + else: + # just get the existing normal Tag + tag_ann = conn.getObject("TagAnnotation", + tag_dict[tag_value][""]) + obj.linkAnnotation(tag_ann) # if there is a TagId given elif tagId: # check if the Tag-Id exists @@ -550,8 +584,12 @@ def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): keys.append(k) assert tagId in keys, (f"The Tag-Id '{tagId}' is not" + " in the permitted selection of Tags") - tag_ann = conn.getObject("TagAnnotation", tagId) - obj.linkAnnotation(tag_ann) + # check if the Tag already exists on the object + if check_tag(tagId, obj): + continue + else: + tag_ann = conn.getObject("TagAnnotation", tagId) + obj.linkAnnotation(tag_ann) print(f"TagAnnotation:{tag_ann.id} created on {obj}") # return the updated tag dictionary From 17d1d72b9bb6f843de12f85775b2e90311b09c8f Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jan 2024 13:58:09 +0100 Subject: [PATCH 072/127] tag feature validation --- omero/annotation_scripts/KeyVal_from_csv.py | 394 +++++++++----------- 1 file changed, 185 insertions(+), 209 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 163ba6fc9..6b1d684d0 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -32,6 +32,8 @@ from omero.util.populate_roi import DownloadingOriginalFileProvider import csv +from collections import defaultdict, OrderedDict +import re CHILD_OBJECTS = { @@ -152,6 +154,9 @@ def main_loop(conn, script_params): result_obj = None + # Dictionaries needed for the tags + tag_d, tagset_d, tagtree_d, tagid_d = None, None, None, None + # One file output per given ID source_objects = conn.getObjects(source_type, source_ids) for source_object, file_ann_id in zip(source_objects, file_ids): @@ -184,7 +189,7 @@ def main_loop(conn, script_params): if el in header] assert (idx_id != -1) or (idx_name != -1), "Neither \ - the column for the objects name or the objects index were found" + the column for the objects' name or the objects' index were found" use_id = idx_id != -1 # use the obj_idx column if exist if not use_id: @@ -213,6 +218,17 @@ def main_loop(conn, script_params): for target_obj in target_obj_l} ntarget_processed += len(target_d) + if tag_d is None and "tag" in [h.lower() for h in header]: + # Create the tag dictionary a single time if needed + tag_d, tagset_d, tagtree_d, tagid_d = get_tag_dict( + conn, use_personal_tags + ) + # Replace the tags in the CSV by the tag_id to use + rows, tag_d, tagset_d, tagtree_d, tagid_d = preprocess_tag_rows( + conn, header, rows, tag_d, tagset_d, tagtree_d, tagid_d, + create_new_tags, split_on + ) + ok_idxs = [i for i in range(len(header)) if i not in cols_to_ignore] for row in rows: # Iterate the CSV rows and search for the matching target @@ -247,8 +263,8 @@ def main_loop(conn, script_params): parsed_head = [header[i] for i in ok_idxs] updated = annotate_object( - conn, target_obj, parsed_row, parsed_head, parsed_ns, - exclude_empty_value, use_personal_tags, create_new_tags + conn, target_obj, parsed_row, parsed_head, + parsed_ns, exclude_empty_value, tagid_d, split_on ) if updated: @@ -346,33 +362,24 @@ def read_csv(conn, original_file, delimiter): return rows, header, namespaces -def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, - use_personal_tags, create_new_tags): +def annotate_object(conn, obj, row, header, namespaces, + exclude_empty_value, tagid_d, split_on): updated = False - tag_dict = {} print(f"-->processing {obj}") - for curr_ns in set(namespaces): + for curr_ns in list(OrderedDict.fromkeys(namespaces)): updated = False kv_list = [] + tag_id_l = [] for ns, h, r in zip(namespaces, header, row): if ns == curr_ns and (len(r) > 0 or not exclude_empty_value): - # check for "tag" in header and create&link a TagAnnotation if h.lower() == "tag": - # create a dict of existing tags, once - if len(tag_dict) == 0: - tag_dict = get_tag_dict(conn, use_personal_tags) - # create a list of tags - tags_raw = [tag.strip() for tag in r.split(",")] - tags = [] - # separate the TagSet from the Tag --> - # [[Tag1, TagSet(or TagId)], [Tag2]] - for tag in tags_raw: - tags.append([x.replace("]", "") for x in - tag.split("[")]) - # annotate the Tags and return the updated tag dictionary - tag_dict = tag_annotation(conn, obj, tags, tag_dict, - create_new_tags) - updated = True + if r == "": + continue + # check for "tag" in header and create&link a TagAnnotation + if split_on == "": # Default join for tags is "," + tag_id_l.extend(r.split(",")) + else: # given split_on is used (ahead of this function) + tag_id_l.append(r) else: kv_list.append([h, r]) if len(kv_list) > 0: # Always exclude empty KV pairs @@ -384,12 +391,23 @@ def annotate_object(conn, obj, row, header, namespaces, exclude_empty_value, obj.linkAnnotation(map_ann) print(f"MapAnnotation:{map_ann.id} created on {obj}") updated = True + if len(tag_id_l) > 0: + exist_ids = [ann.getId() for ann in obj.listAnnotations()] + for tag_id in tag_id_l: + tag_id = int(tag_id) + if tag_id not in exist_ids: + tag_ann = tagid_d[tag_id] + obj.linkAnnotation(tag_ann) + exist_ids.append(tag_id) + print(f"TagAnnotation:{tag_ann.id} created on {obj}") + updated = True + return updated def get_tag_dict(conn, use_personal_tags): - """Gets a dict of all existing Tag Names with their - respective OMERO IDs as values. + """ + Generate dictionnaries of the tags in the group. Parameters: -------------- @@ -400,200 +418,158 @@ def get_tag_dict(conn, use_personal_tags): Returns: ------------- - tag_dict: ``dictionary`` {tag_name : {tagSet_name : tag_id}} - tagSet_name: - "" for a TagAnnotation w/o TagSet parent\n - "*" for a TagSet\n - "" for a TagAnnotation with a TagSet parent + tag_d: dictionary of tag_ids {"tagA": [12], "tagB":[34,56]} + tagset_d: dictionary of tagset_ids {"tagsetX":[78]} + tagtree_d: dictionary of tags in tagsets {"tagsetX":{"tagA":[12]}} + tagid_d: dictionary of tag objects {12:tagA_obj, 34:tagB_obj} + """ - meta = conn.getMetadataService() - taglist = meta.loadSpecifiedAnnotations("TagAnnotation", "", "", None) - tag_dict = {} - for tag in taglist: - if use_personal_tags and tag.getDetails()._owner._id._val !=\ - conn.getUserId(): + tagtree_d = defaultdict(lambda: defaultdict(list)) + tag_d, tagset_d = defaultdict(list), defaultdict(list) + tagid_d = {} + + max_id = -1 + + uid = conn.getUserId() + for tag in conn.getObjects("TagAnnotation"): + is_owner = tag.getOwner().id == uid + if use_personal_tags and not is_owner: continue - name = tag.getTextValue().getValue() - tag_id = tag.getId().getValue() - namespace = tag.getNs() - if namespace is not None and\ - namespace._val == "openmicroscopy.org/omero/insight/tagset": - parents = {"*": tag_id} + + tagid_d[tag.id] = tag + max_id = max(max_id, tag.id) + tagname = tag.getValue() + if (tag.getNs() == NSINSIGHTTAGSET): + # It's a tagset + tagset_d[tagname].append((int(is_owner), tag.id)) + for lk in conn.getAnnotationLinks("TagAnnotation", + parent_ids=[tag.id]): + # Add all tags of this tagset in the tagtree + cname = lk.child.textValue.val + cid = lk.child.id.val + cown = int(lk.child.getDetails().owner.id.val == uid) + tagtree_d[tagname][cname].append((cown, cid)) else: - parents = {"": tag_id} - # check if it has parents, assert the namespace is correct and - # then add them as a dict - raw_links = list(conn.getAnnotationLinks("TagAnnotation", - ann_ids=[tag_id])) - if raw_links: - parents = {link.parent.textValue.val: tag_id for link - in raw_links if link.parent.ns.val == - "openmicroscopy.org/omero/insight/tagset"} - if name not in tag_dict: - tag_dict[name] = {} - tag_dict[name].update(parents) - - return tag_dict - - -def check_tag(id, obj): - """check if the Tag already exists on the object. + tag_d[tagname].append((int(is_owner), tag.id)) + + # Sorting the tag by index (and if owned or not) + # to keep only one + for k, v in tag_d.items(): + v.sort(key=lambda x: (x[0]*max_id + x[1])) + tag_d[k] = v[0][1] + for k, v in tagset_d.items(): + v.sort(key=lambda x: (x[0]*max_id + x[1])) + tagset_d[k] = v[0][1] + for k1, v1 in tagtree_d.items(): + for k2, v2 in v1.items(): + v2.sort(key=lambda x: (x[0]*max_id + x[1])) + tagtree_d[k1][k2] = v2[0][1] + + return tag_d, tagset_d, tagtree_d, tagid_d + + +def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, + tagtree_d, tagid_d, + create_new_tags, split_on): """ - tag_ids = [] - # get a list of all Annotations of the object - annotations = list(obj.listAnnotations()) - for ann in annotations: - if type(ann) is TagAnnotationWrapper: - tag_ids.append(ann.id) - if id in tag_ids: - return True - else: - return False - - -def tag_annotation(conn, obj, tags, tag_dict, create_new_tags): - """Create a TagAnnotation on an Object. - If the Tag already exists use it. + Replace the tags in the rows by tag_ids. + All done in preprocessing means that the script will fail before + annotations process starts. """ + regx_tag = re.compile(r"([^\[\]]+)?(?:\[(\d+)\])?(?:\[([^[\]]+)\])?") update = conn.getUpdateService() - for tag in tags: - tag_value = tag[0] - if len(tag) > 1: - # check if it is an Id or a TagSet name - if tag[1].strip().isnumeric(): - tagId = int(tag[1].strip()) - tagSet = "" - else: - tagSet = tag[1] - else: - tagSet = "" - tagId = int() - - # if the Tag does not exist - # check if a Tag Set exists with the same name as the tag - if tag_value not in tag_dict or tag_value in tag_dict and "" not\ - in tag_dict[tag_value]: - assert create_new_tags is True, (f"Tag '{tag_value}'" + - " does not exist but" + - " creation of new Tags" + - " is not permitted") - # create TagAnnotation - tag_ann = omero.gateway.TagAnnotationWrapper(conn) - tag_ann.setValue(tag_value) - tag_ann.save() - obj.linkAnnotation(tag_ann) - print(f"created new Tag '{tag_value}'.") - # update tag dictionary - if tag_value not in tag_dict: - tag_dict[tag_value] = {"": tag_ann.id} - else: - tag_dict[tag_value][""] = tag_ann.id - - if tagSet: - # check if the TagSet exists, - # respect potential Tag with the same name - if tagSet not in tag_dict or tagSet in tag_dict\ - and "*" not in tag_dict[tagSet]: - # create new TagSet - parent_tag = TagAnnotationI() - parent_tag.textValue = rstring(tagSet) - parent_tag.ns = rstring(NSINSIGHTTAGSET) - parent_tag = update.saveAndReturnObject(parent_tag) - print(f"Created new TagSet {tagSet} {parent_tag.id.val}") - # update tag dictionary - if tagSet in tag_dict: - tag_dict[tagSet]["*"] = parent_tag.id - else: - tag_dict[tagSet] = {"*": parent_tag.id} - else: - # or get existing - parent_tag = conn.getObject("TagAnnotation", - tag_dict[tagSet]["*"])._obj - # create a Link and link Tag and TagSet - link = AnnotationAnnotationLinkI() - link.parent = parent_tag - link.child = tag_ann._obj - update.saveObject(link) - - # if the Tag does exist - else: - # if the correct Tag-TagSet combo does not exist - if tagSet and tagSet not in tag_dict[tag_value]: - # if the TagSet does not exist - if tagSet not in tag_dict or "*" not in tag_dict[tagSet]: - assert create_new_tags is True, (f"Tag Set '{tagSet}'" + - " does not exist but" + - " creation of new Tags" + - " is not permitted") - # create new TagSet - parent_tag = TagAnnotationI() - parent_tag.textValue = rstring(tagSet) - parent_tag.ns = rstring(NSINSIGHTTAGSET) - parent_tag = update.saveAndReturnObject(parent_tag) - print(f"Created new TagSet {tagSet} {parent_tag.id.val}") - # update tag dictionary - tag_dict[tagSet] = {"*": parent_tag.id} - # if the TagSet does exist but does not contain the Tag - else: - parent_tag = conn.getObject("TagAnnotation", - tag_dict[tagSet]["*"])._obj - # create TagAnnotation - print(f"creating new TagAnnotation for '{tag_value}'" + - f"in the TagSet '{tagSet}'") - tag_ann = omero.gateway.TagAnnotationWrapper(conn) - tag_ann.setValue(tag_value) - tag_ann.save() - # update tag dictionary - if tag_value in tag_dict: - tag_dict[tag_value][tagSet] = tag_ann.id - else: - tag_dict[tag_value] = {tagSet: tag_ann.id} - # create a Link and link Tag and TagSet - link = AnnotationAnnotationLinkI() - link.parent = parent_tag - link.child = tag_ann._obj - update.saveObject(link) - obj.linkAnnotation(tag_ann) - # if the correct Tag-TagSet combo exists - elif tagSet and tagSet in tag_dict[tag_value]: - id = tag_dict[tag_value][tagSet] - # check if the Tag already exists on the object - if check_tag(id, obj): + + col_idxs = [i for i in range(len(header)) if header[i].lower() == "tag"] + res_rows = [] + for row in rows: + for col_idx in col_idxs: + values = row[col_idx] + tagid_l = [] + if split_on == "": + split_on = "," + values = values.split(split_on) + + for val in values: + val.strip() + # matching a regex to the value + re_match = regx_tag.match(val) + if re_match is None: continue - else: - tag_ann = conn.getObject("TagAnnotation", - tag_dict[tag_value][tagSet]) - obj.linkAnnotation(tag_ann) - # if there is just a normal Tag without Tag Set - elif not tagSet and not tagId: - id = tag_dict[tag_value][""] - # check if the Tag already exists on the object - if check_tag(id, obj): + tagname, tagid, tagset = re_match.groups() + has_tagset = (tagset is not None and tagset != "") + if tagid is not None: + # If an ID is found, take precedence + assert int(tagid) in tagid_d.keys(), \ + (f"The Tag ID:'{tagid}' is not" + + " in the permitted selection of Tags") + tag_o = tagid_d[tagid] + if tagname is not None or tagname != "": + assert tag_o.getValue() == tagname, ( + f"The tag {tagname} doesn't correspond" + + f" to the tag on the server with ID:{tagid}" + ) + tagid_l.append(str(tagid)) + # We found the tag continue - else: - # just get the existing normal Tag - tag_ann = conn.getObject("TagAnnotation", - tag_dict[tag_value][""]) - obj.linkAnnotation(tag_ann) - # if there is a TagId given - elif tagId: - # check if the Tag-Id exists - keys = [] - for key in tag_dict.values(): - for k in key.values(): - keys.append(k) - assert tagId in keys, (f"The Tag-Id '{tagId}' is not" + - " in the permitted selection of Tags") - # check if the Tag already exists on the object - if check_tag(tagId, obj): + elif tagname is None or tagname == "": continue - else: - tag_ann = conn.getObject("TagAnnotation", tagId) - obj.linkAnnotation(tag_ann) - print(f"TagAnnotation:{tag_ann.id} created on {obj}") - # return the updated tag dictionary - return tag_dict + if not has_tagset: + tag_exist = tagname in tag_d.keys() + assert (tag_exist or create_new_tags), ( + f"Tag '{tagname}'" + + " does not exist while" + + " creation of new Tags" + + " is not permitted" + ) + if not tag_exist: + tag_o = omero.gateway.TagAnnotationWrapper(conn) + tag_o.setValue(tagname) + tag_o.save() + tagid_d[tag_o.id] = tag_o + tag_d[tagname] = tag_o.id + print(f"creating new Tag for '{tagname}'") + tagid_l.append(str(tag_d[tagname])) + + else: # has tagset + tagset_exist = tagset in tagset_d.keys() + tag_exist = tagset_exist and (tagname in tagtree_d[tagset].keys()) + assert (tag_exist or create_new_tags), ( + f"Tag '{tagname}' " + + f"in TagSet '{tagset}'" + + " does not exist while" + + " creation of new Tags" + + " is not permitted" + ) + if not tag_exist: + tag_o = omero.gateway.TagAnnotationWrapper(conn) + tag_o.setValue(tagname) + tag_o.save() + tagid_d[tag_o.id] = tag_o + tag_d[tagname] = tag_o.id + if not tagset_exist: + tagset_o = omero.gateway.TagAnnotationWrapper(conn) + tagset_o.setValue(tagset) + tagset_o.setNs(NSINSIGHTTAGSET) + tagset_o.save() + tagid_d[tagset_o.id] = conn.getObject("TagAnnotation", tagset_o.id) + tagset_d[tagset] = tagset_o.id + print(f"Created new TagSet {tagset}:{tagset_o.id}") + # else: + tagset_o = tagid_d[tagset_d[tagset]] + link = AnnotationAnnotationLinkI() + link.parent = tagset_o._obj + link.child = tag_o._obj + update.saveObject(link) + tagtree_d[tagset][tagname] = tag_o.id + print(f"creating new Tag for '{tagname}' " + + f"in the TagSet '{tagset}'") + tagid_l.append(str(tagtree_d[tagset][tagname])) + + # joined list of tag_ids instead of ambiguous names + row[col_idx] = split_on.join(tagid_l) + res_rows.append(row) + return res_rows, tag_d, tagset_d, tagtree_d, tagid_d def link_file_ann(conn, object_type, object_, file_ann): From ab0a41d7984b99ffcf3ff0799a8d5656e274dce8 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jan 2024 13:59:02 +0100 Subject: [PATCH 073/127] Add exclusion of parent columns by default (same as generated by export) --- omero/annotation_scripts/KeyVal_from_csv.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 6b1d684d0..e23b886c7 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -671,11 +671,12 @@ def run_script(): scripts.List( "Columns to exclude", optional=False, grouping="2.2", - default=",", + default=",,", description="List of columns in the .csv file to exclude " + "from the key-value pair import. " + " and correspond to the two " + - "following parameters.").ofType(rstring("")), + "following parameters. corresponds " + + "to the six container types.").ofType(rstring("")), scripts.String( "Target ID colname", optional=False, grouping="2.3", @@ -800,6 +801,11 @@ def parameters_parsing(client): to_exclude = list(map(lambda x: x.replace('', params["Target name colname"]), to_exclude)) + if "" in to_exclude: + to_exclude.remove("") + to_exclude.extend(["PROJECT", "DATASET", "SCREEN", + "PLATE", "RUN", "WELL"]) + params["Columns to exclude"] = to_exclude if params["Separator"] == "guess": From 0bc3b2b637f83063433512974f7e569cb6627df7 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jan 2024 14:00:15 +0100 Subject: [PATCH 074/127] Small fixes --- omero/annotation_scripts/Export_to_csv.py | 2 +- omero/annotation_scripts/Remove_KeyVal.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index c0d13b77b..e5589a287 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -63,7 +63,7 @@ def get_obj_name(omero_obj): """ Helper function """ if omero_obj.OMERO_CLASS == "Well": - return omero_obj.getWellPos() + return omero_obj.getWellPos().upper() else: return omero_obj.getName() diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 3037b5ca1..baf69813d 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -275,7 +275,8 @@ def parameters_parsing(client): # unwrap rtypes to String, Integer etc params[key] = client.getInput(key, unwrap=True) - assert params[AGREEMENT], "Please confirm that you understood the risks." + assert params[AGREEMENT], "Please tick the box to confirm that you " +\ + "understood the risks." if params["Target Data_Type"] == "": params["Target Data_Type"] = params["Data_Type"] From 858dd0161421ee930b4b973713a4bc726ff0b717 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jan 2024 16:52:54 +0100 Subject: [PATCH 075/127] Added PlateACquisition in ancestry of plate images --- omero/annotation_scripts/Export_to_csv.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index e5589a287..a20be152d 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -161,10 +161,13 @@ def main_loop(conn, script_params): obj_id_l.append(target_obj.getId()) obj_name_l.append(get_obj_name(target_obj)) if include_parent: - ancestry = [(o.OMERO_CLASS, get_obj_name(o)) - for o in target_obj.getAncestry() - if o.OMERO_CLASS != "WellSample"] + ancestry = [] + for o in target_obj.getAncestry(): + if o.OMERO_CLASS == "WellSample": + o = o.getPlateAcquisition() + ancestry.append((o.OMERO_CLASS, get_obj_name(o))) obj_ancestry_l.append(ancestry[::-1]) + if result_obj is None: result_obj = target_obj print("\n------------------------------------\n") @@ -174,6 +177,8 @@ def main_loop(conn, script_params): # Complete ancestry for image/dataset/plate without parents norm_ancestry_l = [] if len(obj_ancestry_l) > 0: + # Issue with image that don't have a plateacquisition + # if combined with images that have max_level = max(map(lambda x: len(x), obj_ancestry_l)) for ancestry in obj_ancestry_l: norm_ancestry_l.append([("", "")] * From dd89ce4ed9e7331ca7efa0ea4a97d68b1bf83a6e Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Mon, 29 Jan 2024 22:33:27 +0100 Subject: [PATCH 076/127] Export of tags support --- omero/annotation_scripts/Export_to_csv.py | 73 ++++++++++++++++++--- omero/annotation_scripts/KeyVal_from_csv.py | 3 +- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index a20be152d..9a80be7ac 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -26,7 +26,7 @@ import omero from omero.gateway import BlitzGateway from omero.rtypes import rstring, rlong, robject -from omero.constants.metadata import NSCLIENTMAPANNOTATION +from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET import omero.scripts as scripts import tempfile @@ -137,11 +137,14 @@ def main_loop(conn, script_params): separator = script_params["Separator"] include_parent = script_params["Include column(s) of parents name"] include_namespace = script_params["Include namespace"] + include_tags = script_params["Include tags"] # One file output per given ID obj_ancestry_l = [] annotations_d = defaultdict(list) - obj_id_l, obj_name_l = [], [] + if include_tags: + all_tag_d = get_all_tags(conn) + obj_id_l, obj_name_l, tagannotation_l = [], [], [] for source_object in conn.getObjects(source_type, source_ids): result_obj = source_object @@ -153,10 +156,13 @@ def main_loop(conn, script_params): target_type, is_tag): annotations_d[0].append([]) # (when no ns exported, all ann in 0) for ns in namespace_l: - next_ann_l = get_existing_map_annotions(target_obj, - ns) + next_ann_l = get_existing_map_annotations(target_obj, + ns) annotations_d[ns].append(next_ann_l) annotations_d[0][-1].extend(next_ann_l) + if include_tags: + tagannotation_l.append(get_existing_tag_annotations(target_obj, + all_tag_d)) obj_id_l.append(target_obj.getId()) obj_name_l.append(get_obj_name(target_obj)) @@ -185,7 +191,8 @@ def main_loop(conn, script_params): (max_level - len(ancestry)) + ancestry) - ns_row, header_row, rows = build_rows(annotations_d, include_namespace) + ns_row, header_row, rows = build_rows(annotations_d, tagannotation_l, + include_namespace) ns_row, header_row, rows = sort_concat_rows(ns_row, header_row, rows, obj_id_l, obj_name_l, norm_ancestry_l) @@ -200,7 +207,27 @@ def main_loop(conn, script_params): return message, file_ann, result_obj -def get_existing_map_annotions(obj, namespace): +def get_all_tags(conn): + all_tag_d = {} + for tag in conn.getObjects("TagAnnotation"): + + tagname = tag.getValue() + if (tag.getNs() == NSINSIGHTTAGSET): + # It's a tagset, set all tag_id to "tagname[tagset_name]" + for lk in conn.getAnnotationLinks("TagAnnotation", + parent_ids=[tag.id]): + child_id = int(lk.child.id.val) + child_name = lk.child.textValue.val + all_tag_d[child_id] = f"{child_name}[{tagname}]" + elif tag.id not in all_tag_d.keys(): + # Normal tag and not in the dict yet + # (if found as part of a tagset, it is not overwritten) + all_tag_d[int(tag.id)] = tagname + + return all_tag_d + + +def get_existing_map_annotations(obj, namespace): "Return list of KV with updated keys with NS and occurences" annotation_l = [] for ann in obj.listAnnotations(ns=namespace): @@ -209,7 +236,17 @@ def get_existing_map_annotions(obj, namespace): return annotation_l -def build_rows(annotation_dict_l, include_namespace): +def get_existing_tag_annotations(obj, all_tag_d): + "Return list of tag names with tagset if any" + annotation_l = [] + for ann in obj.listAnnotations(): + if (isinstance(ann, omero.gateway.TagAnnotationWrapper) + and ann.getId() in all_tag_d.keys()): + annotation_l.append(all_tag_d[ann.getId()]) + return annotation_l + + +def build_rows(annotation_dict_l, tagannotation_l, include_namespace): ns_row = [] if include_namespace: header_row, rows = [], [[] for i in range(len(annotation_dict_l[0]))] @@ -223,6 +260,16 @@ def build_rows(annotation_dict_l, include_namespace): rows[i].extend(next_row) else: header_row, rows = group_keyvalues(annotation_dict_l[0]) + + if len(tagannotation_l) > 0: + max_tag = max(map(len, tagannotation_l)) + if include_namespace: + ns_row.extend([""] * max_tag) + header_row.extend(["TAG"] * max_tag) + for i, tag_l in enumerate(tagannotation_l): + rows[i].extend(tag_l) + rows[i].extend([""] * (max_tag - len(tag_l))) + return ns_row, header_row, rows @@ -416,14 +463,20 @@ def run_script(): scripts.Bool( "Include column(s) of parents name", optional=False, grouping="2.2", - description="Weather to include or not the name of the parent(s)" + + description="Whether to include or not the name of the parent(s)" + " objects as columns in the .csv.", default=False), scripts.Bool( "Include namespace", optional=False, grouping="2.3", - description="Weather to include or not the namespace" + - " of the annotations in the .csv.", default=False), + description="Whether to include the annotation namespaces" + + " in the .csv.", default=False), + + scripts.Bool( + "Include tags", optional=False, + grouping="2.4", + description="Whether to include tags in the .csv.", + default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index e23b886c7..97c6ae976 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -26,8 +26,7 @@ from omero.gateway import BlitzGateway, TagAnnotationWrapper from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts -from omero.constants.metadata import NSCLIENTMAPANNOTATION -from omero.constants.metadata import NSINSIGHTTAGSET +from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET from omero.model import AnnotationAnnotationLinkI, TagAnnotationI from omero.util.populate_roi import DownloadingOriginalFileProvider From f517eb7044e3b9022a5fec2d251e5140839a428b Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 31 Jan 2024 09:36:29 +0100 Subject: [PATCH 077/127] Shortened script doc and pointed to annscript-omero-guide repo --- .../Convert_KeyVal_namespace.py | 13 +++---------- omero/annotation_scripts/Export_to_csv.py | 19 +++++-------------- omero/annotation_scripts/KeyVal_from_csv.py | 18 ++++-------------- omero/annotation_scripts/Remove_KeyVal.py | 16 +++++----------- 4 files changed, 17 insertions(+), 49 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index c93a95ad2..18407779b 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -205,19 +205,12 @@ def run_script(): 'Convert_KV_namespace', """ This script converts the namespace of key-value pair annotations. - - TODO: add hyperlink to readthedocs - \t - Parameters: \t - - Data Type: parent-objects type in which target-objects are searched. - - IDs: IDs of the parent-objects. - - Target Data Type: Type of the target-objects that will be changed. - - Old Namespace: Namespace(s) of the annotations to group and change. - - New Namespace: New namespace for the annotations. + Check the guide for more information about the script parameters: + TODO link to omero-guides + https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#converting-the-key-value-pairs-namespace \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation - \t """, # Tabs are needed to add line breaks in the HTML scripts.String( diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 9a80be7ac..7a8b43d14 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -409,23 +409,14 @@ def run_script(): client = scripts.client( 'Export_KV_to_csv.py', """ - This script exports key-value pairs of objects to a .csv file. - Can also export a blank .csv with only of target objects' name and IDs. - (for example by providing a non-existing namespace) - TODO: add hyperlink to readthedocs + This script exports for the selected objects their name, IDs and associated + key-value pairs. \t - Parameters: - \t - - Data Type: parent-objects type in which target-objects are searched. - - IDs: IDs of the parent-objects. - - Target Data Type: target-objects type from which KV-pairs are exported. - - Namespace: Annotations having one of these namespace(s) will be exported. - \t - - Separator: Separator to be used in the .csv file. - - Include column(s) of parents name: Add columns for target-data parents. + Check the guide for more information about the script parameters: + TODO link to omero-guides + https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#exporting-key-value-pairs \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation - \t """, # Tabs are needed to add line breaks in the HTML scripts.String( diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 97c6ae976..2878915bc 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -607,24 +607,14 @@ def run_script(): client = scripts.client( 'Import_KV_from_csv', """ - Reads a .csv file to annotate target objects with key-value pairs. + Reads a .csv file to annotate the given objects with key-value pairs. TODO: add hyperlink to readthedocs \t - Parameters: - \t - - Data Type: parent-objects type in which target-objects are searched. - - IDs: IDs of the parent-objects. - - Target Data Type: Type of the target-objects that will be annotated. - - File_Annotation: IDs of .csv FileAnnotation or input file. - - Namespace: Namespace that will be given to the annotations. - \t - - Separator: Separator used in the .csv file. - - Columns to exclude: Columns name of the .csv file to exclude. - - Target ID colname: Column name in the .csv of the target IDs. - - Target name colname: Column name in the .csv of the target names. + Check the guide for more information about the script parameters: + TODO link to omero-guides + https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#importing-key-value-pairs \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation - \t """, # Tabs are needed to add line breaks in the HTML scripts.String( diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index baf69813d..987264f4e 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -193,20 +193,14 @@ def run_script(): client = scripts.client( 'Remove_KV.py', """ - This script deletes key-value pairs of all child objects founds. - Only key-value pairs of the namespace are deleted. - (default namespace correspond to editable KV pairs in web) - TODO: add hyperlink to readthedocs + This script deletes for the selected objects the key-value pairs + associated to the given namespace. \t - Parameters: - \t - - Data Type: parent-objects type in which target-objects are searched. - - IDs: IDs of the parent-objects. - - Target Data Type: Target-objects type of which KV-pairs are deleted. - - Namespace: Annotations having one of these namespace(s) will be deleted. + Check the guide for more information about the script parameters: + TODO link to omero-guides + https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#deleting-key-value-pairs \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation - \t """, # Tabs are needed to add line breaks in the HTML scripts.String( From 74f3e98c6db53f9a74231bdbbea57e2ccf3bc034 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 31 Jan 2024 09:54:21 +0100 Subject: [PATCH 078/127] modified param name to lowercase --- omero/annotation_scripts/Export_to_csv.py | 2 +- omero/annotation_scripts/KeyVal_from_csv.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 7a8b43d14..5f274f91a 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -479,7 +479,7 @@ def run_script(): keys = ["Data_Type", "IDs", "Target Data_Type", "Namespace (leave blank for default)", "Separator", "Include column(s) of parents name", - "Include namespace"] + "Include namespace", "Include tags"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 2878915bc..2f2252c7c 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -141,8 +141,8 @@ def main_loop(conn, script_params): attach_file = script_params["Attach csv to parents"] exclude_empty_value = script_params["Exclude empty values"] split_on = script_params["Split value on"] - use_personal_tags = script_params["Use only personal Tags"] - create_new_tags = script_params["Create new Tags"] + use_personal_tags = script_params["Use only personal tags"] + create_new_tags = script_params["Create new tags"] file_ann_multiplied = script_params["File_Annotation_multiplied"] ntarget_processed = 0 @@ -698,14 +698,14 @@ def run_script(): "create key duplicates."), scripts.Bool( - "Use only personal Tags", grouping="2.8", default=False, - description="Determines if Tags of other users in the group" + - " can be used on objects.\n Using only personal Tags might" + + "Use only personal tags", grouping="2.8", default=False, + description="Determines if tags of other users in the group" + + " can be used on objects.\n Using only personal tags might" + "lead to multiple Tags with the same name in one OMERO-group."), scripts.Bool( - "Create new Tags", grouping="2.9", default=False, - description="Creates new Tags and Tag Sets if the ones" + + "Create new tags", grouping="2.9", default=False, + description="Creates new tags and tagsets if the ones" + " specified in the .csv do not exist."), authors=["Christian Evenhuis", "Tom Boissonnet"], @@ -721,7 +721,7 @@ def run_script(): "Separator", "Columns to exclude", "Target ID colname", "Target name colname", "Exclude empty values", "Attach csv to parents", "Split value on", - "Use only personal Tags", "Create new Tags"] + "Use only personal tags", "Create new tags"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") From be0beb5900417caf0cde888d066783ec7c57d07c Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 31 Jan 2024 14:18:12 +0100 Subject: [PATCH 079/127] Support * for all namespace selection --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 3 ++- omero/annotation_scripts/Export_to_csv.py | 3 ++- omero/annotation_scripts/Remove_KeyVal.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 18407779b..07bcbf906 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -146,7 +146,8 @@ def get_existing_map_annotions(obj, namespace_l): keyval_l, ann_l = [], [] forbidden_deletion = [] for namespace in namespace_l: - for ann in obj.listAnnotations(ns=namespace): + p = {} if namespace == "*" else {"ns": namespace} + for ann in obj.listAnnotations(**p): if isinstance(ann, omero.gateway.MapAnnotationWrapper): if ann.canEdit(): # If not, skipping it keyval_l.extend([(k, v) for (k, v) in ann.getValue()]) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 5f274f91a..a05e8bbbb 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -230,7 +230,8 @@ def get_all_tags(conn): def get_existing_map_annotations(obj, namespace): "Return list of KV with updated keys with NS and occurences" annotation_l = [] - for ann in obj.listAnnotations(ns=namespace): + p = {} if namespace == "*" else {"ns": namespace} + for ann in obj.listAnnotations(**p): if isinstance(ann, omero.gateway.MapAnnotationWrapper): annotation_l.append(ann) return annotation_l diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 987264f4e..7fe937d63 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -143,8 +143,8 @@ def remove_map_annotations(conn, obj, namespace_l): mapann_ids = [] forbidden_deletion = [] for namespace in namespace_l: - anns = list(obj.listAnnotations(ns=namespace)) - for ann in anns: + p = {} if namespace == "*" else {"ns": namespace} + for ann in obj.listAnnotations(**p): if isinstance(ann, omero.gateway.MapAnnotationWrapper): if ann.canEdit(): # If not, skipping it mapann_ids.append(ann.id) From f45bc7f6af5772e2b62eb1d5ce2bd711c4672d9f Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 31 Jan 2024 22:01:18 +0100 Subject: [PATCH 080/127] * namespace input validation and fix for export --- .../Convert_KeyVal_namespace.py | 6 +++++ omero/annotation_scripts/Export_to_csv.py | 24 +++++++++++++++++-- omero/annotation_scripts/Remove_KeyVal.py | 6 +++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 07bcbf906..15e05b665 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -301,6 +301,12 @@ def parameters_parsing(client): if params["Target Data_Type"] == "Run": params["Target Data_Type"] = "PlateAcquisition" + # Remove duplicate entries from namespace list + tmp = params["Old Namespace (leave blank for default)"] + if "*" in tmp: + tmp = ["*"] + params["Old Namespace (leave blank for default)"] = list(set(tmp)) + return params diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index a05e8bbbb..b0811aea6 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -158,11 +158,13 @@ def main_loop(conn, script_params): for ns in namespace_l: next_ann_l = get_existing_map_annotations(target_obj, ns) - annotations_d[ns].append(next_ann_l) + if ns != "*": + annotations_d[ns].append(next_ann_l) annotations_d[0][-1].extend(next_ann_l) + if include_tags: tagannotation_l.append(get_existing_tag_annotations(target_obj, - all_tag_d)) + all_tag_d)) obj_id_l.append(target_obj.getId()) obj_name_l.append(get_obj_name(target_obj)) @@ -180,6 +182,17 @@ def main_loop(conn, script_params): csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) + if include_namespace and "*" in namespace_l: + # Assign entries of * namespace + ns_set = set() + for ann_l in annotations_d[0]: + ns_set = ns_set.union([ann.getNs() for ann in ann_l]) + for ann_l in annotations_d[0]: + for ns in ns_set: + annotations_d[ns].append([]) + for ann in ann_l: + annotations_d[ann.getNs()][-1].append(ann) + # Complete ancestry for image/dataset/plate without parents norm_ancestry_l = [] if len(obj_ancestry_l) > 0: @@ -541,6 +554,13 @@ def parameters_parsing(client): if params["Target Data_Type"] == "Run": params["Target Data_Type"] = "PlateAcquisition" + # Remove duplicate entries from namespace list + tmp = params["Namespace (leave blank for default)"] + if "*" in tmp: + tmp = ["*"] + params["Namespace (leave blank for default)"] = list(set(tmp)) + + return params diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 7fe937d63..8f301ccb2 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -290,6 +290,12 @@ def parameters_parsing(client): if params["Target Data_Type"] == "Run": params["Target Data_Type"] = "PlateAcquisition" + # Remove duplicate entries from namespace list + tmp = params["Namespace (leave blank for default)"] + if "*" in tmp: + tmp = ["*"] + params["Namespace (leave blank for default)"] = list(set(tmp)) + return params From f4ad5dbdd13243606dce46db9041abf4cb892bfa Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 1 Feb 2024 18:28:11 +0100 Subject: [PATCH 081/127] doc minor changes --- .../Convert_KeyVal_namespace.py | 2 +- omero/annotation_scripts/Export_to_csv.py | 2 +- omero/annotation_scripts/KeyVal_from_csv.py | 15 +++++++-------- omero/annotation_scripts/Remove_KeyVal.py | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 15e05b665..9a4886662 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -207,7 +207,7 @@ def run_script(): """ This script converts the namespace of key-value pair annotations. \t - Check the guide for more information about the script parameters: + Check the guide for more information on parameters and errors: TODO link to omero-guides https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#converting-the-key-value-pairs-namespace \t diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index b0811aea6..ad2fa2ced 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -426,7 +426,7 @@ def run_script(): This script exports for the selected objects their name, IDs and associated key-value pairs. \t - Check the guide for more information about the script parameters: + Check the guide for more information on parameters and errors: TODO link to omero-guides https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#exporting-key-value-pairs \t diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 2f2252c7c..f20af14f5 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -499,8 +499,8 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, if tagid is not None: # If an ID is found, take precedence assert int(tagid) in tagid_d.keys(), \ - (f"The Tag ID:'{tagid}' is not" + - " in the permitted selection of Tags") + (f"The tag ID:'{tagid}' is not" + + " in the permitted selection of tags") tag_o = tagid_d[tagid] if tagname is not None or tagname != "": assert tag_o.getValue() == tagname, ( @@ -518,7 +518,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, assert (tag_exist or create_new_tags), ( f"Tag '{tagname}'" + " does not exist while" + - " creation of new Tags" + + " creation of new tags" + " is not permitted" ) if not tag_exist: @@ -537,7 +537,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, f"Tag '{tagname}' " + f"in TagSet '{tagset}'" + " does not exist while" + - " creation of new Tags" + + " creation of new tags" + " is not permitted" ) if not tag_exist: @@ -562,7 +562,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, update.saveObject(link) tagtree_d[tagset][tagname] = tag_o.id print(f"creating new Tag for '{tagname}' " + - f"in the TagSet '{tagset}'") + f"in the tagset '{tagset}'") tagid_l.append(str(tagtree_d[tagset][tagname])) # joined list of tag_ids instead of ambiguous names @@ -608,9 +608,8 @@ def run_script(): 'Import_KV_from_csv', """ Reads a .csv file to annotate the given objects with key-value pairs. - TODO: add hyperlink to readthedocs \t - Check the guide for more information about the script parameters: + Check the guide for more information on parameters and errors: TODO link to omero-guides https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#importing-key-value-pairs \t @@ -701,7 +700,7 @@ def run_script(): "Use only personal tags", grouping="2.8", default=False, description="Determines if tags of other users in the group" + " can be used on objects.\n Using only personal tags might" + - "lead to multiple Tags with the same name in one OMERO-group."), + "lead to multiple tags with the same name in one OMERO-group."), scripts.Bool( "Create new tags", grouping="2.9", default=False, diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 8f301ccb2..1102f772e 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -196,7 +196,7 @@ def run_script(): This script deletes for the selected objects the key-value pairs associated to the given namespace. \t - Check the guide for more information about the script parameters: + Check the guide for more information on parameters and errors: TODO link to omero-guides https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#deleting-key-value-pairs \t From 67ca20697da6985b11d6213513b950e3651ae7a2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 14:48:17 +0100 Subject: [PATCH 082/127] changed namespace param name --- .../Convert_KeyVal_namespace.py | 20 +++++++++---------- omero/annotation_scripts/Export_to_csv.py | 10 +++++----- omero/annotation_scripts/KeyVal_from_csv.py | 10 +++++----- omero/annotation_scripts/Remove_KeyVal.py | 12 +++++------ 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 9a4886662..b74193b08 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -112,8 +112,8 @@ def main_loop(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] - old_namespace = script_params["Old Namespace (leave blank for default)"] - new_namespace = script_params["New Namespace (leave blank for default)"] + old_namespace = script_params["Old Namespace (blank for default)"] + new_namespace = script_params["New Namespace (blank for default)"] ntarget_processed = 0 ntarget_updated = 0 @@ -231,13 +231,13 @@ def run_script(): values=target_types, default=""), scripts.List( - "Old Namespace (leave blank for default)", optional=True, + "Old Namespace (blank for default)", optional=True, grouping="1.4", description="The namespace(s) of the annotations to " + "group and change.").ofType(rstring("")), scripts.String( - "New Namespace (leave blank for default)", optional=True, + "New Namespace (blank for default)", optional=True, grouping="1.5", description="The new namespace for the annotations."), @@ -250,8 +250,8 @@ def run_script(): params = parameters_parsing(client) print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", - "Old Namespace (leave blank for default)", - "New Namespace (leave blank for default)"] + "Old Namespace (blank for default)", + "New Namespace (blank for default)"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") @@ -276,8 +276,8 @@ def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters params["File_Annotation"] = None - params["Old Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] - params["New Namespace (leave blank for default)"] = NSCLIENTMAPANNOTATION + params["Old Namespace (blank for default)"] = [NSCLIENTMAPANNOTATION] + params["New Namespace (blank for default)"] = NSCLIENTMAPANNOTATION for key in client.getInputKeys(): if client.getInput(key): @@ -302,10 +302,10 @@ def parameters_parsing(client): params["Target Data_Type"] = "PlateAcquisition" # Remove duplicate entries from namespace list - tmp = params["Old Namespace (leave blank for default)"] + tmp = params["Old Namespace (blank for default)"] if "*" in tmp: tmp = ["*"] - params["Old Namespace (leave blank for default)"] = list(set(tmp)) + params["Old Namespace (blank for default)"] = list(set(tmp)) return params diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index ad2fa2ced..9fec56711 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -133,7 +133,7 @@ def main_loop(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] - namespace_l = script_params["Namespace (leave blank for default)"] + namespace_l = script_params["Namespace (blank for default)"] separator = script_params["Separator"] include_parent = script_params["Include column(s) of parents name"] include_namespace = script_params["Include namespace"] @@ -491,7 +491,7 @@ def run_script(): params = parameters_parsing(client) print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", - "Namespace (leave blank for default)", + "Namespace (blank for default)", "Separator", "Include column(s) of parents name", "Include namespace", "Include tags"] for k in keys: @@ -526,7 +526,7 @@ def run_script(): def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters - params["Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] + params["Namespace (blank for default)"] = [NSCLIENTMAPANNOTATION] for key in client.getInputKeys(): if client.getInput(key): @@ -555,10 +555,10 @@ def parameters_parsing(client): params["Target Data_Type"] = "PlateAcquisition" # Remove duplicate entries from namespace list - tmp = params["Namespace (leave blank for default)"] + tmp = params["Namespace (blank for default)"] if "*" in tmp: tmp = ["*"] - params["Namespace (leave blank for default)"] = list(set(tmp)) + params["Namespace (blank for default)"] = list(set(tmp)) return params diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index f20af14f5..f2b317002 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -133,7 +133,7 @@ def main_loop(conn, script_params): target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] file_ids = script_params["File_Annotation"] - namespace = script_params["Namespace (leave blank for default)"] + namespace = script_params["Namespace (for default or from csv)"] to_exclude = script_params["Columns to exclude"] target_id_colname = script_params["Target ID colname"] target_name_colname = script_params["Target name colname"] @@ -640,7 +640,7 @@ def run_script(): "attached CSV file on each parent object."), scripts.String( - "Namespace (leave blank for default)", + "Namespace (blank for default or from csv)", optional=True, grouping="1.4", description="Namespace given to the created key-value " + "pairs annotations. Default is the client" + @@ -699,7 +699,7 @@ def run_script(): scripts.Bool( "Use only personal tags", grouping="2.8", default=False, description="Determines if tags of other users in the group" + - " can be used on objects.\n Using only personal tags might" + + " can be used on objects.\n Using only personal tags might " + "lead to multiple tags with the same name in one OMERO-group."), scripts.Bool( @@ -716,7 +716,7 @@ def run_script(): params = parameters_parsing(client) print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", - "Namespace (leave blank for default)", + "Namespace (blank for default or from csv)", "Separator", "Columns to exclude", "Target ID colname", "Target name colname", "Exclude empty values", "Attach csv to parents", "Split value on", @@ -745,7 +745,7 @@ def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters params["File_Annotation"] = None - params["Namespace (leave blank for default)"] = NSCLIENTMAPANNOTATION + params["Namespace (blank for default or from csv)"] = NSCLIENTMAPANNOTATION params["Split value on"] = "" for key in client.getInputKeys(): diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 1102f772e..cb835659a 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -116,7 +116,7 @@ def main_loop(conn, script_params): source_type = script_params["Data_Type"] target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] - namespace_l = script_params["Namespace (leave blank for default)"] + namespace_l = script_params["Namespace (blank for default)"] nsuccess = 0 ntotal = 0 @@ -219,7 +219,7 @@ def run_script(): values=target_types, default=""), scripts.List( - "Namespace (leave blank for default)", optional=True, + "Namespace (blank for default)", optional=True, grouping="1.3", description="Annotation with these namespace will " + "be deleted. Default is the client" + @@ -240,7 +240,7 @@ def run_script(): params = parameters_parsing(client) print("Input parameters:") keys = ["Data_Type", "IDs", "Target Data_Type", - "Namespace (leave blank for default)"] + "Namespace (blank for default)"] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") @@ -262,7 +262,7 @@ def run_script(): def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters - params["Namespace (leave blank for default)"] = [NSCLIENTMAPANNOTATION] + params["Namespace (blank for default)"] = [NSCLIENTMAPANNOTATION] for key in client.getInputKeys(): if client.getInput(key): @@ -291,10 +291,10 @@ def parameters_parsing(client): params["Target Data_Type"] = "PlateAcquisition" # Remove duplicate entries from namespace list - tmp = params["Namespace (leave blank for default)"] + tmp = params["Namespace (blank for default)"] if "*" in tmp: tmp = ["*"] - params["Namespace (leave blank for default)"] = list(set(tmp)) + params["Namespace (blank for default)"] = list(set(tmp)) return params From 6c8738754a3f572cfb4ab31f96b5b2a512523e81 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 14:50:13 +0100 Subject: [PATCH 083/127] changed namespace param name fix --- omero/annotation_scripts/Export_to_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 9fec56711..3671d7a8a 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -449,7 +449,7 @@ def run_script(): values=target_types, default=""), scripts.List( - "Namespace (leave blank for default)", optional=True, + "Namespace (blank for default)", optional=True, grouping="1.3", description="Namespace(s) to include for the export of key-" + "value pairs annotations. Default is the client" + From 5eb050e01606da5b7bd60d8eefda59d2324a3f05 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 15:23:51 +0100 Subject: [PATCH 084/127] ns param override csv ns --- omero/annotation_scripts/KeyVal_from_csv.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index f2b317002..b852eac3b 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -133,7 +133,7 @@ def main_loop(conn, script_params): target_type = script_params["Target Data_Type"] source_ids = script_params["IDs"] file_ids = script_params["File_Annotation"] - namespace = script_params["Namespace (for default or from csv)"] + namespace = script_params["Namespace (blank for default or from csv)"] to_exclude = script_params["Columns to exclude"] target_id_colname = script_params["Target ID colname"] target_name_colname = script_params["Target name colname"] @@ -171,8 +171,10 @@ def main_loop(conn, script_params): original_file = file_ann.getFile()._obj rows, header, namespaces = read_csv(conn, original_file, separator) - if len(namespaces) == 0: + if namespace is not None: namespaces = [namespace] * len(header) + elif len(namespaces) == 0: + namespaces = [NSCLIENTMAPANNOTATION] * len(header) is_tag = source_type == "TagAnnotation" target_obj_l = target_iterator(conn, source_object, @@ -745,7 +747,7 @@ def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters params["File_Annotation"] = None - params["Namespace (blank for default or from csv)"] = NSCLIENTMAPANNOTATION + params["Namespace (blank for default or from csv)"] = None params["Split value on"] = "" for key in client.getInputKeys(): From 042dc2a601f9f5688c3fd948b9b4500bd09c36c6 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 18:03:38 +0100 Subject: [PATCH 085/127] Refactor param names with global variable --- .../Convert_KeyVal_namespace.py | 61 ++-- omero/annotation_scripts/Export_to_csv.py | 95 ++++--- omero/annotation_scripts/KeyVal_from_csv.py | 263 ++++++++++-------- omero/annotation_scripts/Remove_KeyVal.py | 61 ++-- 4 files changed, 263 insertions(+), 217 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index b74193b08..a8f31d3ce 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -49,6 +49,12 @@ "Screen", "Plate", "Well", "Run"] } +P_DTYPE = "Data_Type" # Do not change +P_IDS = "IDs" # Do not change +P_TARG_DTYPE = "Target Data_Type" +P_OLD_NS = "Old Namespace (blank for default)" +P_NEW_NS = "New Namespace (blank for default)" + def get_children_recursive(source_object, target_type): if CHILD_OBJECTS[source_object.OMERO_CLASS] == target_type: @@ -109,11 +115,11 @@ def main_loop(conn, script_params): - Remove annotations with old namespace - Create annotations with new namespace """ - source_type = script_params["Data_Type"] - target_type = script_params["Target Data_Type"] - source_ids = script_params["IDs"] - old_namespace = script_params["Old Namespace (blank for default)"] - new_namespace = script_params["New Namespace (blank for default)"] + source_type = script_params[P_DTYPE] + target_type = script_params[P_TARG_DTYPE] + source_ids = script_params[P_IDS] + old_namespace = script_params[P_OLD_NS] + new_namespace = script_params[P_NEW_NS] ntarget_processed = 0 ntarget_updated = 0 @@ -215,29 +221,29 @@ def run_script(): """, # Tabs are needed to add line breaks in the HTML scripts.String( - "Data_Type", optional=False, grouping="1", + P_DTYPE, optional=False, grouping="1", description="Parent-data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( - "IDs", optional=False, grouping="1.1", + P_IDS, optional=False, grouping="1.1", description="List of parent-data IDs containing the objects " + "to annotate.").ofType(rlong(0)), scripts.String( - "Target Data_Type", optional=False, grouping="1.2", + P_TARG_DTYPE, optional=False, grouping="1.2", description="The data type for which key-value pair annotations " + "will be converted.", values=target_types, default=""), scripts.List( - "Old Namespace (blank for default)", optional=True, + P_OLD_NS, optional=True, grouping="1.4", description="The namespace(s) of the annotations to " + "group and change.").ofType(rstring("")), scripts.String( - "New Namespace (blank for default)", optional=True, + P_NEW_NS, optional=True, grouping="1.5", description="The new namespace for the annotations."), @@ -249,9 +255,7 @@ def run_script(): try: params = parameters_parsing(client) print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", - "Old Namespace (blank for default)", - "New Namespace (blank for default)"] + keys = [P_DTYPE, P_IDS, P_TARG_DTYPE, P_OLD_NS, P_NEW_NS] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") @@ -275,37 +279,36 @@ def run_script(): def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters - params["File_Annotation"] = None - params["Old Namespace (blank for default)"] = [NSCLIENTMAPANNOTATION] - params["New Namespace (blank for default)"] = NSCLIENTMAPANNOTATION + params[P_OLD_NS] = [NSCLIENTMAPANNOTATION] + params[P_NEW_NS] = NSCLIENTMAPANNOTATION for key in client.getInputKeys(): if client.getInput(key): params[key] = client.getInput(key, unwrap=True) - if params["Target Data_Type"] == "": - params["Target Data_Type"] = params["Data_Type"] - elif " " in params["Target Data_Type"]: + if params[P_TARG_DTYPE] == "": + params[P_TARG_DTYPE] = params[P_DTYPE] + elif " " in params[P_TARG_DTYPE]: # Getting rid of the trailing '---' added for the UI - params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + params[P_TARG_DTYPE] = params[P_TARG_DTYPE].split(" ")[1] - assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + assert params[P_TARG_DTYPE] in ALLOWED_PARAM[params[P_DTYPE]], \ (f"{params['Target Data_Type']} is not a valid target for " + f"{params['Data_Type']}.") - if params["Data_Type"] == "Tag": - params["Data_Type"] = "TagAnnotation" + if params[P_DTYPE] == "Tag": + params[P_DTYPE] = "TagAnnotation" - if params["Data_Type"] == "Run": - params["Data_Type"] = "Acquisition" - if params["Target Data_Type"] == "Run": - params["Target Data_Type"] = "PlateAcquisition" + if params[P_DTYPE] == "Run": + params[P_DTYPE] = "Acquisition" + if params[P_TARG_DTYPE] == "Run": + params[P_TARG_DTYPE] = "PlateAcquisition" # Remove duplicate entries from namespace list - tmp = params["Old Namespace (blank for default)"] + tmp = params[P_OLD_NS] if "*" in tmp: tmp = ["*"] - params["Old Namespace (blank for default)"] = list(set(tmp)) + params[P_OLD_NS] = list(set(tmp)) return params diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 3671d7a8a..ad55fa909 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -55,6 +55,15 @@ "Screen", "Plate", "Well", "Run"] } +P_DTYPE = "Data_Type" # Do not change +P_IDS = "IDs" # Do not change +P_TARG_DTYPE = "Target Data_Type" +P_NAMESPACE = "Namespace (blank for default)" +P_CSVSEP = "CSV separator" +P_INCL_PARENT = "Include column(s) of parents name" +P_INCL_NS = "Include namespace" +P_INCL_TAG = "Include tags" + # Add your OMERO.web URL for direct download from link: # eg https://omero-adress.org/webclient WEBCLIENT_URL = "" @@ -130,14 +139,14 @@ def main_loop(conn, script_params): - Sort rows (useful for wells) - Write a single CSV file """ - source_type = script_params["Data_Type"] - target_type = script_params["Target Data_Type"] - source_ids = script_params["IDs"] - namespace_l = script_params["Namespace (blank for default)"] - separator = script_params["Separator"] - include_parent = script_params["Include column(s) of parents name"] - include_namespace = script_params["Include namespace"] - include_tags = script_params["Include tags"] + source_type = script_params[P_DTYPE] + target_type = script_params[P_TARG_DTYPE] + source_ids = script_params[P_IDS] + namespace_l = script_params[P_NAMESPACE] + separator = script_params[P_CSVSEP] + include_parent = script_params[P_INCL_PARENT] + include_namespace = script_params[P_INCL_NS] + include_tags = script_params[P_INCL_TAG] # One file output per given ID obj_ancestry_l = [] @@ -434,22 +443,22 @@ def run_script(): """, # Tabs are needed to add line breaks in the HTML scripts.String( - "Data_Type", optional=False, grouping="1", + P_DTYPE, optional=False, grouping="1", description="Parent data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( - "IDs", optional=False, grouping="1.1", + P_IDS, optional=False, grouping="1.1", description="List of parent data IDs containing the objects " + "to delete annotation from.").ofType(rlong(0)), scripts.String( - "Target Data_Type", optional=True, grouping="1.2", + P_TARG_DTYPE, optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", values=target_types, default=""), scripts.List( - "Namespace (blank for default)", optional=True, + P_NAMESPACE, optional=True, grouping="1.3", description="Namespace(s) to include for the export of key-" + "value pairs annotations. Default is the client" + @@ -457,28 +466,28 @@ def run_script(): "OMERO.web").ofType(rstring("")), scripts.Bool( - "Advanced parameters", optional=True, grouping="2", default=False, + "Other parameters", optional=True, grouping="2", default=True, description="Ticking or unticking this has no effect"), scripts.String( - "Separator", optional=False, grouping="2.1", + P_CSVSEP, optional=False, grouping="2.1", description="Choose the .csv separator.", values=separators, default="TAB"), scripts.Bool( - "Include column(s) of parents name", optional=False, + P_INCL_PARENT, optional=False, grouping="2.2", description="Whether to include or not the name of the parent(s)" + " objects as columns in the .csv.", default=False), scripts.Bool( - "Include namespace", optional=False, + P_INCL_NS, optional=False, grouping="2.3", description="Whether to include the annotation namespaces" + " in the .csv.", default=False), scripts.Bool( - "Include tags", optional=False, + P_INCL_TAG, optional=False, grouping="2.4", description="Whether to include tags in the .csv.", default=False), @@ -489,14 +498,6 @@ def run_script(): ) try: params = parameters_parsing(client) - print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", - "Namespace (blank for default)", - "Separator", "Include column(s) of parents name", - "Include namespace", "Include tags"] - for k in keys: - print(f"\t- {k}: {params[k]}") - print("\n####################################\n") # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) @@ -526,40 +527,46 @@ def run_script(): def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters - params["Namespace (blank for default)"] = [NSCLIENTMAPANNOTATION] + params[P_NAMESPACE] = [NSCLIENTMAPANNOTATION] for key in client.getInputKeys(): if client.getInput(key): # unwrap rtypes to String, Integer etc params[key] = client.getInput(key, unwrap=True) - if params["Target Data_Type"] == "": - params["Target Data_Type"] = params["Data_Type"] - elif " " in params["Target Data_Type"]: + if params[P_TARG_DTYPE] == "": + params[P_TARG_DTYPE] = params[P_DTYPE] + elif " " in params[P_TARG_DTYPE]: # Getting rid of the trailing '---' added for the UI - params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + params[P_TARG_DTYPE] = params[P_TARG_DTYPE].split(" ")[1] - assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + assert params[P_TARG_DTYPE] in ALLOWED_PARAM[params[P_DTYPE]], \ (f"{params['Target Data_Type']} is not a valid target for " + f"{params['Data_Type']}.") - if params["Separator"] == "TAB": - params["Separator"] = "\t" - - if params["Data_Type"] == "Tag": - params["Data_Type"] = "TagAnnotation" - - if params["Data_Type"] == "Run": - params["Data_Type"] = "Acquisition" - if params["Target Data_Type"] == "Run": - params["Target Data_Type"] = "PlateAcquisition" - # Remove duplicate entries from namespace list - tmp = params["Namespace (blank for default)"] + tmp = params[P_NAMESPACE] if "*" in tmp: tmp = ["*"] - params["Namespace (blank for default)"] = list(set(tmp)) + params[P_NAMESPACE] = list(set(tmp)) + + if params[P_DTYPE] == "Tag": + params[P_DTYPE] = "TagAnnotation" + + if params[P_DTYPE] == "Run": + params[P_DTYPE] = "Acquisition" + if params[P_TARG_DTYPE] == "Run": + params[P_TARG_DTYPE] = "PlateAcquisition" + + print("Input parameters:") + keys = [P_DTYPE, P_IDS, P_TARG_DTYPE, P_NAMESPACE, + P_CSVSEP, P_INCL_PARENT, P_INCL_NS, P_INCL_TAG] + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") + if params[P_CSVSEP] == "TAB": + params[P_CSVSEP] = "\t" return params diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index b852eac3b..a63f038c6 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -27,7 +27,7 @@ from omero.rtypes import rstring, rlong, robject import omero.scripts as scripts from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET -from omero.model import AnnotationAnnotationLinkI, TagAnnotationI +from omero.model import AnnotationAnnotationLinkI from omero.util.populate_roi import DownloadingOriginalFileProvider import csv @@ -56,6 +56,22 @@ "Screen", "Plate", "Well", "Run"] } +P_DTYPE = "Data_Type" # Do not change +P_FILE_ANN = "File_Annotation" # Do not change +P_IDS = "IDs" # Do not change +P_TARG_DTYPE = "Target Data_Type" +P_NAMESPACE = "Namespace (blank for default or from csv)" +P_CSVSEP = "CSV separator" +P_EXCL_COL = "Columns to exclude" +P_TARG_COLID = "Target ID colname" +P_TARG_COLNAME = "Target name colname" +P_EXCL_EMPTY = "Exclude empty values" +P_ATTACH = "Attach CSV file" +P_SPLIT_CELL = "Split values on" +P_IMPORT_TAGS = "Import tags" +P_OWN_TAG = "Only use personal tags" +P_ALLOW_NEWTAG = "Allow tag creation" + def get_obj_name(omero_obj): """ Helper function """ @@ -129,20 +145,21 @@ def main_loop(conn, script_params): - Annotate the objects - (opt) attach the CSV to the source object """ - source_type = script_params["Data_Type"] - target_type = script_params["Target Data_Type"] - source_ids = script_params["IDs"] - file_ids = script_params["File_Annotation"] - namespace = script_params["Namespace (blank for default or from csv)"] - to_exclude = script_params["Columns to exclude"] - target_id_colname = script_params["Target ID colname"] - target_name_colname = script_params["Target name colname"] - separator = script_params["Separator"] - attach_file = script_params["Attach csv to parents"] - exclude_empty_value = script_params["Exclude empty values"] - split_on = script_params["Split value on"] - use_personal_tags = script_params["Use only personal tags"] - create_new_tags = script_params["Create new tags"] + source_type = script_params[P_DTYPE] + target_type = script_params[P_TARG_DTYPE] + source_ids = script_params[P_IDS] + file_ids = script_params[P_FILE_ANN] + namespace = script_params[P_NAMESPACE] + to_exclude = script_params[P_EXCL_COL] + target_id_colname = script_params[P_TARG_COLID] + target_name_colname = script_params[P_TARG_COLNAME] + separator = script_params[P_CSVSEP] + attach_file = script_params[P_ATTACH] + exclude_empty_value = script_params[P_EXCL_EMPTY] + split_on = script_params[P_SPLIT_CELL] + use_personal_tags = script_params[P_OWN_TAG] + create_new_tags = script_params[P_ALLOW_NEWTAG] + import_tags = script_params[P_IMPORT_TAGS] file_ann_multiplied = script_params["File_Annotation_multiplied"] ntarget_processed = 0 @@ -163,14 +180,14 @@ def main_loop(conn, script_params): if file_ann_id is not None: file_ann = conn.getObject("Annotation", oid=file_ann_id) assert file_ann is not None, f"Annotation {file_ann_id} not found" - assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, "The \ - provided annotation ID must reference a FileAnnotation, \ - not a {file_ann.OMERO_TYPE}" + assert file_ann.OMERO_TYPE == omero.model.FileAnnotationI, \ + ("The provided annotation ID must reference a " + + f"FileAnnotation, not a {file_ann.OMERO_TYPE}") else: file_ann = get_original_file(source_object) original_file = file_ann.getFile()._obj - rows, header, namespaces = read_csv(conn, original_file, separator) + rows, header, namespaces = read_csv(conn, original_file, separator, import_tags) if namespace is not None: namespaces = [namespace] * len(header) elif len(namespaces) == 0: @@ -189,8 +206,9 @@ def main_loop(conn, script_params): cols_to_ignore = [header.index(el) for el in to_exclude if el in header] - assert (idx_id != -1) or (idx_name != -1), "Neither \ - the column for the objects' name or the objects' index were found" + assert (idx_id != -1) or (idx_name != -1), \ + ("Neither the column for the objects' name or" + + " the objects' index were found") use_id = idx_id != -1 # use the obj_idx column if exist if not use_id: @@ -200,17 +218,17 @@ def main_loop(conn, script_params): duplicates = {name for name in name_list if name_list.count(name) > 1} print("duplicates:", duplicates) - assert not len(duplicates) > 0, ("The .csv contains" + - f"duplicates {duplicates} which" + - " makes it impossible" + - " to correctly allocate the" + - " annotations.") + assert not len(duplicates) > 0, \ + (f"The .csv contains duplicates {duplicates} which makes" + + " it impossible to correctly allocate the annotations.") + # Identify target-objects by name fail if two have identical names target_d = dict() for target_obj in target_obj_l: name = get_obj_name(target_obj) - assert name not in target_d.keys(), f"Target objects \ - identified by name have at least one duplicate: {name}" + assert name not in target_d.keys(), \ + ("Target objects identified by name have at " + + f"least one duplicate: {name}") target_d[name] = target_obj else: # Setting the dictionnary target_id:target_obj @@ -309,13 +327,14 @@ def get_original_file(omero_obj): # Get the most recent file file_ann = ann - assert file_ann is not None, f"No .csv FileAnnotation was found on \ - {omero_obj.OMERO_CLASS}:{get_obj_name(omero_obj)}:{omero_obj.getId()}" + assert file_ann is not None, \ + (f"No .csv FileAnnotation was found on {omero_obj.OMERO_CLASS}" + + f":{get_obj_name(omero_obj)}:{omero_obj.getId()}") return file_ann -def read_csv(conn, original_file, delimiter): +def read_csv(conn, original_file, delimiter, import_tags): """ Dedicated function to read the CSV file """ print("Using FileAnnotation", f"{original_file.id.val}:{original_file.name.val}") @@ -359,6 +378,13 @@ def read_csv(conn, original_file, delimiter): header = [el.strip() for el in rows[0]] rows = rows[1:] + if not import_tags: + idx_l = [i for i in range(len(header)) if header[i].lower() != "tag"] + header = [header[i] for i in idx_l] + namespaces = [namespaces[i] for i in idx_l] + for j in range(len(rows)): + rows[j] = [rows[j][i] for i in idx_l] + print(f"Header: {header}\n") return rows, header, namespaces @@ -524,7 +550,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, " is not permitted" ) if not tag_exist: - tag_o = omero.gateway.TagAnnotationWrapper(conn) + tag_o = TagAnnotationWrapper(conn) tag_o.setValue(tagname) tag_o.save() tagid_d[tag_o.id] = tag_o @@ -534,7 +560,8 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, else: # has tagset tagset_exist = tagset in tagset_d.keys() - tag_exist = tagset_exist and (tagname in tagtree_d[tagset].keys()) + tag_exist = (tagset_exist + and (tagname in tagtree_d[tagset].keys())) assert (tag_exist or create_new_tags), ( f"Tag '{tagname}' " + f"in TagSet '{tagset}'" + @@ -543,13 +570,13 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, " is not permitted" ) if not tag_exist: - tag_o = omero.gateway.TagAnnotationWrapper(conn) + tag_o = TagAnnotationWrapper(conn) tag_o.setValue(tagname) tag_o.save() tagid_d[tag_o.id] = tag_o tag_d[tagname] = tag_o.id if not tagset_exist: - tagset_o = omero.gateway.TagAnnotationWrapper(conn) + tagset_o = TagAnnotationWrapper(conn) tagset_o.setValue(tagset) tagset_o.setNs(NSINSIGHTTAGSET) tagset_o.save() @@ -619,48 +646,74 @@ def run_script(): """, # Tabs are needed to add line breaks in the HTML scripts.String( - "Data_Type", optional=False, grouping="1", + P_DTYPE, optional=False, grouping="1", description="Parent-data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( - "IDs", optional=False, grouping="1.1", + P_IDS, optional=False, grouping="1.1", description="List of parent-data IDs containing" + " the objects to annotate.").ofType(rlong(0)), scripts.String( - "Target Data_Type", optional=False, grouping="1.2", + P_TARG_DTYPE, optional=False, grouping="1.2", description="The data type which will be annotated. " + "Entries in the .csv correspond to these objects.", values=target_types, default=""), scripts.String( - "File_Annotation", optional=True, grouping="1.3", + P_FILE_ANN, optional=True, grouping="1.3", description="If no file is provided, list of file IDs " + "containing metadata to populate (must match length" + " of 'IDs'). If neither, searches the most recently " + "attached CSV file on each parent object."), scripts.String( - "Namespace (blank for default or from csv)", + P_NAMESPACE, optional=True, grouping="1.4", description="Namespace given to the created key-value " + "pairs annotations. Default is the client" + "namespace, meaning editable in OMERO.web"), scripts.Bool( - "Advanced parameters", optional=True, grouping="2", default=False, + P_IMPORT_TAGS, optional=True, grouping="2", default=True, + description="Untick this to prevent importing tags specified " + + "in the CSV."), + + scripts.Bool( + P_OWN_TAG, grouping="2.1", default=False, + description="Determines if tags of other users in the group" + + " can be used on objects.\n Using only personal tags might " + + "lead to multiple tags with the same name in one OMERO-group."), + + scripts.Bool( + P_ALLOW_NEWTAG, grouping="2.2", default=False, + description="Creates new tags and tagsets if the ones" + + " specified in the .csv do not exist."), + + scripts.Bool( + "Other parameters", optional=True, grouping="3", default=True, description="Ticking or unticking this has no effect"), + scripts.Bool( + P_EXCL_EMPTY, grouping="3.1", default=True, + description="Exclude a key-value if the value is empty."), + scripts.String( - "Separator", optional=False, grouping="2.1", + P_CSVSEP, optional=True, grouping="3.2", description="The separator used in the .csv file. 'guess' will " + "attempt to detetect automatically which of " + ",;\\t is used.", values=separators, default="guess"), + scripts.String( + P_SPLIT_CELL, optional=True, grouping="3.3", + default="", + description="Split cells according to this into multiple " + + "values for a given key."), + scripts.List( - "Columns to exclude", optional=False, grouping="2.2", + P_EXCL_COL, optional=True, grouping="3.4", default=",,", description="List of columns in the .csv file to exclude " + "from the key-value pair import. " + @@ -669,14 +722,14 @@ def run_script(): "to the six container types.").ofType(rstring("")), scripts.String( - "Target ID colname", optional=False, grouping="2.3", + P_TARG_COLID, optional=False, grouping="3.5", default="OBJECT_ID", description="The column name in the .csv containing the id" + " of the objects to annotate. " + "Matches in exclude parameter."), scripts.String( - "Target name colname", optional=False, grouping="2.4", + P_TARG_COLNAME, optional=False, grouping="3.6", default="OBJECT_NAME", description="The column name in the .csv containing the name of " + "the objects to annotate (used if no column " + @@ -684,31 +737,10 @@ def run_script(): " in exclude parameter."), scripts.Bool( - "Exclude empty values", grouping="2.5", default=False, - description="Exclude a key-value if the value is empty."), - - scripts.Bool( - "Attach csv to parents", grouping="2.6", default=False, - description="Attach the given CSV to the parent-data objects" + + P_ATTACH, grouping="3.7", default=False, + description="Attach the given CSV to the selected objects" + "when not already attached to it."), - scripts.String( - "Split value on", optional=True, grouping="2.7", - default="", - description="Split values according to that input to " + - "create key duplicates."), - - scripts.Bool( - "Use only personal tags", grouping="2.8", default=False, - description="Determines if tags of other users in the group" + - " can be used on objects.\n Using only personal tags might " + - "lead to multiple tags with the same name in one OMERO-group."), - - scripts.Bool( - "Create new tags", grouping="2.9", default=False, - description="Creates new tags and tagsets if the ones" + - " specified in the .csv do not exist."), - authors=["Christian Evenhuis", "Tom Boissonnet"], institutions=["MIF UTS", "CAi HHU"], contact="https://forum.image.sc/tag/omero" @@ -716,16 +748,6 @@ def run_script(): try: params = parameters_parsing(client) - print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", "File_Annotation", - "Namespace (blank for default or from csv)", - "Separator", "Columns to exclude", "Target ID colname", - "Target name colname", "Exclude empty values", - "Attach csv to parents", "Split value on", - "Use only personal tags", "Create new tags"] - for k in keys: - print(f"\t- {k}: {params[k]}") - print("\n####################################\n") # wrap client to use the Blitz Gateway conn = BlitzGateway(client_obj=client) @@ -746,73 +768,84 @@ def run_script(): def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters - params["File_Annotation"] = None - params["Namespace (blank for default or from csv)"] = None - params["Split value on"] = "" + params[P_FILE_ANN] = None + params[P_NAMESPACE] = None + params[P_SPLIT_CELL] = "" for key in client.getInputKeys(): if client.getInput(key): params[key] = client.getInput(key, unwrap=True) - if params["Target Data_Type"] == "": - params["Target Data_Type"] = params["Data_Type"] - elif " " in params["Target Data_Type"]: + if params[P_TARG_DTYPE] == "": + params[P_TARG_DTYPE] = params[P_DTYPE] + elif " " in params[P_TARG_DTYPE]: # Getting rid of the trailing '---' added for the UI - params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + params[P_TARG_DTYPE] = params[P_TARG_DTYPE].split(" ")[1] - assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + assert params[P_TARG_DTYPE] in ALLOWED_PARAM[params[P_DTYPE]], \ (f"{params['Target Data_Type']} is not a valid target for " + f"{params['Data_Type']}.") - if params["Data_Type"] == "Tag": - params["Data_Type"] = "TagAnnotation" - assert None not in params["File_Annotation"], "File annotation \ - ID must be given when using Tag as source" + if params[P_DTYPE] == "Tag": + assert None not in params[P_FILE_ANN], \ + "File annotation ID must be given when using Tag as source" - if ((params["File_Annotation"]) is not None - and ("," in params["File_Annotation"])): + if ((params[P_FILE_ANN]) is not None + and ("," in params[P_FILE_ANN])): # List of ID provided, have to do the split - params["File_Annotation"] = params["File_Annotation"].split(",") + params[P_FILE_ANN] = params[P_FILE_ANN].split(",") else: - params["File_Annotation"] = [int(params["File_Annotation"])] - if len(params["File_Annotation"]) == 1: + params[P_FILE_ANN] = [int(params[P_FILE_ANN])] + if len(params[P_FILE_ANN]) == 1: # Poulate the parameter with None or same ID for all source - params["File_Annotation"] *= len(params["IDs"]) + params[P_FILE_ANN] *= len(params[P_IDS]) params["File_Annotation_multiplied"] = True - params["File_Annotation"] = list(map(int, params["File_Annotation"])) + params[P_FILE_ANN] = list(map(int, params[P_FILE_ANN])) - assert len(params["File_Annotation"]) == len(params["IDs"]), "Number of \ - Source IDs and FileAnnotation IDs must match" + assert len(params[P_FILE_ANN]) == len(params[P_IDS]), \ + "Number of IDs and FileAnnotation IDs must match" # Replacing the placeholders and with values from params to_exclude = list(map(lambda x: x.replace('', - params["Target ID colname"]), - params["Columns to exclude"])) + params[P_TARG_COLID]), + params[P_EXCL_COL])) to_exclude = list(map(lambda x: x.replace('', - params["Target name colname"]), + params[P_TARG_COLNAME]), to_exclude)) if "" in to_exclude: to_exclude.remove("") to_exclude.extend(["PROJECT", "DATASET", "SCREEN", "PLATE", "RUN", "WELL"]) - params["Columns to exclude"] = to_exclude - - if params["Separator"] == "guess": - params["Separator"] = None - elif params["Separator"] == "TAB": - params["Separator"] = "\t" + params[P_EXCL_COL] = to_exclude - if params["Data_Type"] == "Run": - params["Data_Type"] = "Acquisition" - if params["Target Data_Type"] == "Run": - params["Target Data_Type"] = "PlateAcquisition" - - assert (params["Separator"] is None - or params["Separator"] not in params["Split value on"]), ( + assert (params[P_CSVSEP] is None + or params[P_CSVSEP] not in params[P_SPLIT_CELL]), ( "Cannot split cells with a character used as CSV separator" ) + print("Input parameters:") + keys = [P_DTYPE, P_IDS, P_TARG_DTYPE, P_FILE_ANN, + P_NAMESPACE, P_CSVSEP, P_EXCL_COL, P_TARG_COLID, + P_TARG_COLNAME, P_EXCL_EMPTY, P_ATTACH, P_SPLIT_CELL, + P_IMPORT_TAGS, P_OWN_TAG, P_ALLOW_NEWTAG] + + for k in keys: + print(f"\t- {k}: {params[k]}") + print("\n####################################\n") + + if params[P_CSVSEP] == "guess": + params[P_CSVSEP] = None + elif params[P_CSVSEP] == "TAB": + params[P_CSVSEP] = "\t" + + if params[P_DTYPE] == "Tag": + params[P_DTYPE] = "TagAnnotation" + if params[P_DTYPE] == "Run": + params[P_DTYPE] = "Acquisition" + if params[P_TARG_DTYPE] == "Run": + params[P_TARG_DTYPE] = "PlateAcquisition" + return params diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index cb835659a..d37caa3df 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -52,8 +52,12 @@ "Screen", "Plate", "Well", "Run"] } -AGREEMENT = ("I understand what I am doing and that this will result " + - "in a batch deletion of key-value pairs from the server") +P_DTYPE = "Data_Type" # Do not change +P_IDS = "IDs" # Do not change +P_TARG_DTYPE = "Target Data_Type" +P_NAMESPACE = "Namespace (blank for default)" +P_AGREEMENT = ("I understand what I am doing and that this will result " + + "in a batch deletion of key-value pairs from the server") def get_children_recursive(source_object, target_type): @@ -113,10 +117,10 @@ def main_loop(conn, script_params): For every object: - Find annotations in the namespace and remove """ - source_type = script_params["Data_Type"] - target_type = script_params["Target Data_Type"] - source_ids = script_params["IDs"] - namespace_l = script_params["Namespace (blank for default)"] + source_type = script_params[P_DTYPE] + target_type = script_params[P_TARG_DTYPE] + source_ids = script_params[P_IDS] + namespace_l = script_params[P_NAMESPACE] nsuccess = 0 ntotal = 0 @@ -204,22 +208,22 @@ def run_script(): """, # Tabs are needed to add line breaks in the HTML scripts.String( - "Data_Type", optional=False, grouping="1", + P_DTYPE, optional=False, grouping="1", description="Parent-data type of the objects to annotate.", values=source_types, default="Dataset"), scripts.List( - "IDs", optional=False, grouping="1.1", + P_IDS, optional=False, grouping="1.1", description="List of parent-data IDs containing the objects " + "to delete annotation from.").ofType(rlong(0)), scripts.String( - "Target Data_Type", optional=True, grouping="1.2", + P_TARG_DTYPE, optional=True, grouping="1.2", description="Choose the object type to delete annotation from.", values=target_types, default=""), scripts.List( - "Namespace (blank for default)", optional=True, + P_NAMESPACE, optional=True, grouping="1.3", description="Annotation with these namespace will " + "be deleted. Default is the client" + @@ -227,7 +231,7 @@ def run_script(): "OMERO.web").ofType(rstring("")), scripts.Bool( - AGREEMENT, optional=False, grouping="2", + P_AGREEMENT, optional=False, grouping="2", description="Make sure that you understood the scope of " + "what will be deleted."), @@ -239,8 +243,7 @@ def run_script(): try: params = parameters_parsing(client) print("Input parameters:") - keys = ["Data_Type", "IDs", "Target Data_Type", - "Namespace (blank for default)"] + keys = [P_DTYPE, P_IDS, P_TARG_DTYPE, P_NAMESPACE] for k in keys: print(f"\t- {k}: {params[k]}") print("\n####################################\n") @@ -262,39 +265,39 @@ def run_script(): def parameters_parsing(client): params = {} # Param dict with defaults for optional parameters - params["Namespace (blank for default)"] = [NSCLIENTMAPANNOTATION] + params[P_NAMESPACE] = [NSCLIENTMAPANNOTATION] for key in client.getInputKeys(): if client.getInput(key): # unwrap rtypes to String, Integer etc params[key] = client.getInput(key, unwrap=True) - assert params[AGREEMENT], "Please tick the box to confirm that you " +\ - "understood the risks." + assert params[P_AGREEMENT], "Please tick the box to confirm that you " +\ + "understood the risks." - if params["Target Data_Type"] == "": - params["Target Data_Type"] = params["Data_Type"] - elif " " in params["Target Data_Type"]: + if params[P_TARG_DTYPE] == "": + params[P_TARG_DTYPE] = params[P_DTYPE] + elif " " in params[P_TARG_DTYPE]: # Getting rid of the trailing '---' added for the UI - params["Target Data_Type"] = params["Target Data_Type"].split(" ")[1] + params[P_TARG_DTYPE] = params[P_TARG_DTYPE].split(" ")[1] - assert params["Target Data_Type"] in ALLOWED_PARAM[params["Data_Type"]], \ + assert params[P_TARG_DTYPE] in ALLOWED_PARAM[params[P_DTYPE]], \ (f"{params['Target Data_Type']} is not a valid target for " + f"{params['Data_Type']}.") - if params["Data_Type"] == "Tag": - params["Data_Type"] = "TagAnnotation" + if params[P_DTYPE] == "Tag": + params[P_DTYPE] = "TagAnnotation" - if params["Data_Type"] == "Run": - params["Data_Type"] = "Acquisition" - if params["Target Data_Type"] == "Run": - params["Target Data_Type"] = "PlateAcquisition" + if params[P_DTYPE] == "Run": + params[P_DTYPE] = "Acquisition" + if params[P_TARG_DTYPE] == "Run": + params[P_TARG_DTYPE] = "PlateAcquisition" # Remove duplicate entries from namespace list - tmp = params["Namespace (blank for default)"] + tmp = params[P_NAMESPACE] if "*" in tmp: tmp = ["*"] - params["Namespace (blank for default)"] = list(set(tmp)) + params[P_NAMESPACE] = list(set(tmp)) return params From 3aa53f20117e6d20d3d1591228cf3d3e25118dd8 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 19:28:40 +0100 Subject: [PATCH 086/127] changed run to acquisition for autofill --- .../Convert_KeyVal_namespace.py | 16 +++++++--------- omero/annotation_scripts/Export_to_csv.py | 16 +++++++--------- omero/annotation_scripts/KeyVal_from_csv.py | 16 +++++++--------- omero/annotation_scripts/Remove_KeyVal.py | 16 +++++++--------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index a8f31d3ce..fe2748674 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -41,12 +41,12 @@ "Project": ["Project", "Dataset", "Image"], "Dataset": ["Dataset", "Image"], "Image": ["Image"], - "Screen": ["Screen", "Plate", "Well", "Run", "Image"], - "Plate": ["Plate", "Well", "Run", "Image"], + "Screen": ["Screen", "Plate", "Well", "Acquisition", "Image"], + "Plate": ["Plate", "Well", "Acquisition", "Image"], "Well": ["Well", "Image"], - "Run": ["Run", "Image"], + "Acquisition": ["Acquisition", "Image"], "Tag": ["Project", "Dataset", "Image", - "Screen", "Plate", "Well", "Run"] + "Screen", "Plate", "Well", "Acquisition"] } P_DTYPE = "Data_Type" # Do not change @@ -196,7 +196,7 @@ def run_script(): source_types = [ rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), - rstring("Run"), rstring("Image"), rstring("Tag"), + rstring("Acquisition"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script @@ -204,7 +204,7 @@ def run_script(): rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("-- Run"), + rstring("-- Well"), rstring("-- Acquisition"), rstring("--- Image") ] @@ -299,9 +299,7 @@ def parameters_parsing(client): if params[P_DTYPE] == "Tag": params[P_DTYPE] = "TagAnnotation" - if params[P_DTYPE] == "Run": - params[P_DTYPE] = "Acquisition" - if params[P_TARG_DTYPE] == "Run": + if params[P_TARG_DTYPE] == "Acquisition": params[P_TARG_DTYPE] = "PlateAcquisition" # Remove duplicate entries from namespace list diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index ad55fa909..47231c4f6 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -47,12 +47,12 @@ "Project": ["Project", "Dataset", "Image"], "Dataset": ["Dataset", "Image"], "Image": ["Image"], - "Screen": ["Screen", "Plate", "Well", "Run", "Image"], - "Plate": ["Plate", "Well", "Run", "Image"], + "Screen": ["Screen", "Plate", "Well", "Acquisition", "Image"], + "Plate": ["Plate", "Well", "Acquisition", "Image"], "Well": ["Well", "Image"], - "Run": ["Run", "Image"], + "Acquisition": ["Acquisition", "Image"], "Tag": ["Project", "Dataset", "Image", - "Screen", "Plate", "Well", "Run"] + "Screen", "Plate", "Well", "Acquisition"] } P_DTYPE = "Data_Type" # Do not change @@ -413,7 +413,7 @@ def run_script(): source_types = [ rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), - rstring("Run"), rstring("Image"), rstring("Tag"), + rstring("Acquisition"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script @@ -421,7 +421,7 @@ def run_script(): rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("-- Run"), + rstring("-- Well"), rstring("-- Acquisition"), rstring("--- Image") ] @@ -553,9 +553,7 @@ def parameters_parsing(client): if params[P_DTYPE] == "Tag": params[P_DTYPE] = "TagAnnotation" - if params[P_DTYPE] == "Run": - params[P_DTYPE] = "Acquisition" - if params[P_TARG_DTYPE] == "Run": + if params[P_TARG_DTYPE] == "Acquisition": params[P_TARG_DTYPE] = "PlateAcquisition" print("Input parameters:") diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index a63f038c6..470fd92c1 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -48,12 +48,12 @@ "Project": ["Project", "Dataset", "Image"], "Dataset": ["Dataset", "Image"], "Image": ["Image"], - "Screen": ["Screen", "Plate", "Well", "Run", "Image"], - "Plate": ["Plate", "Well", "Run", "Image"], + "Screen": ["Screen", "Plate", "Well", "Acquisition", "Image"], + "Plate": ["Plate", "Well", "Acquisition", "Image"], "Well": ["Well", "Image"], - "Run": ["Run", "Image"], + "Acquisition": ["Acquisition", "Image"], "Tag": ["Project", "Dataset", "Image", - "Screen", "Plate", "Well", "Run"] + "Screen", "Plate", "Well", "Acquisition"] } P_DTYPE = "Data_Type" # Do not change @@ -619,7 +619,7 @@ def run_script(): source_types = [ rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), - rstring("Run"), rstring("Image"), rstring("Tag"), + rstring("Acquisition"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script @@ -627,7 +627,7 @@ def run_script(): rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("-- Run"), + rstring("-- Well"), rstring("-- Acquisition"), rstring("--- Image") ] @@ -841,9 +841,7 @@ def parameters_parsing(client): if params[P_DTYPE] == "Tag": params[P_DTYPE] = "TagAnnotation" - if params[P_DTYPE] == "Run": - params[P_DTYPE] = "Acquisition" - if params[P_TARG_DTYPE] == "Run": + if params[P_TARG_DTYPE] == "Acquisition": params[P_TARG_DTYPE] = "PlateAcquisition" return params diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index d37caa3df..fbcd74aad 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -44,12 +44,12 @@ "Project": ["Project", "Dataset", "Image"], "Dataset": ["Dataset", "Image"], "Image": ["Image"], - "Screen": ["Screen", "Plate", "Well", "Run", "Image"], - "Plate": ["Plate", "Well", "Run", "Image"], + "Screen": ["Screen", "Plate", "Well", "Acquisition", "Image"], + "Plate": ["Plate", "Well", "Acquisition", "Image"], "Well": ["Well", "Image"], - "Run": ["Run", "Image"], + "Acquisition": ["Acquisition", "Image"], "Tag": ["Project", "Dataset", "Image", - "Screen", "Plate", "Well", "Run"] + "Screen", "Plate", "Well", "Acquisition"] } P_DTYPE = "Data_Type" # Do not change @@ -179,7 +179,7 @@ def run_script(): source_types = [ rstring("Project"), rstring("Dataset"), rstring("Image"), rstring("Screen"), rstring("Plate"), rstring("Well"), - rstring("Run"), rstring("Image"), rstring("Tag"), + rstring("Acquisition"), rstring("Image"), rstring("Tag"), ] # Duplicate Image for UI, but not a problem for script @@ -187,7 +187,7 @@ def run_script(): rstring(""), rstring("Project"), rstring("- Dataset"), rstring("-- Image"), rstring("Screen"), rstring("- Plate"), - rstring("-- Well"), rstring("-- Run"), + rstring("-- Well"), rstring("-- Acquisition"), rstring("--- Image") ] @@ -288,9 +288,7 @@ def parameters_parsing(client): if params[P_DTYPE] == "Tag": params[P_DTYPE] = "TagAnnotation" - if params[P_DTYPE] == "Run": - params[P_DTYPE] = "Acquisition" - if params[P_TARG_DTYPE] == "Run": + if params[P_TARG_DTYPE] == "Acquisition": params[P_TARG_DTYPE] = "PlateAcquisition" # Remove duplicate entries from namespace list From 16407e6bd58b07cf434939630b27ea2531b86c73 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 19:51:38 +0100 Subject: [PATCH 087/127] script names change --- .../Convert_KeyVal_namespace.py | 4 ++-- omero/annotation_scripts/Export_to_csv.py | 16 ++++++++-------- omero/annotation_scripts/KeyVal_from_csv.py | 10 +++++----- omero/annotation_scripts/Remove_KeyVal.py | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index fe2748674..337f3eac2 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -4,7 +4,7 @@ Convert the namespace of objects key-value pairs. ----------------------------------------------------------------------------- - Copyright (C) 2018 + Copyright (C) 2024 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or @@ -209,7 +209,7 @@ def run_script(): ] client = scripts.client( - 'Convert_KV_namespace', + 'Convert Key-Value pairs namespace', """ This script converts the namespace of key-value pair annotations. \t diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 47231c4f6..e778b82f3 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -6,7 +6,7 @@ and creates a csv file attached to dataset ----------------------------------------------------------------------------- - Copyright (C) 2018 + Copyright (C) 2018 - 2024 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or @@ -430,7 +430,7 @@ def run_script(): # Good practice to put url here to give users more guidance on how to run # your script. client = scripts.client( - 'Export_KV_to_csv.py', + 'Export to CSV', """ This script exports for the selected objects their name, IDs and associated key-value pairs. @@ -471,25 +471,25 @@ def run_script(): scripts.String( P_CSVSEP, optional=False, grouping="2.1", - description="Choose the .csv separator.", + description="Choose the csv separator.", values=separators, default="TAB"), scripts.Bool( P_INCL_PARENT, optional=False, grouping="2.2", - description="Whether to include or not the name of the parent(s)" + - " objects as columns in the .csv.", default=False), + description="Check to include or not the name of the parent(s)" + + " objects as columns in the csv", default=False), scripts.Bool( P_INCL_NS, optional=False, grouping="2.3", - description="Whether to include the annotation namespaces" + - " in the .csv.", default=False), + description="Check to include the annotation namespaces" + + " in the csv file.", default=False), scripts.Bool( P_INCL_TAG, optional=False, grouping="2.4", - description="Whether to include tags in the .csv.", + description="Check to include tags in the csv file.", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/KeyVal_from_csv.py index 470fd92c1..695d59c13 100644 --- a/omero/annotation_scripts/KeyVal_from_csv.py +++ b/omero/annotation_scripts/KeyVal_from_csv.py @@ -1,11 +1,11 @@ # coding=utf-8 """ - KeyVal_from_csv.py + Import_from_csv.py Adds key-value pairs to a target object on OMERO from a CSV file. ----------------------------------------------------------------------------- - Copyright (C) 2018 + Copyright (C) 2018 - 2024 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or @@ -634,7 +634,7 @@ def run_script(): separators = ["guess", ";", ",", "TAB"] client = scripts.client( - 'Import_KV_from_csv', + 'Import from CSV', """ Reads a .csv file to annotate the given objects with key-value pairs. \t @@ -741,8 +741,8 @@ def run_script(): description="Attach the given CSV to the selected objects" + "when not already attached to it."), - authors=["Christian Evenhuis", "Tom Boissonnet"], - institutions=["MIF UTS", "CAi HHU"], + authors=["Christian Evenhuis", "Tom Boissonnet", "Jens Wendt"], + institutions=["MIF UTS", "CAi HHU", "MiN WWU"], contact="https://forum.image.sc/tag/omero" ) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index fbcd74aad..4f2f08332 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ - MIF/Key_Value_remove.py" + Remove_KeyVal.py" Remove all key-value pairs associated with a namespace from objects on OMERO. ----------------------------------------------------------------------------- - Copyright (C) 2018 - 2022 + Copyright (C) 2018 - 2024 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or @@ -195,7 +195,7 @@ def run_script(): # Good practice to put url here to give users more guidance on how to run # your script. client = scripts.client( - 'Remove_KV.py', + 'Remove Key-Value pairs', """ This script deletes for the selected objects the key-value pairs associated to the given namespace. From 51cd20408bc7f88c4ff4244fea7a88f3d5e571d2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 19:52:21 +0100 Subject: [PATCH 088/127] renamed import script --- .../annotation_scripts/{KeyVal_from_csv.py => Import_from_csv.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename omero/annotation_scripts/{KeyVal_from_csv.py => Import_from_csv.py} (100%) diff --git a/omero/annotation_scripts/KeyVal_from_csv.py b/omero/annotation_scripts/Import_from_csv.py similarity index 100% rename from omero/annotation_scripts/KeyVal_from_csv.py rename to omero/annotation_scripts/Import_from_csv.py From 133b7e6c1595375f9dff0bfbbc4e44b06ea42cb4 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 2 Feb 2024 22:06:16 +0100 Subject: [PATCH 089/127] param name adjustements --- omero/annotation_scripts/Export_to_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index e778b82f3..3dfce4064 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -60,7 +60,7 @@ P_TARG_DTYPE = "Target Data_Type" P_NAMESPACE = "Namespace (blank for default)" P_CSVSEP = "CSV separator" -P_INCL_PARENT = "Include column(s) of parents name" +P_INCL_PARENT = "Include parent container names" P_INCL_NS = "Include namespace" P_INCL_TAG = "Include tags" From e7b52630b1330bc1017c4517589ab3b30c038bd1 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Thu, 15 Feb 2024 17:04:30 +0100 Subject: [PATCH 090/127] minor error message change --- omero/annotation_scripts/Import_from_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 695d59c13..ffd3ea543 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -544,7 +544,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, if not has_tagset: tag_exist = tagname in tag_d.keys() assert (tag_exist or create_new_tags), ( - f"Tag '{tagname}'" + + f"The tag '{tagname}'" + " does not exist while" + " creation of new tags" + " is not permitted" @@ -563,7 +563,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, tag_exist = (tagset_exist and (tagname in tagtree_d[tagset].keys())) assert (tag_exist or create_new_tags), ( - f"Tag '{tagname}' " + + f"The tag '{tagname}' " + f"in TagSet '{tagset}'" + " does not exist while" + " creation of new tags" + From 7d7762aee03a4be5f531d22d612f8ce3fdec710e Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 16 Feb 2024 22:15:55 +0100 Subject: [PATCH 091/127] Option to create&merge KV into a new one --- .../Convert_KeyVal_namespace.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 337f3eac2..2961ca7eb 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -54,6 +54,7 @@ P_TARG_DTYPE = "Target Data_Type" P_OLD_NS = "Old Namespace (blank for default)" P_NEW_NS = "New Namespace (blank for default)" +P_MERGE = "Create new and merge" def get_children_recursive(source_object, target_type): @@ -120,6 +121,7 @@ def main_loop(conn, script_params): source_ids = script_params[P_IDS] old_namespace = script_params[P_OLD_NS] new_namespace = script_params[P_NEW_NS] + merge = script_params[P_MERGE] ntarget_processed = 0 ntarget_updated = 0 @@ -131,11 +133,21 @@ def main_loop(conn, script_params): for target_obj in target_iterator(conn, source_object, target_type, is_tag): ntarget_processed += 1 - keyval_l, ann_l = get_existing_map_annotions(target_obj, - old_namespace) + keyval_l, ann_l = get_existing_map_annotations(target_obj, + old_namespace) if len(keyval_l) > 0: - annotate_object(conn, target_obj, keyval_l, new_namespace) - remove_map_annotations(conn, target_obj, ann_l) + if merge: + annotate_object(conn, target_obj, keyval_l, + new_namespace) + remove_map_annotations(conn, target_obj, ann_l) + else: + for ann in ann_l: + try: + ann.setNs(new_namespace) + ann.save() + except Exception: + print(f"Failed to edit {ann}") + continue ntarget_updated += 1 if result_obj is None: result_obj = target_obj @@ -148,7 +160,7 @@ def main_loop(conn, script_params): return message, result_obj -def get_existing_map_annotions(obj, namespace_l): +def get_existing_map_annotations(obj, namespace_l): keyval_l, ann_l = [], [] forbidden_deletion = [] for namespace in namespace_l: @@ -247,6 +259,12 @@ def run_script(): grouping="1.5", description="The new namespace for the annotations."), + scripts.Bool( + P_MERGE, optional=False, + grouping="1.6", + description="Check to merge selected key-value pairs" + + " into a single new one", default=True), + authors=["Tom Boissonnet"], institutions=["CAi HHU"], contact="https://forum.image.sc/tag/omero" From 6fd5caef7af4a58708ad1d10f0f79adb0259e5c5 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 17 Feb 2024 21:49:41 +0100 Subject: [PATCH 092/127] restricted rights limit case handling --- .../Convert_KeyVal_namespace.py | 8 ++++---- omero/annotation_scripts/Export_to_csv.py | 19 ++++++++++++++----- omero/annotation_scripts/Import_from_csv.py | 10 +++++++--- omero/annotation_scripts/Remove_KeyVal.py | 2 +- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 2961ca7eb..4f7dd06f9 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -139,7 +139,7 @@ def main_loop(conn, script_params): if merge: annotate_object(conn, target_obj, keyval_l, new_namespace) - remove_map_annotations(conn, target_obj, ann_l) + remove_map_annotations(conn, ann_l) else: for ann in ann_l: try: @@ -178,7 +178,7 @@ def get_existing_map_annotations(obj, namespace_l): return keyval_l, ann_l -def remove_map_annotations(conn, obj, ann_l): +def remove_map_annotations(conn, ann_l): mapann_ids = [ann.id for ann in ann_l] if len(mapann_ids) == 0: @@ -188,7 +188,7 @@ def remove_map_annotations(conn, obj, ann_l): conn.deleteObjects("Annotation", mapann_ids) return 1 except Exception: - print("Failed to delete links") + print(f"Failed to delete old annotations {mapann_ids}") return 0 @@ -263,7 +263,7 @@ def run_script(): P_MERGE, optional=False, grouping="1.6", description="Check to merge selected key-value pairs" + - " into a single new one", default=True), + " into a single new one", default=False), authors=["Tom Boissonnet"], institutions=["CAi HHU"], diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 3dfce4064..1332225ae 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -223,8 +223,11 @@ def main_loop(conn, script_params): rows.insert(0, ns_row) file_ann = attach_csv(conn, result_obj, rows, separator, csv_name) - message = ("The csv is attached to " + - f"{result_obj.OMERO_CLASS}:{result_obj.getId()}") + if file_ann is None: + message = "The CSV is printed in output, no file could be attached:" + else: + message = ("The csv is attached to " + + f"{result_obj.OMERO_CLASS}:{result_obj.getId()}") return message, file_ann, result_obj @@ -380,6 +383,11 @@ def natural_sort(names): def attach_csv(conn, obj_, rows, separator, csv_name): + if not obj_.canAnnotate() and WEBCLIENT_URL == "": + for row in rows: + print(f"{separator.join(row)}") + return None + # create the tmp directory tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) @@ -392,9 +400,10 @@ def attach_csv(conn, obj_, rows, separator, csv_name): file_ann = conn.createFileAnnfromLocalFile( tmp_file, origFilePathAndName=csv_name, ns='KeyVal_export') - obj_.linkAnnotation(file_ann) - print(f"{file_ann} linked to {obj_}") + if obj_.canAnnotate(): + obj_.linkAnnotation(file_ann) + print(f"{file_ann} linked to {obj_}") # remove the tmp file os.remove(tmp_file) @@ -504,8 +513,8 @@ def run_script(): message, fileann, res_obj = main_loop(conn, params) client.setOutput("Message", rstring(message)) - href = f"{WEBCLIENT_URL}/download_original_file/{fileann.getId()}" if res_obj is not None and fileann is not None: + href = f"{WEBCLIENT_URL}/download_original_file/{fileann.getId()}" if WEBCLIENT_URL != "": url = omero.rtypes.wrap({ "type": "URL", diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 695d59c13..67d1d7606 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -129,8 +129,11 @@ def target_iterator(conn, source_object, target_type, is_tag): print(f"Iterating objects from {source_object}:") for target_obj in target_obj_l: - print(f"\t- {target_obj}") - yield target_obj + if target_obj.canAnnotate(): + print(f"\t- {target_obj}") + yield target_obj + else: + print(f"\t- Annotate {target_obj} is not permitted, skipping") print() @@ -187,7 +190,8 @@ def main_loop(conn, script_params): file_ann = get_original_file(source_object) original_file = file_ann.getFile()._obj - rows, header, namespaces = read_csv(conn, original_file, separator, import_tags) + rows, header, namespaces = read_csv(conn, original_file, + separator, import_tags) if namespace is not None: namespaces = [namespace] * len(header) elif len(namespaces) == 0: diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 4f2f08332..9719c41f4 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -150,7 +150,7 @@ def remove_map_annotations(conn, obj, namespace_l): p = {} if namespace == "*" else {"ns": namespace} for ann in obj.listAnnotations(**p): if isinstance(ann, omero.gateway.MapAnnotationWrapper): - if ann.canEdit(): # If not, skipping it + if ann.canDelete(): # If not, skipping it mapann_ids.append(ann.id) else: forbidden_deletion.append(ann.id) From cecb66b492251dbb42070de8b8b0a0fe554d8a3a Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Mar 2024 12:52:20 +0100 Subject: [PATCH 093/127] explicit handling of csv encoding with utf-8 --- omero/annotation_scripts/Export_to_csv.py | 4 +-- omero/annotation_scripts/Import_from_csv.py | 34 +++++++++++---------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 1332225ae..94b6f82f1 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -189,7 +189,7 @@ def main_loop(conn, script_params): result_obj = target_obj print("\n------------------------------------\n") - csv_name = "{}_keyval.csv".format(get_obj_name(source_object)) + csv_name = f"{get_obj_name(source_object)}_{target_type}-KeyValue.csv" if include_namespace and "*" in namespace_l: # Assign entries of * namespace @@ -391,7 +391,7 @@ def attach_csv(conn, obj_, rows, separator, csv_name): # create the tmp directory tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) - tfile = os.fdopen(fd, 'w') + tfile = os.fdopen(fd, 'w', encoding="utf-8") for row in rows: tfile.write(f"{separator.join(row)}\n") tfile.close() diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 4c7f12bd5..b39ea80fb 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -344,23 +344,25 @@ def read_csv(conn, original_file, delimiter, import_tags): f"{original_file.id.val}:{original_file.name.val}") provider = DownloadingOriginalFileProvider(conn) # read the csv - temp_file = provider.get_original_file_data(original_file) # Needs omero-py 5.9.1 or later - with open(temp_file.name, 'rt', encoding='utf-8-sig') as file_handle: - if delimiter is None: - try: # Detecting csv delimiter from the first line - delimiter = csv.Sniffer().sniff( - file_handle.readline(), ",;\t").delimiter - print(f"Using delimiter {delimiter}", - "after reading one line") - except Exception: - # Send the error back to the UI - assert False, ("Failed to sniff CSV delimiter, " + - "please specify the separator") - - # reset to start and read whole file... - file_handle.seek(0) - rows = list(csv.reader(file_handle, delimiter=delimiter)) + + try: + temp_file = provider.get_original_file_data(original_file) + with open(temp_file.name, mode="rt", encoding='utf-8') as f: + csv_content = f.readlines() + except UnicodeDecodeError as e: + assert False, ("Error while reading the csv, convert your " + + "file to utf-8 encoding" + + str(e)) + + if delimiter is None: + try: + # Sniffing on a maximum of four lines + delimiter = csv.Sniffer().sniff("\n".join(csv_content[:4]), + ",;\t").delimiter + except Exception as e: + assert False, ("Failed to sniff CSV delimiter: " + str(e)) + rows = list(csv.reader(csv_content, delimiter=delimiter)) rowlen = len(rows[0]) error_msg = ( From ebbcf1b6e18634e6696a5880ba8962e5aa8c133b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Mar 2024 13:43:45 +0100 Subject: [PATCH 094/127] capitalize namespace output --- omero/annotation_scripts/Export_to_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 94b6f82f1..7032d04b7 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -375,7 +375,7 @@ def natural_sort(names): for j in range(len(obj_ancestry_l[i])): header_row.insert(j, obj_ancestry_l[i][j][0].upper()) ns_row.insert(j, "") - ns_row[0] = "namespace" + ns_row[0] = "NAMESPACE" print(f"\tColumn names: {header_row}", "\n") From 0d36407bf23c7e56fcccee83c865b6b00f05243f Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Fri, 1 Mar 2024 16:13:29 +0100 Subject: [PATCH 095/127] Added script version --- omero/annotation_scripts/Remove_KeyVal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 9719c41f4..2dece6f29 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -238,6 +238,7 @@ def run_script(): authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", + version="2.0.0", ) try: From bbf79edcf9f3da2b07485eebc92d7debfaa8d6ac Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Mar 2024 16:14:31 +0100 Subject: [PATCH 096/127] added version 2.0.0 to the scripts --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 3 ++- omero/annotation_scripts/Export_to_csv.py | 1 + omero/annotation_scripts/Import_from_csv.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 4f7dd06f9..8873048e9 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -267,7 +267,8 @@ def run_script(): authors=["Tom Boissonnet"], institutions=["CAi HHU"], - contact="https://forum.image.sc/tag/omero" + contact="https://forum.image.sc/tag/omero", + version="2.0.0", ) try: diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 7032d04b7..dcea13064 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -504,6 +504,7 @@ def run_script(): authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], institutions=["University of Technology Sydney", "CAi HHU"], contact="https://forum.image.sc/tag/omero", + version="2.0.0", ) try: params = parameters_parsing(client) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index b39ea80fb..fdf35acba 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -749,7 +749,8 @@ def run_script(): authors=["Christian Evenhuis", "Tom Boissonnet", "Jens Wendt"], institutions=["MIF UTS", "CAi HHU", "MiN WWU"], - contact="https://forum.image.sc/tag/omero" + contact="https://forum.image.sc/tag/omero", + version="2.0.0", ) try: From dae6a360545f8277a627acf10a964ab193a093fd Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Mar 2024 16:37:49 +0100 Subject: [PATCH 097/127] replace doc link to the current readthedoc --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 3 +-- omero/annotation_scripts/Export_to_csv.py | 3 +-- omero/annotation_scripts/Import_from_csv.py | 3 +-- omero/annotation_scripts/Remove_KeyVal.py | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 8873048e9..8249d1b73 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -226,8 +226,7 @@ def run_script(): This script converts the namespace of key-value pair annotations. \t Check the guide for more information on parameters and errors: - TODO link to omero-guides - https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#converting-the-key-value-pairs-namespace + https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation """, # Tabs are needed to add line breaks in the HTML diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index dcea13064..f8716c24f 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -445,8 +445,7 @@ def run_script(): key-value pairs. \t Check the guide for more information on parameters and errors: - TODO link to omero-guides - https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#exporting-key-value-pairs + https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation """, # Tabs are needed to add line breaks in the HTML diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index fdf35acba..3ea28ebf8 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -645,8 +645,7 @@ def run_script(): Reads a .csv file to annotate the given objects with key-value pairs. \t Check the guide for more information on parameters and errors: - TODO link to omero-guides - https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#importing-key-value-pairs + https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation """, # Tabs are needed to add line breaks in the HTML diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 2dece6f29..89d09eda1 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -201,8 +201,7 @@ def run_script(): associated to the given namespace. \t Check the guide for more information on parameters and errors: - TODO link to omero-guides - https://github.com/German-BioImaging/guide-KVpairs-scripts/blob/master/docs/gettingstarted.rst#deleting-key-value-pairs + https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html \t Default namespace: openmicroscopy.org/omero/client/mapAnnotation """, # Tabs are needed to add line breaks in the HTML From 5d6df32095ade496b8abad871984ae0fd1e9828f Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Mar 2024 16:45:35 +0100 Subject: [PATCH 098/127] fix line too long --- omero/annotation_scripts/Import_from_csv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 3ea28ebf8..ff81fbcff 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -586,7 +586,10 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, tagset_o.setValue(tagset) tagset_o.setNs(NSINSIGHTTAGSET) tagset_o.save() - tagid_d[tagset_o.id] = conn.getObject("TagAnnotation", tagset_o.id) + tagid_d[tagset_o.id] = conn.getObject( + "TagAnnotation", + tagset_o.id + ) tagset_d[tagset] = tagset_o.id print(f"Created new TagSet {tagset}:{tagset_o.id}") # else: From 144f6917bd56c7e42d46518ba922addd15f43b6d Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Mar 2024 16:56:26 +0100 Subject: [PATCH 099/127] updated README --- omero/annotation_scripts/README.md | 37 +++++++++++++----------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/omero/annotation_scripts/README.md b/omero/annotation_scripts/README.md index b50f9c597..e3d0197c7 100644 --- a/omero/annotation_scripts/README.md +++ b/omero/annotation_scripts/README.md @@ -7,43 +7,38 @@ This is the central repository for community contributed scripts to [omero-web]( These scripts, in combination with the [omero.forms](https://pypi.org/project/omero-forms), support the bulk annotation workflow described in [this blog post](https://mpievolbio-scicomp.pages.gwdg.de/blog/post/2020-09-03_omerobulkannotation/). +For the new scripts version of 2024, you can follow this guide: +https://guide-kvpairs-scripts.readthedocs.io/en/latest/walkthrough.html + Content ------- -This repository provides five scripts: -* `01-KeyVal_from_Description.py`: Parses a Dataset/Project/Screen description and converts - key:value pairs into map annotations in the same container. -* `01-KeyVal_to_csv.py`: Converts a dataset map annotation into a table with one - record for every image in the dataset. Columns are named according to map -annotation keys. The first column contains the image filename (or id???) -* `03-KeyVal_from_csv.py`: Parses a given csv table attachment and converts each - record into a map annotation for the image identified via the entry in the -first column (filename or image id). -* `04-Remove_KeyVal.py`: Removes all map annotations from a dataset and all - contained images. -* `05-KeyVal_from_Filename.py`: Creates image map annotation by tokenizing the - filename. +This repository provides four scripts: +* `Import_from_csv.py`: Read a csv file and converts each row into a map annotation +for the identified object (image, dataset, project, run, well, plate, screen). +* `Export_to_csv.py`: Exports the map annotations of objects into a csv file. +* `Remove_KeyVal.py`: Removes the key-value pairs of an object associated with +a given namespace. +* `Convert_KeyVal_namespace.py`: Converts the namespace of map annotations. Installation --------------- The scripts must be placed in the `OMERODIR/lib/scripts/omero` directory of your omero installation, preferrentially in a seperatate subdirectory, e.g. `Bulk -Annotation/`. +Annotation/`. -`OMERODIR` -refers to the root directory of you omero server. If you followed the -installation procedures, you should have the `$OMERODIR` environment variable set. -Logged in omero admins can also use the "Upload scripts" button in the *Gears* -menu. +Follow [these instruction](https://omero.readthedocs.io/en/stable/developers/scripts/index.html#downloading-and-installing-scripts) to install/update the scripts, -After installation, the scripts will be accessible in omero web by clicking the *Gears* -icon in the menu bar. +You should also configure the Export_to_csv script so that it returns the csv file as a direct download link: +https://guide-kvpairs-scripts.readthedocs.io/en/latest/setup.html#configuring-the-export-script History -------- This repository started as a fork of [evehuis/omero-user-scripts](). Ownership was transferred to @CFGrote after merging a pull request that fixed a number of bugs and ported the original code from python2.7 to python3.x +In 2023, the scripts were reworked by Tom Boissonnet and Jens Wendt to extend the annotation to all OMERO objects, and to include a new script to convert namespaces of map annotations. + Contributions ---------------- From 0c8d969fce2c400b73f951d7cd7f778b01128c2d Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 16 Mar 2024 19:01:08 +0100 Subject: [PATCH 100/127] set optional true for bool param --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 9 +++------ omero/annotation_scripts/Export_to_csv.py | 8 ++++---- omero/annotation_scripts/Import_from_csv.py | 12 ++++++------ omero/annotation_scripts/Remove_KeyVal.py | 4 ++-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 8249d1b73..4c6c21aee 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -248,19 +248,16 @@ def run_script(): values=target_types, default=""), scripts.List( - P_OLD_NS, optional=True, - grouping="1.4", + P_OLD_NS, optional=True, grouping="1.4", description="The namespace(s) of the annotations to " + "group and change.").ofType(rstring("")), scripts.String( - P_NEW_NS, optional=True, - grouping="1.5", + P_NEW_NS, optional=True, grouping="1.5", description="The new namespace for the annotations."), scripts.Bool( - P_MERGE, optional=False, - grouping="1.6", + P_MERGE, optional=True, grouping="1.6", description="Check to merge selected key-value pairs" + " into a single new one", default=False), diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index f8716c24f..9024b751f 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -461,7 +461,7 @@ def run_script(): "to delete annotation from.").ofType(rlong(0)), scripts.String( - P_TARG_DTYPE, optional=True, grouping="1.2", + P_TARG_DTYPE, optional=False, grouping="1.2", description="Choose the object type to delete annotation from.", values=target_types, default=""), @@ -483,19 +483,19 @@ def run_script(): values=separators, default="TAB"), scripts.Bool( - P_INCL_PARENT, optional=False, + P_INCL_PARENT, optional=True, grouping="2.2", description="Check to include or not the name of the parent(s)" + " objects as columns in the csv", default=False), scripts.Bool( - P_INCL_NS, optional=False, + P_INCL_NS, optional=True, grouping="2.3", description="Check to include the annotation namespaces" + " in the csv file.", default=False), scripts.Bool( - P_INCL_TAG, optional=False, + P_INCL_TAG, optional=True, grouping="2.4", description="Check to include tags in the csv file.", default=False), diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index ff81fbcff..4fc461b53 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -689,13 +689,13 @@ def run_script(): "in the CSV."), scripts.Bool( - P_OWN_TAG, grouping="2.1", default=False, + P_OWN_TAG, optional=True, grouping="2.1", default=False, description="Determines if tags of other users in the group" + " can be used on objects.\n Using only personal tags might " + "lead to multiple tags with the same name in one OMERO-group."), scripts.Bool( - P_ALLOW_NEWTAG, grouping="2.2", default=False, + P_ALLOW_NEWTAG, optional=True, grouping="2.2", default=False, description="Creates new tags and tagsets if the ones" + " specified in the .csv do not exist."), @@ -704,7 +704,7 @@ def run_script(): description="Ticking or unticking this has no effect"), scripts.Bool( - P_EXCL_EMPTY, grouping="3.1", default=True, + P_EXCL_EMPTY, optional=True, grouping="3.1", default=True, description="Exclude a key-value if the value is empty."), scripts.String( @@ -730,14 +730,14 @@ def run_script(): "to the six container types.").ofType(rstring("")), scripts.String( - P_TARG_COLID, optional=False, grouping="3.5", + P_TARG_COLID, optional=True, grouping="3.5", default="OBJECT_ID", description="The column name in the .csv containing the id" + " of the objects to annotate. " + "Matches in exclude parameter."), scripts.String( - P_TARG_COLNAME, optional=False, grouping="3.6", + P_TARG_COLNAME, optional=True, grouping="3.6", default="OBJECT_NAME", description="The column name in the .csv containing the name of " + "the objects to annotate (used if no column " + @@ -745,7 +745,7 @@ def run_script(): " in exclude parameter."), scripts.Bool( - P_ATTACH, grouping="3.7", default=False, + P_ATTACH, optional=True, grouping="3.7", default=False, description="Attach the given CSV to the selected objects" + "when not already attached to it."), diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index 89d09eda1..bd355d018 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -217,7 +217,7 @@ def run_script(): "to delete annotation from.").ofType(rlong(0)), scripts.String( - P_TARG_DTYPE, optional=True, grouping="1.2", + P_TARG_DTYPE, optional=False, grouping="1.2", description="Choose the object type to delete annotation from.", values=target_types, default=""), @@ -230,7 +230,7 @@ def run_script(): "OMERO.web").ofType(rstring("")), scripts.Bool( - P_AGREEMENT, optional=False, grouping="2", + P_AGREEMENT, optional=True, grouping="2", description="Make sure that you understood the scope of " + "what will be deleted."), From 936915ec4a38aae0fedd7f1632bd4624ff74b694 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 16 Mar 2024 19:16:09 +0100 Subject: [PATCH 101/127] added tag in the script description --- omero/annotation_scripts/Export_to_csv.py | 2 +- omero/annotation_scripts/Import_from_csv.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 9024b751f..a67cbd846 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -442,7 +442,7 @@ def run_script(): 'Export to CSV', """ This script exports for the selected objects their name, IDs and associated - key-value pairs. + tags and key-value pairs. \t Check the guide for more information on parameters and errors: https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 4fc461b53..50708b361 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -645,7 +645,8 @@ def run_script(): client = scripts.client( 'Import from CSV', """ - Reads a .csv file to annotate the given objects with key-value pairs. + Reads a .csv file to annotate the given objects with tags and + key-value pairs. \t Check the guide for more information on parameters and errors: https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html From 582a46a20c42195338b20e1f30fa12ab14ca737e Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 16 Mar 2024 19:16:31 +0100 Subject: [PATCH 102/127] changed main loop description --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 4c6c21aee..b28224d2d 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -113,8 +113,10 @@ def main_loop(conn, script_params): """ For every object: - Find annotations in the namespace - - Remove annotations with old namespace - - Create annotations with new namespace + - If merge: + - Remove annotations with old namespace + - Create a merged annotation with new namespace + - Else change the namespace of the annotation (default) """ source_type = script_params[P_DTYPE] target_type = script_params[P_TARG_DTYPE] From 6d51b6bf5182b58640e392d917cc6f3db2de5a07 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Wed, 20 Mar 2024 08:43:22 +0100 Subject: [PATCH 103/127] exclude tag namespace error fix --- omero/annotation_scripts/Import_from_csv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 50708b361..9a78cf831 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -385,9 +385,11 @@ def read_csv(conn, original_file, delimiter, import_tags): rows = rows[1:] if not import_tags: + # We filter out the tag columns idx_l = [i for i in range(len(header)) if header[i].lower() != "tag"] header = [header[i] for i in idx_l] - namespaces = [namespaces[i] for i in idx_l] + if len(namespaces) > 0: + namespaces = [namespaces[i] for i in idx_l] for j in range(len(rows)): rows[j] = [rows[j][i] for i in idx_l] From a7234a9b822d9a7ff848e41976ce7832e2193918 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Wed, 20 Mar 2024 08:51:06 +0100 Subject: [PATCH 104/127] Remove problematic assertion on inner separator --- omero/annotation_scripts/Import_from_csv.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 9a78cf831..21c677652 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -831,11 +831,6 @@ def parameters_parsing(client): params[P_EXCL_COL] = to_exclude - assert (params[P_CSVSEP] is None - or params[P_CSVSEP] not in params[P_SPLIT_CELL]), ( - "Cannot split cells with a character used as CSV separator" - ) - print("Input parameters:") keys = [P_DTYPE, P_IDS, P_TARG_DTYPE, P_FILE_ANN, P_NAMESPACE, P_CSVSEP, P_EXCL_COL, P_TARG_COLID, From 4e05a96f6fd05fb24e1ebaa4307b2fc605abaaad Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Fri, 22 Mar 2024 22:46:31 +0100 Subject: [PATCH 105/127] first test annotation --- .../Convert_KeyVal_namespace.py | 6 +- omero/annotation_scripts/Import_from_csv.py | 12 +- test/integration/test_annotation_scripts.py | 130 ++++++++++++++++++ 3 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 test/integration/test_annotation_scripts.py diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index b28224d2d..b793eee1e 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -156,8 +156,10 @@ def main_loop(conn, script_params): else: print("\tNo MapAnnotation found with that namespace\n") print("\n------------------------------------\n") - message = f"Updated kv pairs to \ - {ntarget_updated}/{ntarget_processed} {target_type}" + message = ( + "Updated kv pairs to " + + f"{ntarget_updated}/{ntarget_processed} {target_type}" + ) return message, result_obj diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 21c677652..2831cd67b 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -301,8 +301,10 @@ def main_loop(conn, script_params): link_file_ann(conn, source_type, source_object, file_ann) print("\n------------------------------------\n") - message = f"Added Annotations to \ - {ntarget_updated}/{ntarget_processed} {target_type}(s)" + message = ( + "Added Annotations to " + + f"{ntarget_updated}/{ntarget_processed} {target_type}(s)" + ) if file_ann_multiplied and len(missing_names) > 0: # subtract the processed names/ids from the @@ -313,8 +315,10 @@ def main_loop(conn, script_params): total_missing_names = len(missing_names) if total_missing_names > 0: - message += f". {total_missing_names} {target_type}(s) not found \ - (using {'ID' if use_id else 'name'} to identify them)." + message += ( + f". {total_missing_names} {target_type}(s) not found " + f"(using {'ID' if use_id else 'name'} to identify them)." + ) return message, result_obj diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py new file mode 100644 index 000000000..ed955096e --- /dev/null +++ b/test/integration/test_annotation_scripts.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Copyright (C) 2016 University of Dundee & Open Microscopy Environment. +# All rights reserved. Use is subject to license terms supplied in LICENSE.txt +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + Integration test for annotation scripts. +""" + +from __future__ import print_function +import omero +from omero.gateway import BlitzGateway +import omero.scripts +import pytest +from script import ScriptTest +from script import run_script +from omero.cmd import Delete2 +from omero.rtypes import wrap, rstring, rlist, rbool +from omero.util.temp_files import create_path + +import_script = "/omero/annotation_scripts/Import_from_csv.py" +export_script = "/omero/annotation_scripts/Export_to_csv.py" +delete_script = "/omero/annotation_scripts/Remove_KeyVal.py" +convert_script = "/omero/util_scripts/Convert_KeyVal_namespace.py" + +DEFAULT_IMPORT_ARGS = { + "CSV separator": rstring("guess"), + "Columns to exclude": rlist([ + rstring(""), + rstring(""), + rstring("") + ]), + "Target ID colname": rstring("OBJECT_ID"), + "Target name colname": rstring("OBJECT_NAME"), + "Exclude empty values": rbool(False), + "Attach CSV file": rbool(False), + "Split values on": rstring(""), + "Import tags": rbool(False), + "Only use personal tags": rbool(False), + "Allow tag creation": rbool(False), +} + +CLIENT, USER = None, None + +def link_file_plate(client, plate, cvs_file): + conn = BlitzGateway(client_obj=client) + fa = conn.createFileAnnfromLocalFile(cvs_file, mimetype="text/csv") + assert fa is not None + assert fa.id > 0 + link = omero.model.PlateAnnotationLinkI() + link.setParent(plate) + link.setChild(omero.model.FileAnnotationI(fa.id, False)) + client.getSession().getUpdateService().saveAndReturnObject(link) + return fa + + +class TestAnnotationScripts(ScriptTest): + + def test_import_name(self): + sid = super(TestAnnotationScripts, self).get_script(import_script) + assert sid > 0 + + client, user = self.new_client_and_user() + + # self.create_test_image(name="testImage", session=self.client.getSession()) + + plates = self.import_plates(client, plate_cols=3, plate_rows=1) + plate = plates[0] + + cvs_file = create_path("test_kvp_name", ".csv") + # create a file annotation + with open(cvs_file.abspath(), 'w') as f: + f.write("OBJECT_NAME,key1\n") + f.write("A1,value1\n") + f.write("A2,value2\n") + f.write("A3,value3\n") + + fa = link_file_plate(client, plate, cvs_file) + + # run the script + args = DEFAULT_IMPORT_ARGS.copy() + args["Data_Type"] = rstring("Plate") + args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["Target Data_Type"] = rstring("-- Well") + args["File_Annotation"] = rstring(str(fa.id)) + + # Making sure tags have no influence here + for import_tag in [True, False]: + for pers_tag in [True, False]: + for tag_creation in [True, False]: + args["Import tags"] = rbool(import_tag) + args["Only use personal tags"] = rbool(pers_tag) + args["Allow tag creation"] = rbool(tag_creation) + message = run_script(client, sid, args, "Message") + assert message._val == "Added Annotations to 3/3 Well(s)" + + + def test_import_id(self): + pass + + def test_import_innersplit(self): + pass + + def test_import_separator(self): + pass + + def test_import_attachfile(self): + pass + + def test_import_colname(self): + pass + + + From e727912f8d56a79a948bdcabbd1e018da3859dc2 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Sat, 6 Apr 2024 14:42:01 +0200 Subject: [PATCH 106/127] tests import and fix --- omero/annotation_scripts/Import_from_csv.py | 3 +- test/integration/test_annotation_scripts.py | 297 ++++++++++++++++++-- 2 files changed, 270 insertions(+), 30 deletions(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 2831cd67b..9550ce944 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -410,6 +410,7 @@ def annotate_object(conn, obj, row, header, namespaces, kv_list = [] tag_id_l = [] for ns, h, r in zip(namespaces, header, row): + r = r.strip() if ns == curr_ns and (len(r) > 0 or not exclude_empty_value): if h.lower() == "tag": if r == "": @@ -529,7 +530,7 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, values = values.split(split_on) for val in values: - val.strip() + val = val.strip() # matching a regex to the value re_match = regx_tag.match(val) if re_match is None: diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index ed955096e..6f6e272a3 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -25,7 +25,9 @@ from __future__ import print_function import omero -from omero.gateway import BlitzGateway +from omero.gateway import BlitzGateway, TagAnnotationWrapper, MapAnnotationWrapper +from omero.model import AnnotationAnnotationLinkI, ImageAnnotationLinkI +from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET import omero.scripts import pytest from script import ScriptTest @@ -37,7 +39,7 @@ import_script = "/omero/annotation_scripts/Import_from_csv.py" export_script = "/omero/annotation_scripts/Export_to_csv.py" delete_script = "/omero/annotation_scripts/Remove_KeyVal.py" -convert_script = "/omero/util_scripts/Convert_KeyVal_namespace.py" +convert_script = "/omero/annotation_scripts/Convert_KeyVal_namespace.py" DEFAULT_IMPORT_ARGS = { "CSV separator": rstring("guess"), @@ -50,13 +52,11 @@ "Target name colname": rstring("OBJECT_NAME"), "Exclude empty values": rbool(False), "Attach CSV file": rbool(False), - "Split values on": rstring(""), "Import tags": rbool(False), "Only use personal tags": rbool(False), "Allow tag creation": rbool(False), } -CLIENT, USER = None, None def link_file_plate(client, plate, cvs_file): conn = BlitzGateway(client_obj=client) @@ -72,7 +72,13 @@ def link_file_plate(client, plate, cvs_file): class TestAnnotationScripts(ScriptTest): - def test_import_name(self): + @pytest.mark.parametrize('import_tag', [True, False]) + @pytest.mark.parametrize('tag_creation', [True, False]) + @pytest.mark.parametrize('ns', [ + "", NSCLIENTMAPANNOTATION, "otherNS" + ]) + @pytest.mark.parametrize('ns_in_csv', [True, False]) + def test_import(self, import_tag, tag_creation, ns, ns_in_csv): sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -80,16 +86,193 @@ def test_import_name(self): # self.create_test_image(name="testImage", session=self.client.getSession()) - plates = self.import_plates(client, plate_cols=3, plate_rows=1) + n_well = 3 + plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) + plate = plates[0] + + cvs_file = create_path("test_kvp_name", ".csv") + # create a file annotation + + ns_str = "NAMESPACE" + "".join([f";{ns}" for i in range(3)]) + with open(cvs_file.abspath(), 'w') as f: + if ns_in_csv: + f.write(ns_str + "\n") + f.write("OBJECT_NAME; key_1; key_2; key_3\n") + f.write("A1; val_A; val_B; val_C" + "\n") + f.write("A2; val_D; val_E; val_F" + "\n") + f.write("A3; val_G; val_H; val_I" + "\n") + + fa = link_file_plate(client, plate, cvs_file) + + # run the script + args = DEFAULT_IMPORT_ARGS.copy() + args["Data_Type"] = rstring("Plate") + args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["Target Data_Type"] = rstring("-- Well") + args["File_Annotation"] = rstring(str(fa.id)) + args["Import tags"] = rbool(import_tag) + args["Allow tag creation"] = rbool(tag_creation) + if not ns_in_csv and ns != "": + args["Namespace (blank for default or from csv)"] = rstring(ns) + + message = run_script(client, sid, args, "Message") + + conn = BlitzGateway(client_obj=client) + assert message._val == f"Added Annotations to {n_well}/{n_well} Well(s)" + plate_o = conn.getObject("Plate", plate.id.val) + list_well = list(plate_o.listChildren()) + list_well = sorted(list_well, key=lambda w: w.getWellPos()) + + well_a1, well_a2, well_a3 = list_well + + assert well_a1.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a2.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a3.getAnnotationCounts()["MapAnnotation"] == 1 + + if ns == "": + ns = NSCLIENTMAPANNOTATION + + value = list(well_a1.listAnnotations(ns=ns))[0].getValue() + assert len(value) == 3 + assert value[0] == ("key_1", "val_A") + assert value[1] == ("key_2", "val_B") + assert value[2] == ("key_3", "val_C") + + value = list(well_a2.listAnnotations(ns=ns))[0].getValue() + assert len(value) == 3 + assert value[0] == ("key_1", "val_D") + assert value[1] == ("key_2", "val_E") + assert value[2] == ("key_3", "val_F") + + value = list(well_a3.listAnnotations(ns=ns))[0].getValue() + assert len(value) == 3 + assert value[0] == ("key_1", "val_G") + assert value[1] == ("key_2", "val_H") + assert value[2] == ("key_3", "val_I") + + + @pytest.mark.parametrize('import_tag', [True, False]) + @pytest.mark.parametrize('tag_creation', [True, False]) + def test_import_tags(self, import_tag, tag_creation): + sid = super(TestAnnotationScripts, self).get_script(import_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + update = conn.getUpdateService() + + if not tag_creation: # Create the tags ahead + tagset_o = TagAnnotationWrapper(conn) + tagset_o.setValue("condition") + tagset_o.setNs(NSINSIGHTTAGSET) + tagset_o.save() + + tag_o = TagAnnotationWrapper(conn) + tag_o.setValue("ctrl") + tag_o.save() + link = AnnotationAnnotationLinkI() + link.parent = tagset_o._obj + link.child = tag_o._obj + update.saveObject(link) + tagset_o = conn.getObject("TagAnnotation", tagset_o.getId()) + + tag_o = TagAnnotationWrapper(conn) + tag_o.setValue("test") + tag_o.save() + link = AnnotationAnnotationLinkI() + link.parent = tagset_o._obj + link.child = tag_o._obj + update.saveObject(link) + + tag_o = TagAnnotationWrapper(conn) + tag_o.setValue("tail") + tag_o.save() + + tag_o = TagAnnotationWrapper(conn) + tag_o.setValue("head") + tag_o.save() + + tag_o = TagAnnotationWrapper(conn) + tag_o.setValue("mouse") + tag_o.save() + + n_well = 3 + plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) + plate = plates[0] + + cvs_file = create_path("test_kvp_name", ".csv") + # create a file annotation + + with open(cvs_file.abspath(), 'w') as f: + f.write("OBJECT_NAME; key_1; tag; tag\n") + f.write("A1; val_A; ctrl[condition]; mouse,tail\n") + f.write("A2; val_B; test[condition],head;\n") + f.write("A3; val_C; ; mouse\n") + + fa = link_file_plate(client, plate, cvs_file) + + # run the script + args = DEFAULT_IMPORT_ARGS.copy() + args["Data_Type"] = rstring("Plate") + args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["Target Data_Type"] = rstring("-- Well") + args["File_Annotation"] = rstring(str(fa.id)) + args["Import tags"] = rbool(import_tag) + args["Allow tag creation"] = rbool(tag_creation) + + message = run_script(client, sid, args, "Message") + + assert message._val == f"Added Annotations to {n_well}/{n_well} Well(s)" + plate_o = conn.getObject("Plate", plate.id.val) + list_well = list(plate_o.listChildren()) + list_well = sorted(list_well, key=lambda w: w.getWellPos()) + well_a1, well_a2, well_a3 = list_well + + if import_tag: + assert well_a1.getAnnotationCounts()["TagAnnotation"] == 3 + assert well_a2.getAnnotationCounts()["TagAnnotation"] == 2 + assert well_a3.getAnnotationCounts()["TagAnnotation"] == 1 + else: + assert well_a1.getAnnotationCounts()["TagAnnotation"] == 0 + assert well_a2.getAnnotationCounts()["TagAnnotation"] == 0 + assert well_a3.getAnnotationCounts()["TagAnnotation"] == 0 + + assert well_a1.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a2.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a3.getAnnotationCounts()["MapAnnotation"] == 1 + + value = list(well_a1.listAnnotations(ns=NSCLIENTMAPANNOTATION))[0].getValue() + assert len(value) == 1 + assert value[0] == ("key_1", "val_A") + + value = list(well_a2.listAnnotations(ns=NSCLIENTMAPANNOTATION))[0].getValue() + assert len(value) == 1 + assert value[0] == ("key_1", "val_B") + + value = list(well_a3.listAnnotations(ns=NSCLIENTMAPANNOTATION))[0].getValue() + assert len(value) == 1 + assert value[0] == ("key_1", "val_C") + + + def test_import_split(self): + sid = super(TestAnnotationScripts, self).get_script(import_script) + assert sid > 0 + + client, user = self.new_client_and_user() + + # self.create_test_image(name="testImage", session=self.client.getSession()) + + n_well = 3 + plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) plate = plates[0] cvs_file = create_path("test_kvp_name", ".csv") # create a file annotation with open(cvs_file.abspath(), 'w') as f: - f.write("OBJECT_NAME,key1\n") - f.write("A1,value1\n") - f.write("A2,value2\n") - f.write("A3,value3\n") + f.write("OBJECT_NAME; key_1; key_2\n") + f.write("A1; val_A,val_B; val_C\n") + f.write("A2; val_D,val_E,val_F;\n") + f.write("A3; ; val_G,val_H\n") fa = link_file_plate(client, plate, cvs_file) @@ -99,32 +282,88 @@ def test_import_name(self): args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) args["Target Data_Type"] = rstring("-- Well") args["File_Annotation"] = rstring(str(fa.id)) + args["Split values on"] = rstring(",") + args["Exclude empty values"] = rbool(False) + + message = run_script(client, sid, args, "Message") + conn = BlitzGateway(client_obj=client) + assert message._val == f"Added Annotations to {n_well}/{n_well} Well(s)" + plate_o = conn.getObject("Plate", plate.id.val) + list_well = list(plate_o.listChildren()) + list_well = sorted(list_well, key=lambda w: w.getWellPos()) + well_a1, well_a2, well_a3 = list_well - # Making sure tags have no influence here - for import_tag in [True, False]: - for pers_tag in [True, False]: - for tag_creation in [True, False]: - args["Import tags"] = rbool(import_tag) - args["Only use personal tags"] = rbool(pers_tag) - args["Allow tag creation"] = rbool(tag_creation) - message = run_script(client, sid, args, "Message") - assert message._val == "Added Annotations to 3/3 Well(s)" + assert well_a1.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a2.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a3.getAnnotationCounts()["MapAnnotation"] == 1 + value = list(well_a1.listAnnotations())[0].getValue() + assert len(value) == 3 + assert value[0] == ("key_1", "val_A") + assert value[1] == ("key_1", "val_B") + assert value[2] == ("key_2", "val_C") - def test_import_id(self): - pass + value = list(well_a2.listAnnotations())[0].getValue() + assert len(value) == 4 + assert value[0] == ("key_1", "val_D") + assert value[1] == ("key_1", "val_E") + assert value[2] == ("key_1", "val_F") + assert value[3] == ("key_2", "") - def test_import_innersplit(self): - pass + value = list(well_a3.listAnnotations())[0].getValue() + assert len(value) == 3 + assert value[0] == ("key_1", "") + assert value[1] == ("key_2", "val_G") + assert value[2] == ("key_2", "val_H") + + def test_import_empty(self): + sid = super(TestAnnotationScripts, self).get_script(import_script) + assert sid > 0 + + client, user = self.new_client_and_user() + + # self.create_test_image(name="testImage", session=self.client.getSession()) + + n_well = 3 + plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) + plate = plates[0] + + cvs_file = create_path("test_kvp_name", ".csv") + # create a file annotation + with open(cvs_file.abspath(), 'w') as f: + f.write("OBJECT_NAME; key_1; key_2\n") + f.write("A1; val_A;\n") + f.write("A2; ;\n") + f.write("A3; ; val_B\n") + + fa = link_file_plate(client, plate, cvs_file) + + # run the script + args = DEFAULT_IMPORT_ARGS.copy() + args["Data_Type"] = rstring("Plate") + args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["Target Data_Type"] = rstring("-- Well") + args["File_Annotation"] = rstring(str(fa.id)) + args["Exclude empty values"] = rbool(True) - def test_import_separator(self): - pass + message = run_script(client, sid, args, "Message") + conn = BlitzGateway(client_obj=client) + assert message._val == f"Added Annotations to {n_well-1}/{n_well} Well(s)" + plate_o = conn.getObject("Plate", plate.id.val) + list_well = list(plate_o.listChildren()) + list_well = sorted(list_well, key=lambda w: w.getWellPos()) + well_a1, well_a2, well_a3 = list_well - def test_import_attachfile(self): - pass + assert well_a1.getAnnotationCounts()["MapAnnotation"] == 1 + assert well_a2.getAnnotationCounts()["MapAnnotation"] == 0 + assert well_a3.getAnnotationCounts()["MapAnnotation"] == 1 - def test_import_colname(self): - pass + value = list(well_a1.listAnnotations())[0].getValue() + assert len(value) == 1 + assert value[0] == ("key_1", "val_A") + value = list(well_a3.listAnnotations())[0].getValue() + assert len(value) == 1 + assert value[0] == ("key_2", "val_B") From 8a3978010f0da98ae1f3ff5b41813f67748e095d Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Sat, 6 Apr 2024 19:43:10 +0200 Subject: [PATCH 107/127] test for remove and convert --- test/integration/test_annotation_scripts.py | 154 +++++++++++++------- 1 file changed, 104 insertions(+), 50 deletions(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 6f6e272a3..8531c4e2f 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -26,19 +26,19 @@ from __future__ import print_function import omero from omero.gateway import BlitzGateway, TagAnnotationWrapper, MapAnnotationWrapper -from omero.model import AnnotationAnnotationLinkI, ImageAnnotationLinkI +from omero.model import AnnotationAnnotationLinkI, ImageAnnotationLinkI, MapAnnotationI from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET import omero.scripts import pytest from script import ScriptTest from script import run_script from omero.cmd import Delete2 -from omero.rtypes import wrap, rstring, rlist, rbool +from omero.rtypes import wrap, rstring, rlist, rbool, rlong from omero.util.temp_files import create_path import_script = "/omero/annotation_scripts/Import_from_csv.py" export_script = "/omero/annotation_scripts/Export_to_csv.py" -delete_script = "/omero/annotation_scripts/Remove_KeyVal.py" +remove_script = "/omero/annotation_scripts/Remove_KeyVal.py" convert_script = "/omero/annotation_scripts/Convert_KeyVal_namespace.py" DEFAULT_IMPORT_ARGS = { @@ -84,8 +84,6 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): client, user = self.new_client_and_user() - # self.create_test_image(name="testImage", session=self.client.getSession()) - n_well = 3 plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) plate = plates[0] @@ -115,10 +113,10 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): if not ns_in_csv and ns != "": args["Namespace (blank for default or from csv)"] = rstring(ns) - message = run_script(client, sid, args, "Message") + msg = run_script(client, sid, args, "Message") conn = BlitzGateway(client_obj=client) - assert message._val == f"Added Annotations to {n_well}/{n_well} Well(s)" + assert msg._val == f"Added Annotations to {n_well}/{n_well} Well(s)" plate_o = conn.getObject("Plate", plate.id.val) list_well = list(plate_o.listChildren()) list_well = sorted(list_well, key=lambda w: w.getWellPos()) @@ -152,7 +150,7 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): @pytest.mark.parametrize('import_tag', [True, False]) - @pytest.mark.parametrize('tag_creation', [True, False]) + @pytest.mark.parametrize('tag_creation', [False]) def test_import_tags(self, import_tag, tag_creation): sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -162,40 +160,26 @@ def test_import_tags(self, import_tag, tag_creation): update = conn.getUpdateService() if not tag_creation: # Create the tags ahead - tagset_o = TagAnnotationWrapper(conn) - tagset_o.setValue("condition") - tagset_o.setNs(NSINSIGHTTAGSET) - tagset_o.save() - - tag_o = TagAnnotationWrapper(conn) - tag_o.setValue("ctrl") - tag_o.save() + self.make_tag(name="tail", client=client) + self.make_tag(name="head", client=client) + self.make_tag(name="mouse", client=client) + + tagset = self.make_tag( + name="condition", ns=NSINSIGHTTAGSET, client=client + ) + tag1 = self.make_tag(name="ctrl", client=client) + tag2 = self.make_tag(name="test", client=client) + link = AnnotationAnnotationLinkI() - link.parent = tagset_o._obj - link.child = tag_o._obj + link.setParent(tagset) + link.setChild(tag1) update.saveObject(link) - tagset_o = conn.getObject("TagAnnotation", tagset_o.getId()) - - tag_o = TagAnnotationWrapper(conn) - tag_o.setValue("test") - tag_o.save() + tagset = conn.getObject("TagAnnotation", tagset.id.val)._obj link = AnnotationAnnotationLinkI() - link.parent = tagset_o._obj - link.child = tag_o._obj + link.setParent(tagset) + link.setChild(tag2) update.saveObject(link) - tag_o = TagAnnotationWrapper(conn) - tag_o.setValue("tail") - tag_o.save() - - tag_o = TagAnnotationWrapper(conn) - tag_o.setValue("head") - tag_o.save() - - tag_o = TagAnnotationWrapper(conn) - tag_o.setValue("mouse") - tag_o.save() - n_well = 3 plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) plate = plates[0] @@ -220,9 +204,9 @@ def test_import_tags(self, import_tag, tag_creation): args["Import tags"] = rbool(import_tag) args["Allow tag creation"] = rbool(tag_creation) - message = run_script(client, sid, args, "Message") + msg = run_script(client, sid, args, "Message") - assert message._val == f"Added Annotations to {n_well}/{n_well} Well(s)" + assert msg._val == f"Added Annotations to {n_well}/{n_well} Well(s)" plate_o = conn.getObject("Plate", plate.id.val) list_well = list(plate_o.listChildren()) list_well = sorted(list_well, key=lambda w: w.getWellPos()) @@ -241,15 +225,18 @@ def test_import_tags(self, import_tag, tag_creation): assert well_a2.getAnnotationCounts()["MapAnnotation"] == 1 assert well_a3.getAnnotationCounts()["MapAnnotation"] == 1 - value = list(well_a1.listAnnotations(ns=NSCLIENTMAPANNOTATION))[0].getValue() + annlist = list(well_a1.listAnnotations(ns=NSCLIENTMAPANNOTATION)) + value = annlist[0].getValue() assert len(value) == 1 assert value[0] == ("key_1", "val_A") - value = list(well_a2.listAnnotations(ns=NSCLIENTMAPANNOTATION))[0].getValue() + annlist = list(well_a2.listAnnotations(ns=NSCLIENTMAPANNOTATION)) + value = annlist[0].getValue() assert len(value) == 1 assert value[0] == ("key_1", "val_B") - value = list(well_a3.listAnnotations(ns=NSCLIENTMAPANNOTATION))[0].getValue() + annlist = list(well_a3.listAnnotations(ns=NSCLIENTMAPANNOTATION)) + value = annlist[0].getValue() assert len(value) == 1 assert value[0] == ("key_1", "val_C") @@ -260,8 +247,6 @@ def test_import_split(self): client, user = self.new_client_and_user() - # self.create_test_image(name="testImage", session=self.client.getSession()) - n_well = 3 plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) plate = plates[0] @@ -285,9 +270,9 @@ def test_import_split(self): args["Split values on"] = rstring(",") args["Exclude empty values"] = rbool(False) - message = run_script(client, sid, args, "Message") + msg = run_script(client, sid, args, "Message") conn = BlitzGateway(client_obj=client) - assert message._val == f"Added Annotations to {n_well}/{n_well} Well(s)" + assert msg._val == f"Added Annotations to {n_well}/{n_well} Well(s)" plate_o = conn.getObject("Plate", plate.id.val) list_well = list(plate_o.listChildren()) list_well = sorted(list_well, key=lambda w: w.getWellPos()) @@ -316,14 +301,13 @@ def test_import_split(self): assert value[1] == ("key_2", "val_G") assert value[2] == ("key_2", "val_H") + def test_import_empty(self): sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 client, user = self.new_client_and_user() - # self.create_test_image(name="testImage", session=self.client.getSession()) - n_well = 3 plates = self.import_plates(client, plate_cols=n_well, plate_rows=1) plate = plates[0] @@ -346,9 +330,9 @@ def test_import_empty(self): args["File_Annotation"] = rstring(str(fa.id)) args["Exclude empty values"] = rbool(True) - message = run_script(client, sid, args, "Message") + msg = run_script(client, sid, args, "Message") conn = BlitzGateway(client_obj=client) - assert message._val == f"Added Annotations to {n_well-1}/{n_well} Well(s)" + assert msg._val == f"Added Annotations to {n_well-1}/{n_well} Well(s)" plate_o = conn.getObject("Plate", plate.id.val) list_well = list(plate_o.listChildren()) list_well = sorted(list_well, key=lambda w: w.getWellPos()) @@ -367,3 +351,73 @@ def test_import_empty(self): assert value[0] == ("key_2", "val_B") + def test_convert(self): + sid = super(TestAnnotationScripts, self).get_script(convert_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + image = self.make_image(name="testImage", client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + args = { + "Data_Type": rstring("Image"), + "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "Target Data_Type": rstring(""), + "Old Namespace (blank for default)": rlist([rstring("test")]), + "New Namespace (blank for default)": rstring("new_ns"), + "Create new and merge": rbool(False) + } + + msg = run_script(client, sid, args, "Message") + + assert msg._val == f"Updated kv pairs to 1/1 Image" + + conn = BlitzGateway(client_obj=client) + image_o = conn.getObject("Image", image.id.val) + + value = list(image_o.listAnnotations(ns="new_ns"))[0].getValue() + assert len(value) == 1 + assert value[0] == ("key_1", "val_A") + + + def test_remove(self): + P_AGREEMENT = ( + "I understand what I am doing and that this will result " + + "in a batch deletion of key-value pairs from the server" + ) + + sid = super(TestAnnotationScripts, self).get_script(remove_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + image = self.make_image(name="testImage", client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) + kv.setNs(rstring("test_delete")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + args = { + "Data_Type": rstring("Image"), + "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "Target Data_Type": rstring(""), + "Namespace (blank for default)": rlist([rstring("test_delete")]), + P_AGREEMENT: rbool(True) + } + + msg = run_script(client, sid, args, "Message") + + assert msg._val == f"Key value data deleted from 1 of 1 objects" + + conn = BlitzGateway(client_obj=client) + image_o = conn.getObject("Image", image.id.val) + + assert len(list(image_o.listAnnotations())) == 0 \ No newline at end of file From d8fc5d68996cef493c7f93f2d827c4805e550a1f Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 10 Apr 2024 13:30:11 +0200 Subject: [PATCH 108/127] Revert file encoding to utf-8-sig --- omero/annotation_scripts/Import_from_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 9550ce944..d230216c5 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -352,7 +352,7 @@ def read_csv(conn, original_file, delimiter, import_tags): try: temp_file = provider.get_original_file_data(original_file) - with open(temp_file.name, mode="rt", encoding='utf-8') as f: + with open(temp_file.name, mode="rt", encoding='utf-8-sig') as f: csv_content = f.readlines() except UnicodeDecodeError as e: assert False, ("Error while reading the csv, convert your " + From a25a2985dc8d3bbc746acee74398bd0c5e1a2fb7 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Tue, 16 Apr 2024 14:00:53 +0200 Subject: [PATCH 109/127] flake8 fix --- test/integration/test_annotation_scripts.py | 30 +++++++++------------ 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 8531c4e2f..28ab878d0 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -25,16 +25,17 @@ from __future__ import print_function import omero -from omero.gateway import BlitzGateway, TagAnnotationWrapper, MapAnnotationWrapper -from omero.model import AnnotationAnnotationLinkI, ImageAnnotationLinkI, MapAnnotationI +from omero.gateway import BlitzGateway +from omero.model import AnnotationAnnotationLinkI, MapAnnotationI from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET +from omero.rtypes import rstring, rlist, rbool +from omero.util.temp_files import create_path import omero.scripts + import pytest from script import ScriptTest from script import run_script -from omero.cmd import Delete2 -from omero.rtypes import wrap, rstring, rlist, rbool, rlong -from omero.util.temp_files import create_path + import_script = "/omero/annotation_scripts/Import_from_csv.py" export_script = "/omero/annotation_scripts/Export_to_csv.py" @@ -148,7 +149,6 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): assert value[1] == ("key_2", "val_H") assert value[2] == ("key_3", "val_I") - @pytest.mark.parametrize('import_tag', [True, False]) @pytest.mark.parametrize('tag_creation', [False]) def test_import_tags(self, import_tag, tag_creation): @@ -240,7 +240,6 @@ def test_import_tags(self, import_tag, tag_creation): assert len(value) == 1 assert value[0] == ("key_1", "val_C") - def test_import_split(self): sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -301,7 +300,6 @@ def test_import_split(self): assert value[1] == ("key_2", "val_G") assert value[2] == ("key_2", "val_H") - def test_import_empty(self): sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -350,14 +348,13 @@ def test_import_empty(self): assert len(value) == 1 assert value[0] == ("key_2", "val_B") - def test_convert(self): sid = super(TestAnnotationScripts, self).get_script(convert_script) assert sid > 0 client, user = self.new_client_and_user() conn = BlitzGateway(client_obj=client) - image = self.make_image(name="testImage", client=client) + image = self.make_image(name="testImage", client=client) kv = MapAnnotationI() kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) @@ -376,7 +373,7 @@ def test_convert(self): msg = run_script(client, sid, args, "Message") - assert msg._val == f"Updated kv pairs to 1/1 Image" + assert msg._val == "Updated kv pairs to 1/1 Image" conn = BlitzGateway(client_obj=client) image_o = conn.getObject("Image", image.id.val) @@ -385,9 +382,8 @@ def test_convert(self): assert len(value) == 1 assert value[0] == ("key_1", "val_A") - def test_remove(self): - P_AGREEMENT = ( + agreement = ( "I understand what I am doing and that this will result " + "in a batch deletion of key-value pairs from the server" ) @@ -397,7 +393,7 @@ def test_remove(self): client, user = self.new_client_and_user() conn = BlitzGateway(client_obj=client) - image = self.make_image(name="testImage", client=client) + image = self.make_image(name="testImage", client=client) kv = MapAnnotationI() kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) @@ -410,14 +406,14 @@ def test_remove(self): "IDs": rlist([omero.rtypes.rlong(image.id.val)]), "Target Data_Type": rstring(""), "Namespace (blank for default)": rlist([rstring("test_delete")]), - P_AGREEMENT: rbool(True) + agreement: rbool(True) } msg = run_script(client, sid, args, "Message") - assert msg._val == f"Key value data deleted from 1 of 1 objects" + assert msg._val == "Key value data deleted from 1 of 1 objects" conn = BlitzGateway(client_obj=client) image_o = conn.getObject("Image", image.id.val) - assert len(list(image_o.listAnnotations())) == 0 \ No newline at end of file + assert len(list(image_o.listAnnotations())) == 0 From e77a535ca771bf1517b90c8c532a72647e1c397b Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Tue, 16 Apr 2024 14:59:35 +0200 Subject: [PATCH 110/127] file attachment without option --- omero/annotation_scripts/Import_from_csv.py | 47 ++++++++++----------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index d230216c5..8d0b8bb22 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -66,7 +66,6 @@ P_TARG_COLID = "Target ID colname" P_TARG_COLNAME = "Target name colname" P_EXCL_EMPTY = "Exclude empty values" -P_ATTACH = "Attach CSV file" P_SPLIT_CELL = "Split values on" P_IMPORT_TAGS = "Import tags" P_OWN_TAG = "Only use personal tags" @@ -157,7 +156,6 @@ def main_loop(conn, script_params): target_id_colname = script_params[P_TARG_COLID] target_name_colname = script_params[P_TARG_COLNAME] separator = script_params[P_CSVSEP] - attach_file = script_params[P_ATTACH] exclude_empty_value = script_params[P_EXCL_EMPTY] split_on = script_params[P_SPLIT_CELL] use_personal_tags = script_params[P_OWN_TAG] @@ -180,6 +178,8 @@ def main_loop(conn, script_params): source_objects = conn.getObjects(source_type, source_ids) for source_object, file_ann_id in zip(source_objects, file_ids): ntarget_updated_curr = 0 + + # Find the file from the user input if file_ann_id is not None: file_ann = conn.getObject("Annotation", oid=file_ann_id) assert file_ann is not None, f"Annotation {file_ann_id} not found" @@ -188,8 +188,20 @@ def main_loop(conn, script_params): f"FileAnnotation, not a {file_ann.OMERO_TYPE}") else: file_ann = get_original_file(source_object) - original_file = file_ann.getFile()._obj + # Get the list of things to annotate + is_tag = source_type == "TagAnnotation" + target_obj_l = list(target_iterator(conn, source_object, + target_type, is_tag)) + + # Find the most suitable object to link the file to + if is_tag and len(target_obj_l) > 0: + obj_to_link = target_obj_l[0] + else: + obj_to_link = source_object + link_file_ann(conn, obj_to_link, file_ann) + + original_file = file_ann.getFile()._obj rows, header, namespaces = read_csv(conn, original_file, separator, import_tags) if namespace is not None: @@ -197,10 +209,6 @@ def main_loop(conn, script_params): elif len(namespaces) == 0: namespaces = [NSCLIENTMAPANNOTATION] * len(header) - is_tag = source_type == "TagAnnotation" - target_obj_l = target_iterator(conn, source_object, - target_type, is_tag) - # Index of the column used to identify the targets. Try for IDs first idx_id, idx_name = -1, -1 if target_id_colname in header: @@ -296,9 +304,6 @@ def main_loop(conn, script_params): ntarget_updated += 1 ntarget_updated_curr += 1 - if ntarget_updated_curr > 0 and attach_file: - # Only attaching if this is successful - link_file_ann(conn, source_type, source_object, file_ann) print("\n------------------------------------\n") message = ( @@ -616,18 +621,15 @@ def preprocess_tag_rows(conn, header, rows, tag_d, tagset_d, return res_rows, tag_d, tagset_d, tagtree_d, tagid_d -def link_file_ann(conn, object_type, object_, file_ann): +def link_file_ann(conn, obj_to_link, file_ann): """Link File Annotation to the Object, if not already linked.""" - # Check for existing links - if object_type == "TagAnnotation": - print("CSV file cannot be attached to the parent tag") - return links = list(conn.getAnnotationLinks( - object_type, parent_ids=[object_.getId()], + obj_to_link.OMERO_CLASS, + parent_ids=[obj_to_link.getId()], ann_ids=[file_ann.getId()] - )) + )) if len(links) == 0: - object_.linkAnnotation(file_ann) + obj_to_link.linkAnnotation(file_ann) def run_script(): @@ -752,11 +754,6 @@ def run_script(): "ID is provided or found in the .csv). Matches " + " in exclude parameter."), - scripts.Bool( - P_ATTACH, optional=True, grouping="3.7", default=False, - description="Attach the given CSV to the selected objects" + - "when not already attached to it."), - authors=["Christian Evenhuis", "Tom Boissonnet", "Jens Wendt"], institutions=["MIF UTS", "CAi HHU", "MiN WWU"], contact="https://forum.image.sc/tag/omero", @@ -804,7 +801,7 @@ def parameters_parsing(client): f"{params['Data_Type']}.") if params[P_DTYPE] == "Tag": - assert None not in params[P_FILE_ANN], \ + assert params[P_FILE_ANN] is not None, \ "File annotation ID must be given when using Tag as source" if ((params[P_FILE_ANN]) is not None @@ -839,7 +836,7 @@ def parameters_parsing(client): print("Input parameters:") keys = [P_DTYPE, P_IDS, P_TARG_DTYPE, P_FILE_ANN, P_NAMESPACE, P_CSVSEP, P_EXCL_COL, P_TARG_COLID, - P_TARG_COLNAME, P_EXCL_EMPTY, P_ATTACH, P_SPLIT_CELL, + P_TARG_COLNAME, P_EXCL_EMPTY, P_SPLIT_CELL, P_IMPORT_TAGS, P_OWN_TAG, P_ALLOW_NEWTAG] for k in keys: From 83a3efa422c153f9721acaa298a6cd57c0ef11cd Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Tue, 16 Apr 2024 22:09:51 +0200 Subject: [PATCH 111/127] removed attachFile param from test --- test/integration/test_annotation_scripts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 28ab878d0..668d9af67 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -52,7 +52,6 @@ "Target ID colname": rstring("OBJECT_ID"), "Target name colname": rstring("OBJECT_NAME"), "Exclude empty values": rbool(False), - "Attach CSV file": rbool(False), "Import tags": rbool(False), "Only use personal tags": rbool(False), "Allow tag creation": rbool(False), From a437292380f4a9ebbe04c40104d71cb52cf2dd32 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Tue, 30 Apr 2024 17:03:15 +0200 Subject: [PATCH 112/127] fix tooltip delete mention --- omero/annotation_scripts/Export_to_csv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index a67cbd846..c77993656 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -458,11 +458,11 @@ def run_script(): scripts.List( P_IDS, optional=False, grouping="1.1", description="List of parent data IDs containing the objects " + - "to delete annotation from.").ofType(rlong(0)), + "to export annotation from.").ofType(rlong(0)), scripts.String( P_TARG_DTYPE, optional=False, grouping="1.2", - description="Choose the object type to delete annotation from.", + description="Choose the object type to export annotation from.", values=target_types, default=""), scripts.List( From 051381ab0f301a805d7d1949ea42b06b5d329f37 Mon Sep 17 00:00:00 2001 From: Tom-TBT Date: Sun, 5 May 2024 02:06:31 +0200 Subject: [PATCH 113/127] clarified tooltips --- .../Convert_KeyVal_namespace.py | 20 +++--- omero/annotation_scripts/Export_to_csv.py | 31 +++++---- omero/annotation_scripts/Import_from_csv.py | 67 +++++++++---------- omero/annotation_scripts/Remove_KeyVal.py | 18 +++-- 4 files changed, 64 insertions(+), 72 deletions(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index b793eee1e..90e44feac 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -227,7 +227,7 @@ def run_script(): client = scripts.client( 'Convert Key-Value pairs namespace', """ - This script converts the namespace of key-value pair annotations. + Converts the namespace of key-value pairs. \t Check the guide for more information on parameters and errors: https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html @@ -237,24 +237,24 @@ def run_script(): scripts.String( P_DTYPE, optional=False, grouping="1", - description="Parent-data type of the objects to annotate.", + description="Data type of the parent objects.", values=source_types, default="Dataset"), scripts.List( P_IDS, optional=False, grouping="1.1", - description="List of parent-data IDs containing the objects " + - "to annotate.").ofType(rlong(0)), + description="IDs of the parent objects").ofType(rlong(0)), scripts.String( P_TARG_DTYPE, optional=False, grouping="1.2", - description="The data type for which key-value pair annotations " + - "will be converted.", + description="Data type to process from the selected " + + "parent objects.", values=target_types, default=""), scripts.List( P_OLD_NS, optional=True, grouping="1.4", - description="The namespace(s) of the annotations to " + - "group and change.").ofType(rstring("")), + description="Namespace(s) of the key-value pairs to " + + "process. Client namespace by default, " + + "'*' for all.").ofType(rstring("")), scripts.String( P_NEW_NS, optional=True, grouping="1.5", @@ -262,8 +262,8 @@ def run_script(): scripts.Bool( P_MERGE, optional=True, grouping="1.6", - description="Check to merge selected key-value pairs" + - " into a single new one", default=False), + description="Check to merge selected key-value pairs " + + "into a single new one", default=False), authors=["Tom Boissonnet"], institutions=["CAi HHU"], diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index c77993656..927144590 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -441,8 +441,8 @@ def run_script(): client = scripts.client( 'Export to CSV', """ - This script exports for the selected objects their name, IDs and associated - tags and key-value pairs. + Exports in a CSV the key-value pairs, tags, name and ID + of the selected objects. \t Check the guide for more information on parameters and errors: https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html @@ -452,26 +452,25 @@ def run_script(): scripts.String( P_DTYPE, optional=False, grouping="1", - description="Parent data type of the objects to annotate.", + description="Data type of the parent objects.", values=source_types, default="Dataset"), scripts.List( P_IDS, optional=False, grouping="1.1", - description="List of parent data IDs containing the objects " + - "to export annotation from.").ofType(rlong(0)), + description="IDs of the parent objects").ofType(rlong(0)), scripts.String( P_TARG_DTYPE, optional=False, grouping="1.2", - description="Choose the object type to export annotation from.", + description="Data type to process from the selected " + + "parent objects.", values=target_types, default=""), scripts.List( P_NAMESPACE, optional=True, grouping="1.3", - description="Namespace(s) to include for the export of key-" + - "value pairs annotations. Default is the client" + - "namespace, meaning editable in " + - "OMERO.web").ofType(rstring("")), + description="Namespace(s) of the key-value pairs " + + "to export. Client namespace by default, " + + "'*' for all.").ofType(rstring("")), scripts.Bool( "Other parameters", optional=True, grouping="2", default=True, @@ -479,25 +478,25 @@ def run_script(): scripts.String( P_CSVSEP, optional=False, grouping="2.1", - description="Choose the csv separator.", + description="Choose the CSV separator.", values=separators, default="TAB"), scripts.Bool( P_INCL_PARENT, optional=True, grouping="2.2", - description="Check to include or not the name of the parent(s)" + - " objects as columns in the csv", default=False), + description="Check to include columns for the parent " + + "containers names", default=False), scripts.Bool( P_INCL_NS, optional=True, grouping="2.3", - description="Check to include the annotation namespaces" + - " in the csv file.", default=False), + description="Check to include the namespaces " + + "of the key-value pairs in the CSV.", default=False), scripts.Bool( P_INCL_TAG, optional=True, grouping="2.4", - description="Check to include tags in the csv file.", + description="Check to include tags in the CSV file.", default=False), authors=["Christian Evenhuis", "MIF", "Tom Boissonnet"], diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 8d0b8bb22..57312590e 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -654,8 +654,7 @@ def run_script(): client = scripts.client( 'Import from CSV', """ - Reads a .csv file to annotate the given objects with tags and - key-value pairs. + Import key-value pairs and tags from a CSV file. \t Check the guide for more information on parameters and errors: https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html @@ -665,49 +664,47 @@ def run_script(): scripts.String( P_DTYPE, optional=False, grouping="1", - description="Parent-data type of the objects to annotate.", + description="Data type of the parent objects.", values=source_types, default="Dataset"), scripts.List( P_IDS, optional=False, grouping="1.1", - description="List of parent-data IDs containing" + - " the objects to annotate.").ofType(rlong(0)), + description="IDs of the parent objects").ofType(rlong(0)), scripts.String( P_TARG_DTYPE, optional=False, grouping="1.2", - description="The data type which will be annotated. " + - "Entries in the .csv correspond to these objects.", + description="Data type to process from the selected " + + "parent objects.", values=target_types, default=""), scripts.String( P_FILE_ANN, optional=True, grouping="1.3", description="If no file is provided, list of file IDs " + - "containing metadata to populate (must match length" + - " of 'IDs'). If neither, searches the most recently " + - "attached CSV file on each parent object."), + "containing metadata to populate (one per ID). " + + "Otherwise, takes the most recent CSV " + + "on each parent object."), scripts.String( P_NAMESPACE, optional=True, grouping="1.4", - description="Namespace given to the created key-value " + - "pairs annotations. Default is the client" + - "namespace, meaning editable in OMERO.web"), + description="Namespace assigned to the key-value pairs. " + + "Default is the client " + + "namespace (editable in OMERO.web)."), scripts.Bool( P_IMPORT_TAGS, optional=True, grouping="2", default=True, - description="Untick this to prevent importing tags specified " + - "in the CSV."), + description="Check this box to allow the import of tags."), scripts.Bool( P_OWN_TAG, optional=True, grouping="2.1", default=False, - description="Determines if tags of other users in the group" + - " can be used on objects.\n Using only personal tags might " + - "lead to multiple tags with the same name in one OMERO-group."), + description="Restrict the usage of tags to the ones owned " + + "by the user. If checked, tags owned by others will not be " + + "considered for the creation of new tags."), scripts.Bool( P_ALLOW_NEWTAG, optional=True, grouping="2.2", default=False, description="Creates new tags and tagsets if the ones" + - " specified in the .csv do not exist."), + " specified in the CSV do not exist."), scripts.Bool( "Other parameters", optional=True, grouping="3", default=True, @@ -715,44 +712,42 @@ def run_script(): scripts.Bool( P_EXCL_EMPTY, optional=True, grouping="3.1", default=True, - description="Exclude a key-value if the value is empty."), + description="Skip the keys with empty values."), scripts.String( P_CSVSEP, optional=True, grouping="3.2", - description="The separator used in the .csv file. 'guess' will " + + description="Separator used in the CSV file. 'guess' will " + "attempt to detetect automatically which of " + - ",;\\t is used.", + ",;\\t to use.", values=separators, default="guess"), scripts.String( P_SPLIT_CELL, optional=True, grouping="3.3", default="", - description="Split cells according to this into multiple " + - "values for a given key."), + description="Separator used to split cells into multiple " + + "key-value pairs."), scripts.List( P_EXCL_COL, optional=True, grouping="3.4", default=",,", - description="List of columns in the .csv file to exclude " + - "from the key-value pair import. " + - " and correspond to the two " + - "following parameters. corresponds " + - "to the six container types.").ofType(rstring("")), + description="Columns to exclude from the key-value pairs. " + + " and correspond to the column name " + + "specified by the next two parameters. " + + " matches all {PROJECT, DATASET, " + + "SCREEN, PLATE, RUN, WELL}.").ofType(rstring("")), scripts.String( P_TARG_COLID, optional=True, grouping="3.5", default="OBJECT_ID", - description="The column name in the .csv containing the id" + - " of the objects to annotate. " + - "Matches in exclude parameter."), + description="The column name in the CSV containing " + + "the objects IDs."), scripts.String( P_TARG_COLNAME, optional=True, grouping="3.6", default="OBJECT_NAME", - description="The column name in the .csv containing the name of " + - "the objects to annotate (used if no column " + - "ID is provided or found in the .csv). Matches " + - " in exclude parameter."), + description="The column name in the CSV containing " + + "the objects names. (used only if the column " + + "ID is not found"), authors=["Christian Evenhuis", "Tom Boissonnet", "Jens Wendt"], institutions=["MIF UTS", "CAi HHU", "MiN WWU"], diff --git a/omero/annotation_scripts/Remove_KeyVal.py b/omero/annotation_scripts/Remove_KeyVal.py index bd355d018..e9c85a6e6 100644 --- a/omero/annotation_scripts/Remove_KeyVal.py +++ b/omero/annotation_scripts/Remove_KeyVal.py @@ -197,8 +197,7 @@ def run_script(): client = scripts.client( 'Remove Key-Value pairs', """ - This script deletes for the selected objects the key-value pairs - associated to the given namespace. + Deletes key-value pairs of the selected objects. \t Check the guide for more information on parameters and errors: https://guide-kvpairs-scripts.readthedocs.io/en/latest/index.html @@ -208,26 +207,25 @@ def run_script(): scripts.String( P_DTYPE, optional=False, grouping="1", - description="Parent-data type of the objects to annotate.", + description="Data type of the parent objects.", values=source_types, default="Dataset"), scripts.List( P_IDS, optional=False, grouping="1.1", - description="List of parent-data IDs containing the objects " + - "to delete annotation from.").ofType(rlong(0)), + description="IDs of the parent objects").ofType(rlong(0)), scripts.String( P_TARG_DTYPE, optional=False, grouping="1.2", - description="Choose the object type to delete annotation from.", + description="Data type to process from the selected " + + "parent objects.", values=target_types, default=""), scripts.List( P_NAMESPACE, optional=True, grouping="1.3", - description="Annotation with these namespace will " + - "be deleted. Default is the client" + - " namespace, meaning editable in " + - "OMERO.web").ofType(rstring("")), + description="Namespace(s) of the key-value pairs to " + + "delete. Client namespace by default, " + + "'*' for all.").ofType(rstring("")), scripts.Bool( P_AGREEMENT, optional=True, grouping="2", From d66f818e8ed8dd1195a92d2d2944048215e2dd61 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Sun, 5 May 2024 18:06:36 +0200 Subject: [PATCH 114/127] convert test with and without merge --- test/integration/test_annotation_scripts.py | 84 +++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 668d9af67..df8634d2a 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -381,6 +381,90 @@ def test_convert(self): assert len(value) == 1 assert value[0] == ("key_1", "val_A") + def test_convert_no_merge(self): + sid = super(TestAnnotationScripts, self).get_script(convert_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + image = self.make_image(name="testImage", client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_2", "val_B")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + args = { + "Data_Type": rstring("Image"), + "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "Target Data_Type": rstring(""), + "Old Namespace (blank for default)": rlist([rstring("test")]), + "New Namespace (blank for default)": rstring("new_ns"), + "Create new and merge": rbool(False) + } + + msg = run_script(client, sid, args, "Message") + + assert msg._val == "Updated kv pairs to 1/1 Image" + + conn = BlitzGateway(client_obj=client) + image_o = conn.getObject("Image", image.id.val) + + list_ann = list(image_o.listAnnotations(ns="new_ns")) + assert len(list_ann) == 2 + value = list_ann[0].getValue() + assert len(value) == 1 + value = list_ann[1].getValue() + assert len(value) == 1 + + def test_convert_merge(self): + sid = super(TestAnnotationScripts, self).get_script(convert_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + image = self.make_image(name="testImage", client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_2", "val_B")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + args = { + "Data_Type": rstring("Image"), + "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "Target Data_Type": rstring(""), + "Old Namespace (blank for default)": rlist([rstring("test")]), + "New Namespace (blank for default)": rstring("new_ns"), + "Create new and merge": rbool(True) + } + + msg = run_script(client, sid, args, "Message") + + assert msg._val == "Updated kv pairs to 1/1 Image" + + conn = BlitzGateway(client_obj=client) + image_o = conn.getObject("Image", image.id.val) + + list_ann = list(image_o.listAnnotations(ns="new_ns")) + assert len(list_ann) == 1 + value = list_ann[0].getValue() + assert len(value) == 2 + def test_remove(self): agreement = ( "I understand what I am doing and that this will result " + From ae9436beeac495656490bbf1ede0f6e8bfd2762b Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Sun, 5 May 2024 22:42:03 +0200 Subject: [PATCH 115/127] export script tests --- test/integration/test_annotation_scripts.py | 173 +++++++++++++++++++- 1 file changed, 164 insertions(+), 9 deletions(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index df8634d2a..f38ff2690 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -28,9 +28,10 @@ from omero.gateway import BlitzGateway from omero.model import AnnotationAnnotationLinkI, MapAnnotationI from omero.constants.metadata import NSCLIENTMAPANNOTATION, NSINSIGHTTAGSET -from omero.rtypes import rstring, rlist, rbool +from omero.rtypes import rstring, rlist, rbool, rlong from omero.util.temp_files import create_path import omero.scripts +from script import get_file_contents import pytest from script import ScriptTest @@ -105,7 +106,7 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): # run the script args = DEFAULT_IMPORT_ARGS.copy() args["Data_Type"] = rstring("Plate") - args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["IDs"] = rlist([rlong(plate.id.val)]) args["Target Data_Type"] = rstring("-- Well") args["File_Annotation"] = rstring(str(fa.id)) args["Import tags"] = rbool(import_tag) @@ -197,7 +198,7 @@ def test_import_tags(self, import_tag, tag_creation): # run the script args = DEFAULT_IMPORT_ARGS.copy() args["Data_Type"] = rstring("Plate") - args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["IDs"] = rlist([rlong(plate.id.val)]) args["Target Data_Type"] = rstring("-- Well") args["File_Annotation"] = rstring(str(fa.id)) args["Import tags"] = rbool(import_tag) @@ -262,7 +263,7 @@ def test_import_split(self): # run the script args = DEFAULT_IMPORT_ARGS.copy() args["Data_Type"] = rstring("Plate") - args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["IDs"] = rlist([rlong(plate.id.val)]) args["Target Data_Type"] = rstring("-- Well") args["File_Annotation"] = rstring(str(fa.id)) args["Split values on"] = rstring(",") @@ -322,7 +323,7 @@ def test_import_empty(self): # run the script args = DEFAULT_IMPORT_ARGS.copy() args["Data_Type"] = rstring("Plate") - args["IDs"] = rlist([omero.rtypes.rlong(plate.id.val)]) + args["IDs"] = rlist([rlong(plate.id.val)]) args["Target Data_Type"] = rstring("-- Well") args["File_Annotation"] = rstring(str(fa.id)) args["Exclude empty values"] = rbool(True) @@ -363,7 +364,7 @@ def test_convert(self): args = { "Data_Type": rstring("Image"), - "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "IDs": rlist([rlong(image.id.val)]), "Target Data_Type": rstring(""), "Old Namespace (blank for default)": rlist([rstring("test")]), "New Namespace (blank for default)": rstring("new_ns"), @@ -403,7 +404,7 @@ def test_convert_no_merge(self): args = { "Data_Type": rstring("Image"), - "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "IDs": rlist([rlong(image.id.val)]), "Target Data_Type": rstring(""), "Old Namespace (blank for default)": rlist([rstring("test")]), "New Namespace (blank for default)": rstring("new_ns"), @@ -446,7 +447,7 @@ def test_convert_merge(self): args = { "Data_Type": rstring("Image"), - "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "IDs": rlist([rlong(image.id.val)]), "Target Data_Type": rstring(""), "Old Namespace (blank for default)": rlist([rstring("test")]), "New Namespace (blank for default)": rstring("new_ns"), @@ -486,7 +487,7 @@ def test_remove(self): args = { "Data_Type": rstring("Image"), - "IDs": rlist([omero.rtypes.rlong(image.id.val)]), + "IDs": rlist([rlong(image.id.val)]), "Target Data_Type": rstring(""), "Namespace (blank for default)": rlist([rstring("test_delete")]), agreement: rbool(True) @@ -500,3 +501,157 @@ def test_remove(self): image_o = conn.getObject("Image", image.id.val) assert len(list(image_o.listAnnotations())) == 0 + + def test_export(self): + sid = super(TestAnnotationScripts, self).get_script(export_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + image = self.make_image(name="testImage", client=client) + + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_A"), + omero.model.NamedValue("key_2", "val_B")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image, kv, client=client) + + args = { + "Data_Type": rstring("Image"), + "IDs": rlist([rlong(image.id.val)]), + "Target Data_Type": rstring(""), + "Namespace (blank for default)": rlist([rstring("test")]), + "CSV separator": rstring("TAB"), + "Include parent container names": rbool(False), + "Include namespace": rbool(False), + "Include tags": rbool(False) + } + + msg = run_script(client, sid, args, "Message") + + assert msg._val == f"The csv is attached to Image:{image.id.val}" + + conn = BlitzGateway(client_obj=client) + img_o = conn.getObject("Image", image.id.val) + + file_ann = img_o.getAnnotation(ns="KeyVal_export") + fid = file_ann.getFile().getId() + csv_text = get_file_contents(self.new_client(user=user), fid) + lines = csv_text.split("\n") + assert len(lines) == 3 + assert lines[-1] == "" # Last empty line + key_l = lines[0].split("\t") + assert key_l[0] == "OBJECT_ID" + assert key_l[1] == "OBJECT_NAME" + assert "key_1" in key_l + assert "key_2" in key_l + + img1_l = lines[1].split("\t") + assert img1_l[0] == str(image.id.val) + assert img1_l[1] == "testImage" + assert "val_A" in img1_l + assert "val_B" in img1_l + + @pytest.mark.parametrize('same_ns', [True, False]) + def test_export_all_opt(self, same_ns): + sid = super(TestAnnotationScripts, self).get_script(export_script) + assert sid > 0 + + client, user = self.new_client_and_user() + conn = BlitzGateway(client_obj=client) + update = conn.getUpdateService() + + # making tags + tagset = self.make_tag( + name="condition", ns=NSINSIGHTTAGSET, client=client + ) + tag1 = self.make_tag(name="ctrl", client=client) + tag2 = self.make_tag(name="test", client=client) + + link = AnnotationAnnotationLinkI() + link.setParent(tagset) + link.setChild(tag1) + update.saveObject(link) + tagset = conn.getObject("TagAnnotation", tagset.id.val)._obj + link = AnnotationAnnotationLinkI() + link.setParent(tagset) + link.setChild(tag2) + update.saveObject(link) + + image1 = self.make_image(name="testImage1", client=client) + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_A"), + omero.model.NamedValue("key_2", "val_B")]) + kv.setNs(rstring("test")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image1, kv, client=client) + self.link(image1, tag1, client=client) + + image2 = self.make_image(name="testImage2", client=client) + kv = MapAnnotationI() + kv.setMapValue([omero.model.NamedValue("key_1", "val_C"), + omero.model.NamedValue("key_2", "val_D")]) + if same_ns: + kv.setNs(rstring("test")) + else: + kv.setNs(rstring("other")) + kv = client.sf.getUpdateService().saveAndReturnObject(kv) + self.link(image2, kv, client=client) + self.link(image2, tag2, client=client) + + ns_l = [rstring("test")] + if not same_ns: + ns_l.append(rstring("other")) + + args = { + "Data_Type": rstring("Image"), + "IDs": rlist([rlong(image1.id.val), rlong(image2.id.val)]), + "Target Data_Type": rstring(""), + "Namespace (blank for default)": rlist(ns_l), + "CSV separator": rstring("TAB"), + "Include parent container names": rbool(True), + "Include namespace": rbool(True), + "Include tags": rbool(True) + } + + run_script(client, sid, args, "Message") + + conn = BlitzGateway(client_obj=client) + img1_o = conn.getObject("Image", image1.id.val) + img2_o = conn.getObject("Image", image2.id.val) + + file_ann = img1_o.getAnnotation(ns="KeyVal_export") + if file_ann is None: + file_ann = img2_o.getAnnotation(ns="KeyVal_export") + + fid = file_ann.getFile().getId() + csv_text = get_file_contents(self.new_client(user=user), fid) + lines = csv_text.split("\n") + assert len(lines) == 5 + ns_l = lines[0].split("\t") + assert ns_l[0] == "NAMESPACE" + key_l = lines[1].split("\t") + img1_l = lines[2].split("\t") + img2_l = lines[3].split("\t") + assert len(ns_l) == len(key_l) + assert len(key_l) == len(img1_l) + assert len(img1_l) == len(img2_l) + if same_ns: + assert len(key_l) == 5 + k1_pos = key_l.index("key_1") + assert img1_l[k1_pos] == "val_A" + assert img2_l[k1_pos] == "val_C" + k2_pos = key_l.index("key_2") + assert img1_l[k2_pos] == "val_B" + assert img2_l[k2_pos] == "val_D" + else: + assert len(key_l) == 7 + ns1_pos = ns_l.index("test") + ns2_pos = ns_l.index("other") + assert img1_l[ns2_pos] == "" + assert img2_l[ns1_pos] == "" + + tag_pos = key_l.index("TAG") + assert img1_l[tag_pos] == "ctrl[condition]" + assert img2_l[tag_pos] == "test[condition]" From e73217a2d39d0c2325a8f5ed2beef2293132d468 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 8 May 2024 13:45:19 +0200 Subject: [PATCH 116/127] added missing True test parameter --- test/integration/test_annotation_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index f38ff2690..42e242b57 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -150,7 +150,7 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): assert value[2] == ("key_3", "val_I") @pytest.mark.parametrize('import_tag', [True, False]) - @pytest.mark.parametrize('tag_creation', [False]) + @pytest.mark.parametrize('tag_creation', [True, False]) def test_import_tags(self, import_tag, tag_creation): sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 From e05c3d3e47f4bc6a5b664848dce74b3afb3b1ea3 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 8 May 2024 13:48:15 +0200 Subject: [PATCH 117/127] remove convert test duplicate with parametrization --- test/integration/test_annotation_scripts.py | 61 +++++---------------- 1 file changed, 13 insertions(+), 48 deletions(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 42e242b57..c42153437 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -382,7 +382,8 @@ def test_convert(self): assert len(value) == 1 assert value[0] == ("key_1", "val_A") - def test_convert_no_merge(self): + @pytest.mark.parametrize('merge', [True, False]) + def test_convert_no_merge(self, merge): sid = super(TestAnnotationScripts, self).get_script(convert_script) assert sid > 0 @@ -408,50 +409,7 @@ def test_convert_no_merge(self): "Target Data_Type": rstring(""), "Old Namespace (blank for default)": rlist([rstring("test")]), "New Namespace (blank for default)": rstring("new_ns"), - "Create new and merge": rbool(False) - } - - msg = run_script(client, sid, args, "Message") - - assert msg._val == "Updated kv pairs to 1/1 Image" - - conn = BlitzGateway(client_obj=client) - image_o = conn.getObject("Image", image.id.val) - - list_ann = list(image_o.listAnnotations(ns="new_ns")) - assert len(list_ann) == 2 - value = list_ann[0].getValue() - assert len(value) == 1 - value = list_ann[1].getValue() - assert len(value) == 1 - - def test_convert_merge(self): - sid = super(TestAnnotationScripts, self).get_script(convert_script) - assert sid > 0 - - client, user = self.new_client_and_user() - conn = BlitzGateway(client_obj=client) - image = self.make_image(name="testImage", client=client) - - kv = MapAnnotationI() - kv.setMapValue([omero.model.NamedValue("key_1", "val_A")]) - kv.setNs(rstring("test")) - kv = client.sf.getUpdateService().saveAndReturnObject(kv) - self.link(image, kv, client=client) - - kv = MapAnnotationI() - kv.setMapValue([omero.model.NamedValue("key_2", "val_B")]) - kv.setNs(rstring("test")) - kv = client.sf.getUpdateService().saveAndReturnObject(kv) - self.link(image, kv, client=client) - - args = { - "Data_Type": rstring("Image"), - "IDs": rlist([rlong(image.id.val)]), - "Target Data_Type": rstring(""), - "Old Namespace (blank for default)": rlist([rstring("test")]), - "New Namespace (blank for default)": rstring("new_ns"), - "Create new and merge": rbool(True) + "Create new and merge": rbool(merge) } msg = run_script(client, sid, args, "Message") @@ -462,9 +420,16 @@ def test_convert_merge(self): image_o = conn.getObject("Image", image.id.val) list_ann = list(image_o.listAnnotations(ns="new_ns")) - assert len(list_ann) == 1 - value = list_ann[0].getValue() - assert len(value) == 2 + if not merge: + assert len(list_ann) == 2 + value = list_ann[0].getValue() + assert len(value) == 1 + value = list_ann[1].getValue() + assert len(value) == 1 + else: + assert len(list_ann) == 1 + value = list_ann[0].getValue() + assert len(value) == 2 def test_remove(self): agreement = ( From 5d94201037649ffa87c22323f44fa2642b4f3ddd Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 8 May 2024 15:07:49 +0200 Subject: [PATCH 118/127] delete agreement parametrization --- test/integration/test_annotation_scripts.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index c42153437..4f1dda49f 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -431,7 +431,8 @@ def test_convert_no_merge(self, merge): value = list_ann[0].getValue() assert len(value) == 2 - def test_remove(self): + @pytest.mark.parametrize('agree_check', [True, False]) + def test_remove(self, agree_check): agreement = ( "I understand what I am doing and that this will result " + "in a batch deletion of key-value pairs from the server" @@ -455,17 +456,17 @@ def test_remove(self): "IDs": rlist([rlong(image.id.val)]), "Target Data_Type": rstring(""), "Namespace (blank for default)": rlist([rstring("test_delete")]), - agreement: rbool(True) + agreement: rbool(agree_check) } msg = run_script(client, sid, args, "Message") - - assert msg._val == "Key value data deleted from 1 of 1 objects" - - conn = BlitzGateway(client_obj=client) - image_o = conn.getObject("Image", image.id.val) - - assert len(list(image_o.listAnnotations())) == 0 + if not agree_check: # There should be an AssertionError, returning None + assert msg is None + else: + assert msg._val == "Key value data deleted from 1 of 1 objects" + conn = BlitzGateway(client_obj=client) + image_o = conn.getObject("Image", image.id.val) + assert len(list(image_o.listAnnotations())) == 0 def test_export(self): sid = super(TestAnnotationScripts, self).get_script(export_script) From 8dc923b3206cbc1334dbcb7eead6e60f222982c3 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 8 May 2024 15:25:07 +0200 Subject: [PATCH 119/127] added docstring to tests --- test/integration/test_annotation_scripts.py | 33 ++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 4f1dda49f..5b3bd8a8a 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -80,6 +80,9 @@ class TestAnnotationScripts(ScriptTest): ]) @pytest.mark.parametrize('ns_in_csv', [True, False]) def test_import(self, import_tag, tag_creation, ns, ns_in_csv): + """ + Test various import option with a simple CSV + """ sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -152,6 +155,9 @@ def test_import(self, import_tag, tag_creation, ns, ns_in_csv): @pytest.mark.parametrize('import_tag', [True, False]) @pytest.mark.parametrize('tag_creation', [True, False]) def test_import_tags(self, import_tag, tag_creation): + """ + Test the import of tags from a CSV with tag information + """ sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -241,6 +247,9 @@ def test_import_tags(self, import_tag, tag_creation): assert value[0] == ("key_1", "val_C") def test_import_split(self): + """ + Test the import of KV with inner cell splitting + """ sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -301,6 +310,9 @@ def test_import_split(self): assert value[2] == ("key_2", "val_H") def test_import_empty(self): + """ + Test the import from a CSV with exclusion of empty cells + """ sid = super(TestAnnotationScripts, self).get_script(import_script) assert sid > 0 @@ -349,6 +361,9 @@ def test_import_empty(self): assert value[0] == ("key_2", "val_B") def test_convert(self): + """ + Test the conversion of KV pairs namespace + """ sid = super(TestAnnotationScripts, self).get_script(convert_script) assert sid > 0 @@ -384,6 +399,10 @@ def test_convert(self): @pytest.mark.parametrize('merge', [True, False]) def test_convert_no_merge(self, merge): + """ + Test the conversion of KV pairs namespace with different + merging options + """ sid = super(TestAnnotationScripts, self).get_script(convert_script) assert sid > 0 @@ -433,6 +452,11 @@ def test_convert_no_merge(self, merge): @pytest.mark.parametrize('agree_check', [True, False]) def test_remove(self, agree_check): + """ + Test the removal of KV pairs, and if the script fails without the + agreement checked. + """ + agreement = ( "I understand what I am doing and that this will result " + "in a batch deletion of key-value pairs from the server" @@ -460,7 +484,7 @@ def test_remove(self, agree_check): } msg = run_script(client, sid, args, "Message") - if not agree_check: # There should be an AssertionError, returning None + if not agree_check: # should be an AssertionError, returning None assert msg is None else: assert msg._val == "Key value data deleted from 1 of 1 objects" @@ -469,6 +493,9 @@ def test_remove(self, agree_check): assert len(list(image_o.listAnnotations())) == 0 def test_export(self): + """ + Test the export of KV pairs into a CSV + """ sid = super(TestAnnotationScripts, self).get_script(export_script) assert sid > 0 @@ -521,6 +548,10 @@ def test_export(self): @pytest.mark.parametrize('same_ns', [True, False]) def test_export_all_opt(self, same_ns): + """ + Test the export of two KV pairs into a CSV with all options checked + (namespace, parent container, tags). + """ sid = super(TestAnnotationScripts, self).get_script(export_script) assert sid > 0 From 48e01e61e4371d558d60993fa68501a4e3110047 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 5 Jun 2024 13:01:55 +0100 Subject: [PATCH 120/127] year of update to 2024 --- omero/annotation_scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/README.md b/omero/annotation_scripts/README.md index e3d0197c7..371aafe88 100644 --- a/omero/annotation_scripts/README.md +++ b/omero/annotation_scripts/README.md @@ -37,7 +37,7 @@ History This repository started as a fork of [evehuis/omero-user-scripts](). Ownership was transferred to @CFGrote after merging a pull request that fixed a number of bugs and ported the original code from python2.7 to python3.x -In 2023, the scripts were reworked by Tom Boissonnet and Jens Wendt to extend the annotation to all OMERO objects, and to include a new script to convert namespaces of map annotations. +In 2024, the scripts were reworked by Tom Boissonnet and Jens Wendt to extend the annotation to all OMERO objects, and to include a new script to convert namespaces of map annotations. Contributions From 3fd54e94c7c83e4c00d1b3ebe6f88263ee0273a6 Mon Sep 17 00:00:00 2001 From: Tom Boissonnet Date: Wed, 5 Jun 2024 15:26:25 +0200 Subject: [PATCH 121/127] added affiliation --- omero/annotation_scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omero/annotation_scripts/README.md b/omero/annotation_scripts/README.md index 371aafe88..87073c8ee 100644 --- a/omero/annotation_scripts/README.md +++ b/omero/annotation_scripts/README.md @@ -37,7 +37,7 @@ History This repository started as a fork of [evehuis/omero-user-scripts](). Ownership was transferred to @CFGrote after merging a pull request that fixed a number of bugs and ported the original code from python2.7 to python3.x -In 2024, the scripts were reworked by Tom Boissonnet and Jens Wendt to extend the annotation to all OMERO objects, and to include a new script to convert namespaces of map annotations. +In 2024, the scripts were reworked by Tom Boissonnet (HHU Düsseldorf) and Jens Wendt (WWU Münster) to extend the annotation to all OMERO objects, and to include a new script to convert namespaces of map annotations. Contributions From d9054e0ba7e1046d19e84c0a3f97dbf22d6386bc Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jul 2024 16:10:56 +0200 Subject: [PATCH 122/127] inlude existing KV having target ns with merge option --- omero/annotation_scripts/Convert_KeyVal_namespace.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/omero/annotation_scripts/Convert_KeyVal_namespace.py b/omero/annotation_scripts/Convert_KeyVal_namespace.py index 90e44feac..004ad89ba 100644 --- a/omero/annotation_scripts/Convert_KeyVal_namespace.py +++ b/omero/annotation_scripts/Convert_KeyVal_namespace.py @@ -263,7 +263,9 @@ def run_script(): scripts.Bool( P_MERGE, optional=True, grouping="1.6", description="Check to merge selected key-value pairs " + - "into a single new one", default=False), + "into a single new one (will also include " + + "existing key-value pairs having the New Namespace)", + default=False), authors=["Tom Boissonnet"], institutions=["CAi HHU"], @@ -321,6 +323,9 @@ def parameters_parsing(client): if params[P_TARG_DTYPE] == "Acquisition": params[P_TARG_DTYPE] = "PlateAcquisition" + if params[P_MERGE]: + # If merge, also include existing target NS + params[P_OLD_NS].append(params[P_NEW_NS]) # Remove duplicate entries from namespace list tmp = params[P_OLD_NS] if "*" in tmp: From 9ad2ff48acbd50cfa068a4e3efd343e87469aff3 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jul 2024 16:37:56 +0200 Subject: [PATCH 123/127] output with csv.writer and includes sep metadata --- omero/annotation_scripts/Export_to_csv.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index 927144590..e4438f701 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -32,6 +32,7 @@ import tempfile import os import re +import csv from collections import OrderedDict, defaultdict CHILD_OBJECTS = { @@ -391,10 +392,14 @@ def attach_csv(conn, obj_, rows, separator, csv_name): # create the tmp directory tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) - tfile = os.fdopen(fd, 'w', encoding="utf-8") - for row in rows: - tfile.write(f"{separator.join(row)}\n") - tfile.close() + with os.fdopen(fd, 'w', encoding="utf-8") as tfile: + tfile.write(f"sep={separator}\r\n") # Indicates separator for excel + csvwriter = csv.writer(tfile, + delimiter=separator, + quotechar='"', + quoting=csv.QUOTE_MINIMAL) + for row in rows: + csvwriter.writerow(row) # link it to the object file_ann = conn.createFileAnnfromLocalFile( From 3d4098d821963d5e2157556f240e3c1c7f17557f Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jul 2024 16:54:17 +0200 Subject: [PATCH 124/127] handle sep= metadata of the csv --- omero/annotation_scripts/Import_from_csv.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index 57312590e..bcb19f9d0 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -364,6 +364,14 @@ def read_csv(conn, original_file, delimiter, import_tags): "file to utf-8 encoding" + str(e)) + # Read delimiter from CSV first line if exist + re_delimiter = re.compile("sep=(?P.?)") + match = re_delimiter.match(csv_content[0]) + if match: # Need to discard first row + csv_content = csv_content[1:] + if delimiter is None: # (and we detect delimiter if not given) + delimiter = match.group('delimiter') + if delimiter is None: try: # Sniffing on a maximum of four lines From e227863208afa763715cef73a3be23c098fad370 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jul 2024 16:57:05 +0200 Subject: [PATCH 125/127] added | option for csv separator --- omero/annotation_scripts/Export_to_csv.py | 2 +- omero/annotation_scripts/Import_from_csv.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index e4438f701..cf15d0736 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -439,7 +439,7 @@ def run_script(): rstring("--- Image") ] - separators = [";", ",", "TAB"] + separators = [";", ",", "TAB", "|"] # Here we define the script name and description. # Good practice to put url here to give users more guidance on how to run # your script. diff --git a/omero/annotation_scripts/Import_from_csv.py b/omero/annotation_scripts/Import_from_csv.py index bcb19f9d0..408d09416 100644 --- a/omero/annotation_scripts/Import_from_csv.py +++ b/omero/annotation_scripts/Import_from_csv.py @@ -376,7 +376,7 @@ def read_csv(conn, original_file, delimiter, import_tags): try: # Sniffing on a maximum of four lines delimiter = csv.Sniffer().sniff("\n".join(csv_content[:4]), - ",;\t").delimiter + "|,;\t").delimiter except Exception as e: assert False, ("Failed to sniff CSV delimiter: " + str(e)) rows = list(csv.reader(csv_content, delimiter=delimiter)) From 000b1cee64159fffb6459fd820868b9a5b4c552c Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 16 Jul 2024 17:03:52 +0200 Subject: [PATCH 126/127] handle new sep= metadata in export --- test/integration/test_annotation_scripts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/test_annotation_scripts.py b/test/integration/test_annotation_scripts.py index 5b3bd8a8a..db916fff5 100644 --- a/test/integration/test_annotation_scripts.py +++ b/test/integration/test_annotation_scripts.py @@ -532,6 +532,7 @@ def test_export(self): fid = file_ann.getFile().getId() csv_text = get_file_contents(self.new_client(user=user), fid) lines = csv_text.split("\n") + lines = lines[1:] # Ignore sep= metadata assert len(lines) == 3 assert lines[-1] == "" # Last empty line key_l = lines[0].split("\t") @@ -625,6 +626,7 @@ def test_export_all_opt(self, same_ns): fid = file_ann.getFile().getId() csv_text = get_file_contents(self.new_client(user=user), fid) lines = csv_text.split("\n") + lines = lines[1:] # Ignore sep= metadata assert len(lines) == 5 ns_l = lines[0].split("\t") assert ns_l[0] == "NAMESPACE" From a48786fff1a4ff6be52015d3a5604f8299ecccaf Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 18 Jul 2024 09:44:42 +0200 Subject: [PATCH 127/127] replaced \r\n ending by \n --- omero/annotation_scripts/Export_to_csv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/omero/annotation_scripts/Export_to_csv.py b/omero/annotation_scripts/Export_to_csv.py index cf15d0736..f12805a5f 100644 --- a/omero/annotation_scripts/Export_to_csv.py +++ b/omero/annotation_scripts/Export_to_csv.py @@ -393,11 +393,12 @@ def attach_csv(conn, obj_, rows, separator, csv_name): tmp_dir = tempfile.mkdtemp(prefix='MIF_meta') (fd, tmp_file) = tempfile.mkstemp(dir=tmp_dir, text=True) with os.fdopen(fd, 'w', encoding="utf-8") as tfile: - tfile.write(f"sep={separator}\r\n") # Indicates separator for excel + tfile.write(f"sep={separator}\n") # Indicates separator for excel csvwriter = csv.writer(tfile, delimiter=separator, quotechar='"', - quoting=csv.QUOTE_MINIMAL) + quoting=csv.QUOTE_MINIMAL, + lineterminator="\n") for row in rows: csvwriter.writerow(row)