Skip to content

Commit

Permalink
Merge pull request #47 from CAVEconnectome/soft_delete_reference_anno…
Browse files Browse the repository at this point in the history 
…tation_on_updates

Soft delete reference annotation on updates
  • Loading branch information
@fcollman
fcollman authored May 13, 2024
2 parents f88c4cd + be8f30b commit 3a1d75c
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 96 deletions.
115 changes: 30 additions & 85 deletions dynamicannotationdb/annotation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,33 +96,18 @@ def create_table(
existing_tables = self.db._check_table_is_unique(table_name)

if table_metadata:
reference_table, track_updates = self.schema._parse_schema_metadata_params(
reference_table, _ = self.schema._parse_schema_metadata_params(
schema_type, table_name, table_metadata, existing_tables
)
else:
reference_table = None
track_updates = None

AnnotationModel = self.schema.create_annotation_model(
table_name,
schema_type,
table_metadata=table_metadata,
with_crud_columns=with_crud_columns,
)
if hasattr(AnnotationModel, "target_id") and reference_table:

reference_table_name = self.db.get_table_sql_metadata(reference_table)
logging.info(
f"{table_name} is targeting reference table: {reference_table_name}"
)
if track_updates:
self.create_reference_update_trigger(
table_name, reference_table, AnnotationModel
)
description += (
f" [Note: This table '{AnnotationModel.__name__}' will update the 'target_id' "
f"foreign_key when updates are made to the '{reference_table}' table] "
)

self.db.base.metadata.tables[AnnotationModel.__name__].create(
bind=self.db.engine
Expand Down Expand Up @@ -229,48 +214,6 @@ def update_table_metadata(
logging.info(f"Table: {table_name} metadata updated ")
return self.db.get_table_metadata(table_name)

def create_reference_update_trigger(self, table_name, reference_table, model):
func_name = f"{table_name}_update_reference_id"
func = DDL(
f"""
CREATE or REPLACE function {func_name}()
returns TRIGGER
as $func$
begin
if EXISTS
(SELECT 1
FROM information_schema.columns
WHERE table_name='{reference_table}'
AND column_name='superceded_id') THEN
update {table_name} ref
set target_id = new.superceded_id
where ref.target_id = old.id;
return new;
else
return NULL;
END if;
end;
$func$ language plpgsql;
"""
)
trigger = DDL(
f"""CREATE TRIGGER update_{table_name}_target_id AFTER UPDATE ON {reference_table}
FOR EACH ROW EXECUTE PROCEDURE {func_name}();"""
)

event.listen(
model.__table__,
"after_create",
func.execute_if(dialect="postgresql"),
)

event.listen(
model.__table__,
"after_create",
trigger.execute_if(dialect="postgresql"),
)
return True

def delete_table(self, table_name: str) -> bool:
"""Marks a table for deletion, which will
remove it from user visible calls
Expand Down Expand Up @@ -410,7 +353,8 @@ def update_annotation(self, table_name: str, annotation: dict) -> str:
table_name : str
name of targeted table to update annotations
annotation : dict
new data for that annotation
new data for that annotation, allows for partial updates but
requires an 'id' field to target the row
Returns
-------
Expand All @@ -427,8 +371,27 @@ def update_annotation(self, table_name: str, annotation: dict) -> str:
return "Annotation requires an 'id' to update targeted row"
schema_type, AnnotationModel = self._load_model(table_name)

try:
old_anno = (
self.db.cached_session.query(AnnotationModel)
.filter(AnnotationModel.id == anno_id)
.one()
)
except NoAnnotationsFoundWithID as e:
raise f"No result found for {anno_id}. Error: {e}" from e

if old_anno.superceded_id:
raise UpdateAnnotationError(anno_id, old_anno.superceded_id)

# Merge old data with new changes
old_data = {
column.name: getattr(old_anno, column.name)
for column in old_anno.__table__.columns
}
updated_data = {**old_data, **annotation}

new_annotation, __ = self.schema.split_flattened_schema_data(
schema_type, annotation
schema_type, updated_data
)

if hasattr(AnnotationModel, "created"):
Expand All @@ -438,32 +401,14 @@ def update_annotation(self, table_name: str, annotation: dict) -> str:

new_data = AnnotationModel(**new_annotation)

try:
old_anno = (
self.db.cached_session.query(AnnotationModel)
.filter(AnnotationModel.id == anno_id)
.one()
)
except NoAnnotationsFoundWithID as e:
raise f"No result found for {anno_id}. Error: {e}" from e
if hasattr(AnnotationModel, "target_id"):
new_data_map = self.db.get_automap_items(new_data)
for column_name, value in new_data_map.items():
setattr(old_anno, column_name, value)
old_anno.valid = True
update_map = {anno_id: old_anno.id}
else:
if old_anno.superceded_id:
raise UpdateAnnotationError(anno_id, old_anno.superceded_id)

self.db.cached_session.add(new_data)
self.db.cached_session.flush()
self.db.cached_session.add(new_data)
self.db.cached_session.flush()

deleted_time = datetime.datetime.utcnow()
old_anno.deleted = deleted_time
old_anno.superceded_id = new_data.id
old_anno.valid = False
update_map = {anno_id: new_data.id}
deleted_time = datetime.datetime.utcnow()
old_anno.deleted = deleted_time
old_anno.superceded_id = new_data.id
old_anno.valid = False
update_map = {anno_id: new_data.id}

(
self.db.cached_session.query(AnnoMetadata)
Expand Down
97 changes: 87 additions & 10 deletions tests/test_annotation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ def test_create_table(dadb_interface, annotation_metadata):


def test_create_all_schema_types(dadb_interface, annotation_metadata):

vx = annotation_metadata["voxel_resolution_x"]
vy = annotation_metadata["voxel_resolution_y"]
vz = annotation_metadata["voxel_resolution_z"]
Expand Down Expand Up @@ -75,14 +74,43 @@ def test_create_reference_table(dadb_interface, annotation_metadata):
voxel_resolution_z=vz,
table_metadata=table_metadata,
flat_segmentation_source=None,
with_crud_columns=False,
with_crud_columns=True,
)
assert table_name == table

table_info = dadb_interface.database.get_table_metadata(table)
assert table_info["reference_table"] == "anno_test"


def test_create_nested_reference_table(dadb_interface, annotation_metadata):
table_name = "reference_tag"
schema_type = "reference_tag"
vx = annotation_metadata["voxel_resolution_x"]
vy = annotation_metadata["voxel_resolution_y"]
vz = annotation_metadata["voxel_resolution_z"]

table_metadata = {
"reference_table": "presynaptic_bouton_types",
"track_target_id_updates": True,
}
table = dadb_interface.annotation.create_table(
table_name,
schema_type,
description="tags on 'presynaptic_bouton_types' table",
user_id="[email protected]",
voxel_resolution_x=vx,
voxel_resolution_y=vy,
voxel_resolution_z=vz,
table_metadata=table_metadata,
flat_segmentation_source=None,
with_crud_columns=True,
)
assert table_name == table

table_info = dadb_interface.database.get_table_metadata(table)
assert table_info["reference_table"] == "presynaptic_bouton_types"


def test_bad_schema_reference_table(dadb_interface, annotation_metadata):
table_name = "bad_reference_table"
schema_type = "synapse"
Expand Down Expand Up @@ -138,6 +166,20 @@ def test_insert_reference_annotation(dadb_interface, annotation_metadata):
assert inserted_id == [1]


def test_insert_nested_reference_tag_annotation(dadb_interface, annotation_metadata):
table_name = "reference_tag"

test_data = [
{
"tag": "here is a tag",
"target_id": 1,
}
]
inserted_id = dadb_interface.annotation.insert_annotations(table_name, test_data)

assert inserted_id == [1]


def test_insert_another_annotation(dadb_interface, annotation_metadata):
table_name = annotation_metadata["table_name"]

Expand All @@ -154,12 +196,22 @@ def test_insert_another_annotation(dadb_interface, annotation_metadata):
assert inserted_id == [2]


def test_get_annotation(dadb_interface, annotation_metadata):
def test_get_valid_annotation(dadb_interface, annotation_metadata):
table_name = annotation_metadata["table_name"]
test_data = dadb_interface.annotation.get_annotations(table_name, [1])
logging.info(test_data)

assert test_data[0]["id"] == 1
assert test_data[0]["valid"] is True


def test_get_reference_annotation(dadb_interface, annotation_metadata):
table_name = "presynaptic_bouton_types"
test_data = dadb_interface.annotation.get_annotations(table_name, [1])
logging.info(test_data)

assert test_data[0]["id"] == 1
assert test_data[0]["target_id"] == 1


def test_update_annotation(dadb_interface, annotation_metadata):
Expand All @@ -180,13 +232,22 @@ def test_update_annotation(dadb_interface, annotation_metadata):
assert test_data[0]["superceded_id"] == 3


def test_get_reference_annotation(dadb_interface, annotation_metadata):
def test_get_not_valid_annotation(dadb_interface, annotation_metadata):
table_name = annotation_metadata["table_name"]
test_data = dadb_interface.annotation.get_annotations(table_name, [1])
logging.info(test_data)

assert test_data[0]["id"] == 1
assert test_data[0]["valid"] is False


def test_get_reference_annotation_again(dadb_interface, annotation_metadata):
table_name = "presynaptic_bouton_types"
test_data = dadb_interface.annotation.get_annotations(table_name, [1])
logging.info(test_data)

assert test_data[0]["id"] == 1
assert test_data[0]["target_id"] == 3
assert test_data[0]["target_id"] == 1


def test_update_reference_annotation(dadb_interface, annotation_metadata):
Expand All @@ -195,20 +256,36 @@ def test_update_reference_annotation(dadb_interface, annotation_metadata):
test_data = {
"id": 1,
"bouton_type": "basmati",
"target_id": 3,
}

update_map = dadb_interface.annotation.update_annotation(table_name, test_data)

assert update_map == {1: 1}
test_data = dadb_interface.annotation.get_annotations(table_name, [1])
assert update_map == {1: 2}
# return values from newly updated row
test_data = dadb_interface.annotation.get_annotations(table_name, [2])
assert test_data[0]["bouton_type"] == "basmati"


def test_nested_update_reference_annotation(dadb_interface, annotation_metadata):
table_name = "reference_tag"

test_data = {
"tag": "here is a updated tag",
"id": 1,
}

update_map = dadb_interface.annotation.update_annotation(table_name, test_data)

assert update_map == {1: 2}
# return values from newly updated row
test_data = dadb_interface.annotation.get_annotations(table_name, [2])
assert test_data[0]["tag"] == "here is a updated tag"


def test_delete_reference_annotation(dadb_interface, annotation_metadata):
table_name = "presynaptic_bouton_types"

ids_to_delete = [1]
ids_to_delete = [2]
is_deleted = dadb_interface.annotation.delete_annotation(table_name, ids_to_delete)

assert is_deleted == ids_to_delete
Expand All @@ -217,7 +294,7 @@ def test_delete_reference_annotation(dadb_interface, annotation_metadata):
def test_delete_annotation(dadb_interface, annotation_metadata):
table_name = annotation_metadata["table_name"]

ids_to_delete = [1]
ids_to_delete = [3]
is_deleted = dadb_interface.annotation.delete_annotation(table_name, ids_to_delete)

assert is_deleted == ids_to_delete
Expand Down
2 changes: 1 addition & 1 deletion tests/test_database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def test_get_table_valid_row_count(dadb_interface, annotation_metadata):

result = dadb_interface.database.get_table_row_count(table_name, filter_valid=True)
logging.info(f"{table_name} valid row count: {result}")
assert result == 2
assert result == 1


def test_get_table_valid_timestamp_row_count(dadb_interface, annotation_metadata):
Expand Down

0 comments on commit 3a1d75c

Please sign in to comment.