Skip to content

Commit

Permalink
[r] Move table_name to version field in AnVIL bundles (#6639)
Browse files Browse the repository at this point in the history
  • Loading branch information
nadove-ucsc committed Oct 17, 2024
1 parent a2d2d2b commit 5532879
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 88 deletions.
2 changes: 1 addition & 1 deletion src/azul/indexer/document.py
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ def _from_index(cls,
) -> 'ContributionCoordinates[CataloguedEntityReference]':
entity_type = index_name.qualifier
assert index_name.doc_type is DocumentType.contribution
entity_id, deleted, bundle_uuid, bundle_version = document_id.split('_')
entity_id, deleted, bundle_uuid, bundle_version = document_id.split('_', 3)
if deleted == 'deleted':
deleted = True
elif deleted == 'exists':
Expand Down
2 changes: 0 additions & 2 deletions src/azul/plugins/metadata/anvil/indexer/transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -423,8 +423,6 @@ def reconcile_inner_entities(cls,
) -> tuple[JSON, BundleFQID]:
this_entity, this_bundle = this
that_entity, that_bundle = that
# All AnVIL bundles use a fixed known version
assert this_bundle.version == that_bundle.version, (this, that)
if this_entity.keys() == that_entity.keys():
return this
else:
Expand Down
24 changes: 11 additions & 13 deletions src/azul/plugins/repository/tdr_anvil/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
AbstractSet,
Callable,
Iterable,
cast,
)

import attrs
Expand Down Expand Up @@ -116,16 +117,14 @@ class BundleType(Enum):


class TDRAnvilBundleFQIDJSON(SourcedBundleFQIDJSON):
table_name: str
pass


@attrs.frozen(kw_only=True)
class TDRAnvilBundleFQID(TDRBundleFQID):
table_name: BundleType = attrs.field(converter=BundleType)

def to_json(self) -> TDRAnvilBundleFQIDJSON:
return dict(super().to_json(),
table_name=self.table_name.value)
return cast(TDRAnvilBundleFQIDJSON, super().to_json())


class TDRAnvilBundle(AnvilBundle[TDRAnvilBundleFQID], TDRBundle):
Expand Down Expand Up @@ -228,13 +227,12 @@ def _list_bundles(self,
bundles.append(TDRAnvilBundleFQID(
source=source,
uuid=bundle_uuid,
version=self._version,
table_name=BundleType(row['table_name'])
version=row['table_name']
))
return bundles

def resolve_bundle(self, fqid: SourcedBundleFQIDJSON) -> TDRAnvilBundleFQID:
if 'table_name' not in fqid:
if fqid['version'] is None:
# Resolution of bundles without the table name is expensive, so we
# only support it during canning.
assert not config.is_in_lambda, ('Bundle FQID lacks table name', fqid)
Expand All @@ -254,18 +252,18 @@ def resolve_bundle(self, fqid: SourcedBundleFQIDJSON) -> TDRAnvilBundleFQID:
return super().resolve_bundle(fqid)

def _emulate_bundle(self, bundle_fqid: TDRAnvilBundleFQID) -> TDRAnvilBundle:
if bundle_fqid.table_name is BundleType.primary:
if bundle_fqid.version == BundleType.primary.value:
log.info('Bundle %r is a primary bundle', bundle_fqid.uuid)
return self._primary_bundle(bundle_fqid)
elif bundle_fqid.table_name is BundleType.supplementary:
elif bundle_fqid.version == BundleType.supplementary.value:
log.info('Bundle %r is a supplementary bundle', bundle_fqid.uuid)
return self._supplementary_bundle(bundle_fqid)
elif bundle_fqid.table_name is BundleType.duos:
elif bundle_fqid.version == BundleType.duos.value:
assert config.duos_service_url is not None, bundle_fqid
log.info('Bundle %r is a DUOS bundle', bundle_fqid.uuid)
return self._duos_bundle(bundle_fqid)
else:
assert False, bundle_fqid.table_name
assert False, bundle_fqid.version

def _primary_bundle(self, bundle_fqid: TDRAnvilBundleFQID) -> TDRAnvilBundle:
source = bundle_fqid.source
Expand Down Expand Up @@ -321,7 +319,7 @@ def _supplementary_bundle(self, bundle_fqid: TDRAnvilBundleFQID) -> TDRAnvilBund
self.bundle_uuid_version,
self.datarepo_row_uuid_version)
source = bundle_fqid.source.spec
table_name = bundle_fqid.table_name.value
table_name = bundle_fqid.version
result = TDRAnvilBundle(fqid=bundle_fqid)
columns = self._columns(table_name)
bundle_entity = dict(one(self._run_sql(f'''
Expand Down Expand Up @@ -367,7 +365,7 @@ def _bundle_entity(self, bundle_fqid: TDRAnvilBundleFQID) -> KeyReference:
entity_id = uuids.change_version(bundle_uuid,
self.bundle_uuid_version,
self.datarepo_row_uuid_version)
table_name = bundle_fqid.table_name.value
table_name = bundle_fqid.version
entity_type = table_name.removeprefix('anvil_')
pk_column = entity_type + '_id'
bundle_entity = one(self._run_sql(f'''
Expand Down
64 changes: 32 additions & 32 deletions test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.results.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5532879

Please sign in to comment.