Skip to content

Commit

Permalink
Check that dimension > 0 at shard creation
Browse files Browse the repository at this point in the history
  • Loading branch information
javitonino committed Jun 6, 2024
1 parent d6972e2 commit a2fdb60
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 8 deletions.
19 changes: 12 additions & 7 deletions nucliadb/src/migrations/0022_fill_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,33 +38,38 @@ async def migrate(context: ExecutionContext) -> None: ...

async def migrate_kb(context: ExecutionContext, kbid: str) -> None:
if not await needs_fixing(kbid):
print("Model metadata already filled")
logger.info("Model metadata already filled", extra={"kbid": kbid})
return

# Get learning config
learning_config = await learning_proxy.get_configuration(kbid)
if learning_config is None:
print("KB has no learning configuration")
logger.error("KB has no learning configuration", extra={"kbid": kbid})
return

model_metadata = parse_model_metadata_from_learning_config(learning_config)
if model_metadata.vector_dimension is None:
logger.error("Vector dimension not set in learning config")
logger.error("Vector dimension not set in learning config", extra={"kbid": kbid})
return

async with datamanagers.with_rw_transaction() as txn:
shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
if shards is None:
print("KB has no shards")
logger.error("KB has no shards", extra={"kbid": kbid})
return

shards.model.CopyFrom(model_metadata)
await datamanagers.cluster.update_kb_shards(txn, kbid=kbid, shards=shards)
await txn.commit()
print("Model metadata filled")

logger.info("Model metadata filled", extra={"kbid": kbid})


async def needs_fixing(kbid: str) -> bool:
async with datamanagers.with_ro_transaction() as txn:
shards = await datamanagers.cluster.get_kb_shards(txn, kbid=kbid)
if shards is None:
logger.warning("KB has no shards")
logger.error("KB has no shards", extra={"kbid": kbid})
return False
print(shards.model)

return shards.model is None or shards.model.vector_dimension == 0
5 changes: 4 additions & 1 deletion nucliadb_vectors/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,12 @@ impl TryFrom<VectorIndexConfig> for VectorConfig {

fn try_from(proto: VectorIndexConfig) -> Result<Self, Self::Error> {
let vector_type = match (proto.vector_type(), proto.vector_dimension) {
(ProtoVectorType::DenseF32, Some(dim)) => VectorType::DenseF32 {
(ProtoVectorType::DenseF32, Some(dim)) if dim > 0 => VectorType::DenseF32 {
dimension: dim as usize,
},
(ProtoVectorType::DenseF32, Some(_)) => {
return Err(VectorErr::InvalidConfiguration("Vector dimensionc cannot be 0"))
}
(ProtoVectorType::DenseF32, None) => VectorType::DenseF32Unaligned,
};
Ok(VectorConfig {
Expand Down

0 comments on commit a2fdb60

Please sign in to comment.