diff --git a/multigraph.py b/multigraph.py index 7a9830e..78df0ab 100644 --- a/multigraph.py +++ b/multigraph.py @@ -116,16 +116,18 @@ def parse_arguments(): if not args.no_dataframe: save_path = f'data/metagraph/{netuid}/df.parquet' - blocks = range(start_block, end_block, step_size) + blocks = sorted(range(start_block, end_block, -step_size)) df_loaded = None if os.path.exists(save_path): df_loaded = pd.read_parquet(save_path) blocks = [block for block in blocks if block not in df_loaded.block.unique()] print(f'Loaded dataframe from {save_path!r}. {len(df_loaded)} rows. {len(blocks)} blocks to process.') - if len(blocks)==0: - print('No blocks to process.') - sys.exit(0) - + + if len(blocks) == 0: + print('No blocks to process.') + sys.exit(0) + + print(f'Loading metagraphs from {blocks[0]}-{blocks[-1]} ({len(blocks)}) to create dataframe.') df = load_metagraphs(blocks[0], blocks[-1], block_step=step_size, datadir=datadir) if df_loaded is not None: df = pd.concat([df, df_loaded], ignore_index=True) diff --git a/multistats.py b/multistats.py index 4552830..66cffb4 100644 --- a/multistats.py +++ b/multistats.py @@ -16,6 +16,17 @@ api= wandb.Api(timeout=60) wandb.login(anonymous="allow") +import json +# Store the original json.loads +original_json_loads = json.loads + +def relaxed_json_loads(s, *args, **kwargs): + kwargs.pop("strict", None) # remove the strict argument if it exists + return original_json_loads(s, strict=False, *args, **kwargs) + +# Patch the library's json.loads +json.loads = relaxed_json_loads + def pull_wandb_runs(project='openvalidators', filters=None, min_steps=50, max_steps=100_000, ntop=10, netuid=None, summary_filters=None ): # TODO: speed this up by storing older runs @@ -264,7 +275,7 @@ def parse_arguments(): print(args) filters = None# {"tags": {"$in": [f'1.1.{i}' for i in range(10)]}} - # filters={'tags': {'$in': ['5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3']}} # Is foundation validator + if args.load_runs and os.path.exists('data/wandb.csv'): df_runs = pd.read_csv('data/wandb.csv') assert len(df_runs) >= args.ntop, f'Loaded {len(df_runs)} runs, but expected at least {args.ntop}'