diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70784b3e..3baefa5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8","3.9","3.10"] + python-version: ["3.8","3.9","3.10", "3.11"] env: DEBUG: TRUE steps: diff --git a/README.md b/README.md index 5faddd68..99fdf197 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ It is a tool to visualize DAO metrics. Currently, it shows DAO from [DAOstack](h ## Set-up & Running (Download app) You can either install it on your local machine, or if you prefer it, you can use the official docker image. -> If you only want to retrieve the data used by our application, you can follow [this guide](./cache_scripts/README.md) instead +> If you only want to retrieve the data used by our application, go to [grasia/dao-scripts](https://github.com/Grasia/dao-scripts) instead The easiest method by far to download and run the application is to use pip to install it @@ -51,7 +51,7 @@ Then, you can run the app using the commands `daoa-cache-scripts` and `daoa-serv Before launching the app, you have to run the following script in order to enable the cache stored in `datawarehouse`: ``` -daoa-cache-scripts +dao-scripts ``` After a few minutes, you can now run the app with: @@ -118,7 +118,7 @@ docker run --name dao-analyzer -it -p80:80 ghcr.io/grasia/dao-analyzer:latest-ca Now, you can update the datawarehouse using: ``` -docker exec -it dao-analyzer python -m cache_scripts +docker exec -it dao-analyzer dao-scripts ``` You can even add it to your system as a cron job to update it daily, weekly, etc... diff --git a/cache_scripts/CHANGELOG.md b/cache_scripts/CHANGELOG.md deleted file mode 100644 index f53c5fb4..00000000 --- a/cache_scripts/CHANGELOG.md +++ /dev/null @@ -1,53 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -## 1.1.9 - 2023-05-11 -- Obtaining textual fields from DAOstack proposals - - title - - description - - url - - confidence - - confidenceThreshold - -## 1.1.8 - 2023-01-23 -- Updated cache-scripts to get more daostack parameters - - thresholdConst - - minimumDaoBounty - - daoBountyConst - -## 1.1.7 - 2022-12-13 -- Obtaining more fields from DAOstack proposals - - queuedVotePeriodLimit - - boostedVotePeriodLimit - -## 1.1.6 - 2022-10-22 -- Obtaining more time fields from DAOstack proposals - -## 1.1.5 - 2022-10-17 -- Remove DAOstack phantom DAOs [#120](https://github.com/Grasia/dao-analyzer/issues/120) -- Added option to obtain non-registered DAOs from DAOstack (`--daostack-all`) - -## 1.1.4 - 2022-07-15 -- Added postProcessor to add a `dao` field to reputation mints and burns -- Not getting reputation mints/burns of amount 0 (not useful) - -## 1.1.3 - 2022-07-11 -- Added competitionId to daostack proposals - -## 1.1.2 - 2022-06-29 -- Added ReputationMint and ReputationBurn collectors to DAOstack - -## 1.1.1 - 2022-06-10 -- Added originalCreator field to Aragon Voting subgraph - -## 1.1.0 - 2022-05 -- Used `_change_block` filter to make every subgraph updatable -- Fixed cryptocompare error -- Fixed requests to `_blocks` endpoint -- Added --skip-token-balances option to cli - -## 1.0.3 - 2022-03-24 -- Obtaining assets of DAOs -- Added BlockScout balances collector -- Added CryptoCompare token prices collector -- Some changes on Class sctructure \ No newline at end of file diff --git a/cache_scripts/README.md b/cache_scripts/README.md deleted file mode 100644 index 9eefb3f9..00000000 --- a/cache_scripts/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# DAO-Analyzer's cache-scripts - -## Set-up & Running - -The easiest method by far to download and run the application is to use pip to install it - -``` -pip install dao-analyzer -``` - -Then, you can use this script using the command `daoa-cache-scripts` - -### Download -Enter in your terminal (git must be installed) and write down: - -``` -git clone https://github.com/Grasia/dao-analyzer -``` - -After that, move to repository root directory with: - -``` -cd dao-analyzer -``` - -### Installation -All code has been tested on Linux, but it should work on Windows and macOS, 'cause it just uses the python environment. - -So, you must install the following dependencies to run the tool: - -* python3 (3.10 or later) -* python3-pip - -Now, install the Python dependencies: - -`pip3 install -r requirements.txt` - -If you don't want to share Python dependencies among other projects, you should use a virtual environment, such as [virtualenv](https://docs.python-guide.org/dev/virtualenvs/). - -### How to run it? -If you want all the data used in the app, you can just use: - -``` -python3 -m cache_scripts -``` - -this will create a folder called `datawarehouse` with a lot of files in apache's arrow format. - -You can import those files to `pandas` with `read_feather`. For example: - -```python -pd.read_feather('datawarehouse/aragon/apps.arr') -``` - -## Usage guide -If you don't want all the data (and it can take a lot of time), you have a lot of options available to select whichever data you want. The full `--help` output is - -``` -usage: daoa-cache-scripts [-h] [-V] [-p [{aragon,daohaus,daostack} ...]] - [--ignore-errors | --no-ignore-errors] [-d] [-f] [-F] [--skip-daohaus-names] - [-n {mainnet,_theGraph,arbitrum,xdai,polygon} [{mainnet,_theGraph,arbitrum,xdai,polygon} ...]] - [-c COLLECTORS [COLLECTORS ...]] [--block-datetime BLOCK_DATETIME] - [-D DATAWAREHOUSE] [--cc-api-key CC_API_KEY] - -Main script to populate dao-analyzer cache - -options: - -h, --help show this help message and exit - -V, --version Displays the version and exits - -p [{aragon,daohaus,daostack} ...], --platforms [{aragon,daohaus,daostack} ...] - The platforms to update. Every platform is updated by default. - --ignore-errors, --no-ignore-errors - Whether to ignore errors and continue (default: True) - -d, --debug Shows debug info - -f, --force Removes the cache before updating - -F, --delete-force Removes the datawarehouse folder before doing anything - --skip-daohaus-names Skips the step of getting Daohaus Moloch's names, which takes some time - -n {mainnet,_theGraph,arbitrum,xdai,polygon} [{mainnet,_theGraph,arbitrum,xdai,polygon} ...], --networks {mainnet,_theGraph,arbitrum,xdai,polygon} [{mainnet,_theGraph,arbitrum,xdai,polygon} ...] - Networks to update. Every network is updated by default - -c COLLECTORS [COLLECTORS ...], --collectors COLLECTORS [COLLECTORS ...] - Collectors to run. For example: aragon/casts - --block-datetime BLOCK_DATETIME - Get data up to a block datetime (input in ISO format) - -D DATAWAREHOUSE, --datawarehouse DATAWAREHOUSE - Specifies the destination folder of the datawarehouse - --cc-api-key CC_API_KEY - Set the CryptoCompare API key (overrides environment variable) -``` - -### Getting only data from a platform -You can select the platform to download data about with the `--platform` selector. Let's download only data for daostack and aragon: - -``` -daoa-cache-scripts --platforms daostack aragon -``` - -### Getting only data from a network -You can select the chain to get data from with the `--networks` switch. For example, to get data only for xdai network, you can do: - -``` -daoa-cache-scripts --networks xdai -``` \ No newline at end of file diff --git a/cache_scripts/__init__.py b/cache_scripts/__init__.py deleted file mode 100644 index 3f22f142..00000000 --- a/cache_scripts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.1.9" \ No newline at end of file diff --git a/cache_scripts/__main__.py b/cache_scripts/__main__.py deleted file mode 100644 index 413eeee3..00000000 --- a/cache_scripts/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .main import main - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/cache_scripts/aragon/__init__.py b/cache_scripts/aragon/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cache_scripts/aragon/dao_names.json b/cache_scripts/aragon/dao_names.json deleted file mode 100644 index 38908a81..00000000 --- a/cache_scripts/aragon/dao_names.json +++ /dev/null @@ -1,256 +0,0 @@ -{ - "mainnet": [ - { - "address": "0xf47917b108ca4b820ccea2587546fbb9f7564b56", - "domain": "dcl.eth", - "name": "Decentraland", - "source": null - }, - { - "address": "0xfe1f2de598f42ce67bb9aad5ad473f0272d09b74", - "domain": "meloncouncil.eth", - "name": "Melon Council", - "source": null - }, - { - "address": "0x2de83b50af29678774d5abc4a7cb2a588762f28c", - "domain": "governance.aragonproject.eth", - "name": "Aragon Governance", - "source": null - }, - { - "address": "0x635193983512c621e6a3e15ee1dbf36f0c0db8e0", - "domain": "a1.aragonid.eth", - "name": "Aragon One", - "source": null - }, - { - "address": "0x67757a18eda83125270ef94dcec7658eb39bd8a5", - "domain": "blankdao.aragonid.eth", - "name": "BlankDAO", - "source": null - }, - { - "address": "0xcd3d9b832bff15e0a519610372c6aac651872dde", - "domain": "", - "name": "MyBit", - "source": null - }, - { - "address": "0x0ee165029b09d91a54687041adbc705f6376c67f", - "domain": "", - "name": "Livepeer", - "source": null - }, - { - "address": "0x5aad137d8f7d2dc6e1b2548c059b1483360bcc6a", - "domain": "brightid.aragonid.eth", - "name": "BrightID", - "source": null - }, - { - "address": "0x5feed010a99f695852f8eb7b12e77cf6ecd7be17", - "domain": "lightwave.aragonid.eth", - "name": "Lightwave", - "source": null - }, - { - "address": "0x40204daacb1480019a7a6826c699903df94ee019", - "domain": "network.aragonid.eth", - "name": "Aragon Network", - "source": null - }, - { - "address": "0x08ac31dd93c16f1f6c4a0fae540ba1ad52f581d0", - "domain": "sf.aragonid.eth", - "name": "Saint Fame", - "source": null - }, - { - "address": "0xa365a8429fcefdbe1e684dddda3531b6e8d96e75", - "domain": "lexdao.aragonid.eth", - "name": "lexDAO", - "source": null - }, - { - "address": "0x0c188b183ff758500d1d18b432313d10e9f6b8a4", - "domain": "piedao.aragonid.eth", - "name": "PieDAO", - "source": null - }, - { - "address": "0xc9fe36760d8fe233307e26b094de1f4fa090a12a", - "domain": "millstone.aragonid.eth", - "name": "Millstone", - "source": null - }, - { - "address": "0x4eef8cff7fd9bfab6cbcdf05b74d2161cadaff52", - "domain": "valtech.aragonid.eth", - "name": "Valtech", - "source": null - }, - { - "address": "0x2732fd9fd5f0e84b1b774cf5e6f5c812eafd455b", - "domain": "pnetwork.aragonid.eth", - "name": "pNetwork", - "source": null - }, - { - "address": "0xe00f7b744ab8333d64ed940dd36ed9398d8edbd2", - "domain": "kek.aragonid.eth", - "name": "Cryptokek.com", - "source": null - }, - { - "address": "0x7809e69cf83fcb768da9e7a698edc9f159b7d6f4", - "domain": "nucypherdao.aragonid.eth", - "name": "NuCypher DAO", - "source": null - }, - { - "address": "0xc63627255269ddc12de3bee4d7e6526dda59af70", - "domain": "6ef0c0b0.aragonid.eth", - "name": "AAVE", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x054086d40cf8fd5bf6200eda7f9c6877b0302dd1", - "domain": "aavegotchi.aragonid.eth", - "name": "Aavegotchi", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x313faebe5a6b9bb6824e89468c1100402b013127", - "domain": "dao.robonomics.eth", - "name": "Airalab", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xacf1921f2298d977843fe8b6d56d57e9060799ef", - "domain": "aragonchina.aragonid.eth", - "name": "Aragon China", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xf590e576a3353491164f70ccdfcb7e26c55f5cc0", - "domain": "barnbridgelaunch.aragonid.eth", - "name": "BarnBridge", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x6167696cfa005a09fa08387b8adcfe6f75e68b96", - "domain": "blq.aragonid.eth", - "name": "Blequity", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x23b5db33e824822e8d98698c4929be2ff1ba70e4", - "domain": "cadcad.aragonid.eth", - "name": "cadCAD", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x547dc95b3fbf42ae3467827a8a6ca08226e3288f", - "domain": "collab19.aragonid.eth", - "name": "Collab-19", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x117d1f8cb4bf6e74bc5d667f15af7b810bacccd6", - "domain": "cybercongress.aragonid.eth", - "name": "Cyber Congress", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xc45417092c7ba66052c92a4ad871fc60bdbc7009", - "domain": "eulerfoundation.aragonid.eth", - "name": "Cyber Foundation", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x238c1c129c00920d11520f25647d2b47ddbb3f87", - "domain": "dhedge.aragonid.eth", - "name": "dHEDGE", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x57ebe61f5f8303ad944136b293c1836b3803b4c0", - "domain": null, - "name": "DAONUTS", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x37f70a5161cc7cca39e2877457c00647353bee74", - "domain": "dappnode.aragonid.eth", - "name": "DAppNode", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x15bc735a248eb9790b9d4b85e44cf4c878667209", - "domain": "elimuai.aragonid.eth", - "name": "Elimu.ai", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xf7627121e08a1b3ad67c0fdf020dc1e695d20cbf", - "domain": "helpdao.aragonid.eth", - "name": "HelpDAO", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xd94aecff012668167c05c8690a42cac303c9cb4b", - "domain": "mstable.aragonid.eth", - "name": "mStable", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xe59cd35e68a8c51adda1c9f4681024066e13ff22", - "domain": "metagame.aragonid.eth", - "name": "MetaGame", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0xbaf2eb7b0649b8c28970e7d9e8f5dee9b6f6d9fe", - "domain": "nftx.aragonid.eth", - "name": "NFTX", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x60c0d7a799f41be34527d6da6e17d7e885338388", - "domain": "openesquire.aragonid.eth", - "name": "OpenESQ", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x0e5646fa4d398a81db51afd74547476fc0a0888d", - "domain": "pluricentra.aragonid.eth", - "name": "Pluricentra", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x0b6f260c19677d6f939e8b8e8fa4ccf67953ebac", - "domain": "rdao.aragonid.eth", - "name": "rDAI", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x001cd74c9a99b6c68e93fe69595124407b37aa8e", - "domain": "shilldao.aragonid.eth", - "name": "ShillDAO", - "source": "https://poweredby.aragon.org/" - }, - { - "address": "0x1c532bc3b37d05e30aae367e4facdcfc98f8a426", - "domain": null, - "name": "UniDAO", - "source": "https://poweredby.aragon.org/" - } - ], - "xdai": [ - { - "address": "0x8ccbeab14b5ac4a431fffc39f4bec4089020a155", - "name": "1hive" - } - ] -} \ No newline at end of file diff --git a/cache_scripts/aragon/runner.py b/cache_scripts/aragon/runner.py deleted file mode 100644 index 6672f420..00000000 --- a/cache_scripts/aragon/runner.py +++ /dev/null @@ -1,236 +0,0 @@ -""" - Descp: Aragon Runner and Collectors - - Created on: 02-nov-2021 - - Copyright 2021 David Davó - -""" -from typing import List - -import pkgutil -from gql.dsl import DSLField -import pandas as pd -import numpy as np -import json - -from ..aragon import __name__ as aragon_module_name -from ..common.cryptocompare import CCPricesCollector -from ..common import ENDPOINTS, Collector -from ..common.graphql import GraphQLCollector, GraphQLRunner -from ..common.blockscout import BlockscoutBallancesCollector - -class AppsCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('apps', runner, endpoint=ENDPOINTS[network]['aragon'], network=network) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.apps(**kwargs).select( - ds.App.id, - ds.App.isForwarder, - ds.App.isUpgradeable, - ds.App.repoName, - ds.App.repoAddress, - ds.App.organization.select(ds.Organization.id) - ) - -class BalancesCollector(BlockscoutBallancesCollector): - def __init__(self, runner, base, network: str): - super().__init__(runner, addr_key='recoveryVault', base=base, network=network) - -class CastsCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('casts', runner, endpoint=ENDPOINTS[network]['aragon_voting'], network=network, pbar_enabled=False) - - @self.postprocessor - def changeColumnNames(df: pd.DataFrame) -> pd.DataFrame: - df = df.rename(columns={ - 'voterId':'voter', - 'voteAppAddress':'appAddress', - 'voteOrgAddress':'orgAddress'}) - return df - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.casts(**kwargs).select( - ds.Cast.id, - ds.Cast.vote.select(ds.Vote.id), - ds.Cast.voter.select(ds.Voter.id), - ds.Cast.supports, - ds.Cast.stake, - ds.Cast.createdAt, - ds.Cast.vote.select( - ds.Vote.orgAddress, - ds.Vote.appAddress - ) - ) - -class OrganizationsCollector(GraphQLCollector): - DAO_NAMES=pkgutil.get_data(aragon_module_name, 'dao_names.json') - - def __init__(self, runner, network: str): - super().__init__('organizations', runner, endpoint=ENDPOINTS[network]['aragon'], network=network) - - @self.postprocessor - def set_dead_recoveryVault(df: pd.DataFrame) -> pd.DataFrame: - if df.empty: return df - - df['recoveryVault'] = df['recoveryVault'].replace(r'^0x0+$', np.NaN, regex=True) - return df - - @self.postprocessor - def apply_names(df: pd.DataFrame) -> pd.DataFrame: - names_dict = json.loads(self.DAO_NAMES) - - if self.network not in names_dict.keys() or \ - not names_dict[self.network] or \ - df.empty: - return df - - names_df = pd.json_normalize(names_dict[self.network]) - names_df['id'] = names_df['address'].str.lower() - names_df['name'] = names_df['name'].fillna(names_df['domain']) - names_df = names_df[['id', 'name']] - df = df.merge(names_df, on='id', how='left') - - return df - - @self.postprocessor - def copy_id(df: pd.DataFrame) -> pd.DataFrame: - df['orgAddress'] = df['id'] - return df - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.organizations(**kwargs).select( - ds.Organization.id, - ds.Organization.createdAt, - ds.Organization.recoveryVault - ) - -class MiniMeTokensCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('miniMeTokens', runner, endpoint=ENDPOINTS[network]['aragon_tokens'], network=network, pbar_enabled=False) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.miniMeTokens(**kwargs).select( - ds.MiniMeToken.id, - ds.MiniMeToken.address, - ds.MiniMeToken.totalSupply, - ds.MiniMeToken.transferable, - ds.MiniMeToken.name, - ds.MiniMeToken.symbol, - ds.MiniMeToken.orgAddress, - ds.MiniMeToken.appAddress, - ds.MiniMeToken.lastUpdateAt - ) - -class TokenHoldersCollector(GraphQLCollector): - def __init__(self, runner: GraphQLRunner, network: str): - super().__init__('tokenHolders', runner, endpoint=ENDPOINTS[network]['aragon_tokens'], network=network) - - @self.postprocessor - def add_minitokens(df: pd.DataFrame) -> pd.DataFrame: - tokens = runner.filterCollector(name='miniMeTokens', network=network).df - tokens = tokens.rename(columns={'address':'tokenAddress', 'orgAddress':'organizationAddress'}) - return df.merge(tokens[['tokenAddress', 'organizationAddress']], on='tokenAddress', how='left') - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.tokenHolders(**kwargs).select( - ds.TokenHolder.id, - ds.TokenHolder.address, - ds.TokenHolder.tokenAddress, - ds.TokenHolder.lastUpdateAt, - ds.TokenHolder.balance - ) - -class TokenPricesCollector(CCPricesCollector): - pass - -class ReposCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('repos', runner, network=network, endpoint=ENDPOINTS[network]['aragon']) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.repos(**kwargs).select( - ds.Repo.id, - ds.Repo.address, - ds.Repo.name, - ds.Repo.node, - ds.Repo.appCount - ) - -class TransactionsCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('transactions', runner, network=network, endpoint=ENDPOINTS[network]['aragon_finance']) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.transactions(**kwargs).select( - ds.Transaction.id, - ds.Transaction.orgAddress, - ds.Transaction.appAddress, - ds.Transaction.token, - ds.Transaction.entity, - ds.Transaction.isIncoming, - ds.Transaction.amount, - ds.Transaction.date, - ds.Transaction.reference - ) - -class VotesCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('votes', runner, network=network, endpoint=ENDPOINTS[network]['aragon_voting']) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.votes(**kwargs).select( - ds.Vote.id, - ds.Vote.orgAddress, - ds.Vote.appAddress, - ds.Vote.creator, - ds.Vote.originalCreator, - ds.Vote.metadata, - ds.Vote.executed, - ds.Vote.executedAt, - ds.Vote.startDate, - ds.Vote.supportRequiredPct, - ds.Vote.minAcceptQuorum, - ds.Vote.yea, - ds.Vote.nay, - ds.Vote.voteNum, - ds.Vote.votingPower - ) - -class AragonRunner(GraphQLRunner): - name: str = 'aragon' - - def __init__(self, dw=None): - super().__init__(dw) - self._collectors: List[Collector] = [] - ## TODO: Fix aragon-tokens xdai subgraph and redeploy - self.networks = ['mainnet'] - - for n in self.networks: - self._collectors.extend([ - AppsCollector(self, n), - CastsCollector(self, n), - MiniMeTokensCollector(self, n), - ReposCollector(self, n), - TransactionsCollector(self, n), - TokenHoldersCollector(self, n), - VotesCollector(self, n) - ]) - oc = OrganizationsCollector(self, n) - bc = BalancesCollector(self, oc, n) - self._collectors += [oc, bc] - - self._collectors.append(CCPricesCollector(self)) - - @property - def collectors(self) -> List[Collector]: - return self._collectors \ No newline at end of file diff --git a/cache_scripts/argparser.py b/cache_scripts/argparser.py deleted file mode 100644 index 00cb696d..00000000 --- a/cache_scripts/argparser.py +++ /dev/null @@ -1,105 +0,0 @@ -from argparse import ArgumentParser, BooleanOptionalAction, SUPPRESS -from typing import List - -from datetime import datetime -import pathlib -import os - -from . import config - -class CacheScriptsArgParser(ArgumentParser): - def __init__(self, available_platforms: List[str], available_networks: List[str]): - super().__init__(description="Main script to populate dao-analyzer cache") - - self.add_argument( - "-V", "--version", - action='store_true', - dest='display_version', - help="Displays the version and exits" - ) - self.add_argument( - "-p", "--platforms", - choices=available_platforms, - nargs='*', - type=str, - default=available_platforms, - help="The platforms to update. Every platform is updated by default." - ) - self.add_argument( - "--ignore-errors", - default=True, - action=BooleanOptionalAction, - help="Whether to ignore errors and continue") - self.add_argument( - "-d", "--debug", - action='store_true', default=False, - help="Shows debug info" - ) - self.add_argument( - "-f", "--force", - action='store_true', default=False, - help="Removes the cache before updating" - ) - self.add_argument( - "-F", "--delete-force", - action="store_true", default=False, - help="Removes the datawarehouse folder before doing anything" - ) - self.add_argument( - "--skip-daohaus-names", - action="store_true", default=False, - help="Skips the step of getting Daohaus Moloch's names, which takes some time" - ) - self.add_argument( - "--skip-token-balances", - action="store_true", default=False, - help="Skips the step of getting every DAO token balances, which takes some time" - ) - self.add_argument( - "-n", "--networks", - nargs="+", - required=False, - choices=available_networks, - default=available_networks, - help="Networks to update. Every network is updated by default" - ) - self.add_argument( - "-c", "--collectors", - nargs="+", - required=False, - help="Collectors to run. For example: aragon/casts" - ) - self.add_argument( - "--block-datetime", - required=False, - type=datetime.fromisoformat, - help="Get data up to a block datetime (input in ISO format)" - ) - self.add_argument( - "-D", "--datawarehouse", - help="Specifies the destination folder of the datawarehouse", - required=False, - type=pathlib.Path, - default=config.DEFAULT_DATAWAREHOUSE - ) - self.add_argument( - "--cc-api-key", - help="Set the CryptoCompare API key (overrides environment variable)", - required=False, - type=str, - default=os.getenv('DAOA_CC_API_KEY') - ) - self.add_argument( - "--only-updatable", - help=SUPPRESS, # "Run only updatable collectors (only for testing)", - action='store_true', - required=False, - default=False - ) - self.add_argument( - "--daostack-all", - help="Obtain all DAOs in DAOstack, not only registered ones", - action='store_true', - required=False, - default=False, - ) \ No newline at end of file diff --git a/cache_scripts/common/__init__.py b/cache_scripts/common/__init__.py deleted file mode 100644 index 7da71af6..00000000 --- a/cache_scripts/common/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" - Descp: common pieces of the cache_scripts entry point - - Created on: 16-feb-2022 - - Copyright 2022 David Davó - -""" - -from .common import Collector, Runner, ENDPOINTS - -__all__ = [ - 'Collector', - 'Runner', - 'ENDPOINTS', -] \ No newline at end of file diff --git a/cache_scripts/common/api_requester.py b/cache_scripts/common/api_requester.py deleted file mode 100644 index cdfd51b5..00000000 --- a/cache_scripts/common/api_requester.py +++ /dev/null @@ -1,240 +0,0 @@ -""" - Descp: Functions to fetch from endpoint given queries. - - Created on: 10-jul-2020 - - Copyright 2020-2021 Youssef 'FRYoussef' El Faqir El Rhazoui - -""" - -from gql import Client, gql -from gql.dsl import DSLField, DSLQuery, DSLSchema, DSLType, dsl_gql -from gql.transport.requests import RequestsHTTPTransport -import re -import requests -from functools import partial - -import logging -import sys -from tqdm import tqdm -from typing import Dict, List, Union, Iterable - -class GQLQueryException(Exception): - def __init__(self, errors, msg="Errors in GraphQL Query"): - super().__init__(msg) - self.__errors = errors - - def errorsString(self) -> str: - return '\n'.join([f"> {e['message']}" for e in self.__errors]) - - def __str__(self): - return super().__str__() + ":\n" + self.errorsString() - -class IndexProgressBar(tqdm): - def __init__(self, total=0xffff): - super().__init__(delay=1, total=total, file=sys.stdout, desc="Requesting", - bar_format="{l_bar}{bar}[{elapsed}<{remaining}{postfix}]", dynamic_ncols=True, - postfix={"requested":0}) - self.requested = 0 - - def progress(self, last_index: str, new_items: int): - self.requested += new_items - self.set_postfix(refresh=False, requested=self.requested) - - if not last_index: - last_index = "0x0" - - match = re.search(r"0x[\da-fA-F]+", last_index) - if match: - self.update(int(match[0][:6], 0) - self.n) - else: - raise ValueError(f"{last_index} doesn't contain any hex values") - - def complete(self): - self.update(self.total - self.n) - -class RequestProgressSpinner: - def __init__(self): - self.prev_lastindex = "" - self.toFinish = False - self.total = 0 - - def progress(self, last_index: str, new_items: int): - filler = " " * max(0, len(self.prev_lastindex) - len(last_index)) - self.total += new_items - print(f"Requesting... Total: {self.total:5d}. Requested until {last_index}"+filler, end='\r', flush=True) - self.prev_lastindex = last_index - self.toFinish = True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - # To remove the last \r - if self.toFinish: - print() - - def complete(self): - pass - -class GQLRequester: - ELEMS_PER_CHUNK: int = 1000 - - def __init__(self, endpoint: str, pbar_enabled: bool=True, introspection=True) -> None: - self.__transport = RequestsHTTPTransport(endpoint) - self.__client: Client = Client(transport=self.__transport, fetch_schema_from_transport=introspection) - self.pbar = IndexProgressBar if pbar_enabled else RequestProgressSpinner - - logging.debug(f"Invoked ApiRequester with endpoint: {endpoint}") - - def get_schema(self) -> DSLSchema: - with self.__client: - assert(self.__client.schema is not None) - return DSLSchema(self.__client.schema) - - def request(self, query: DSLQuery | DSLField | str) -> Dict: - """ - Requests data from endpoint. - """ - if isinstance(query, DSLField): - query = DSLQuery(query) - - logging.debug(f"Requesting: {query}") - - if isinstance(query, DSLQuery): - result = self.__client.execute(dsl_gql(query)) - else: - result = self.__client.execute(gql(query)) - - if "errors" in result: - raise GQLQueryException(result["errors"]) - - return result - - def request_single(self, q: DSLQuery | DSLField | str) -> Dict: - result = self.request(q) - if result and len(result.values()) == 1: - return next(iter(result.values())) - else: - raise - - def n_requests(self, query:DSLType, index='id', last_index: str = "", block_hash: str = None) -> List[Dict]: - """ - Requests all chunks from endpoint. - - Parameters: - * query: json to request - * index: dict key to use as index - * last_index: used to continue the request - * block_hash: make the request to that block hash - """ - elements: List[Dict] = list() - result = Dict - - # do-while structure - exit: bool = False - - with self.pbar() as pbar: - while not exit: - query_args = { - "where": {index+"_gt": last_index}, - "first": self.ELEMS_PER_CHUNK - } - if block_hash: - query_args["block"] = {"hash": block_hash} - - result = self.request_single(query(**query_args)) - elements.extend(result) - - # if return data (result) has no elements, we have finished - if result: - assert(last_index != result[-1][index]) - pbar.progress(last_index=last_index, new_items=len(result)) - last_index = result[-1][index] - else: - pbar.complete() - exit = True - - return elements - -class CryptoCompareQueryException(Exception): - def __init__(self, errors, msg="Errors in CryptoCompare Query"): - super().__init__(msg) - self.__errors = errors - - def errorsString(self) -> str: - return self.__errors - - def __str__(self): - return super().__str__() + ":\n" + self.errorsString() - -class CryptoCompareRequester: - BASEURL = 'https://min-api.cryptocompare.com/data/' - - def __init__(self, api_key: str = None, pbar_enabled: bool = True): - self.logger = logging.getLogger('ccrequester') - self.pbar = partial(tqdm, delay=1, file=sys.stdout, desc="Requesting", - dynamic_ncols=True) - - if not api_key: - logging.warning(f'Invalid api key: {api_key}') - api_key = "" - - self.api_key = api_key - - def _build_headers(self) -> Dict[str, str]: - return { - 'Authorization': 'Apikey ' + self.api_key - } - - def _request(self, url: str, params=None): - if params is None: - params = {} - - params['extraParams'] = 'dao-analyzer' - - r = requests.get(url, params=params, headers=self._build_headers()) - self.logger.debug(f'Response status: {r.status_code}, ok: {r.ok}, content: {r.content}') - - # There are two kinds of requests - # - "Complex" ones which have Response, Message, Type, etc fields - # - "Simple" ones where the data is the response per se - if r.ok: - j = r.json() - if 'Data' not in j: - return j - if "HasWarning" in j and j["HasWarning"]: - logging.warning("Warning in query", r.url, ":", j["Message"]) - if j["Type"] == 100: - return j['Data'] - - raise CryptoCompareQueryException(j["Message"]) - - def get_available_coin_list(self): - return self._request(self.BASEURL + 'blockchain/list').values() - - def get_symbols_price(self, fsyms: Union[str, Iterable[str]], tsyms: Iterable[str] = ['USD', 'EUR', 'ETH'], relaxedValidation=False): - if isinstance(fsyms, str): - fsyms = [fsyms] - elif not isinstance(fsyms, Iterable): - raise TypeError("fsyms must be an Iterable[str] or str") - else: - fsyms = list(fsyms) - - mi = 25 # max items - # Every partition needs to have at least a known value. Else it could fail. - # That's why we always include BTC - partitions = [fsyms[i:i+mi]+['BTC'] for i in range(0, len(fsyms), mi)] - - params = { - 'tsyms': ','.join(tsyms), - 'relaxedValidation': str(relaxedValidation).lower() - } - - ret = {} - for p in self.pbar(partitions): - params['fsyms'] = ','.join(p) - - ret.update(self._request(self.BASEURL + 'pricemulti', params=params)) - - return ret diff --git a/cache_scripts/common/blockscout.py b/cache_scripts/common/blockscout.py deleted file mode 100644 index 24ee4d59..00000000 --- a/cache_scripts/common/blockscout.py +++ /dev/null @@ -1,126 +0,0 @@ -import pandas as pd -import requests -import logging -from functools import partial -from tqdm import tqdm -from time import sleep -from typing import Union - -import numpy as np - -from ..metadata import Block -from .. import config - -from . import ENDPOINTS -from .common import NetworkCollector, solve_decimals -from .cryptocompare import cc_postprocessor -from .graphql import GraphQLCollector - -MSG_NO_TOKENS_FOUND = "No tokens found" - -class BlockscoutBallancesCollector(NetworkCollector): - ERR_SLEEP = 60 - - def __init__(self, runner, base: GraphQLCollector, name: str='tokenBalances', network: str='mainnet', addr_key: str='id'): - """ Initializes a ballance collector that uses blockscout - - Parameters: - name (str): The name of the collector (and filename) - runner (Runner): The runner in which it's based - network (str): The network to run on - base (GraphQLCollector): The DAOs/organizations collector to get the DAO list from - index_key (str): The key to use to access the address in the base array - """ - super().__init__(name, runner, network) - self.base = base - self.addr_key = addr_key - - def verify(self) -> bool: - if config.skip_token_balances: - logging.warning('Skipping token balances because --skip-token-balances flag was set') - return False - else: - return super().verify() - - @property - def endpoint(self) -> str: - return ENDPOINTS[self.network]['blockscout'] - - def _get_from_address(self, addr: str, retry: int = 0, maxretries: int = 3, block: Union[int, Block] = None, ignore_errors=False) -> pd.DataFrame: # noqa: C901 - if retry >= maxretries: - raise ValueError(f"Too many retries {retry}/{maxretries}") - - blockn = block - if blockn is None: - blockn = 'latest' - elif isinstance(blockn, Block): - blockn = blockn.number - - r = requests.get(ENDPOINTS[self.network]['blockscout'], params={ - 'module': 'account', - 'action': 'tokenlist', - 'block': blockn, - 'address': addr - }) - - if (r.ok): - j = r.json() - if j['status'] == str(1) and j['message'] == 'OK': - df = pd.DataFrame(j['result']) - # Only ERC-20 tokens (not NFTs or others) - df = df[df.type == 'ERC-20'] - df = df.replace(r'^\s*$', np.nan, regex=True) - df = df.dropna() - - # Calculate decimals - solve_decimals(df) - - # Add index - df['address'] = addr - df['network'] = self.network - df['id'] = 'token-' + df['contractAddress'] + '-org-' + df['address'] - return df - elif j['message'] == MSG_NO_TOKENS_FOUND: - return pd.DataFrame() - else: - logging.warning(f"Status {j['status']}, message: {j['message']}") - return pd.DataFrame() - elif r.status_code == 429: # Too many requests - logging.warning(f"Too many requests, sleep and retry {retry}/{maxretries} time") - sleep(self.ERR_SLEEP) - return self._get_from_address(addr, retry=retry+1, maxretries=maxretries) - elif r.status_code == 503: - logging.warning(f"Service unavailable, sleep and retry {retry}/{maxretries} time") - sleep(self.ERR_SLEEP) - return self._get_from_address(addr, retry=retry+1, maxretries=maxretries) - elif r.status_code == 504: # Gateway Time-out (Response too large) - logging.warning(f"Requests returned Gateway Time-out, ignoring response for addr {addr}") - return pd.DataFrame() - else: - logging.error(f'Requests failed for address "{addr}" with status code {r.status_code}: {r.reason}') - if ignore_errors: - return pd.DataFrame() - else: - raise ValueError(f"Requests failed for address {addr[:12]}... with status code {r.status_code}: {r.reason}") - - def run(self, force=False, block: Block = None, prev_block: Block = None, **kwargs): - # For each of the DAOs in the df, get the token balance - addresses = self.base.df[self.addr_key].drop_duplicates() - - if addresses.empty: - logging.warning("No addresses returned, not running blockscout collector") - return - - ptqdm = partial(tqdm, delay=1, desc="Requesting token balances", - unit='req', dynamic_ncols=True) - toApply = partial(self._get_from_address, block=block, ignore_errors=True) - df = pd.concat(map(toApply, ptqdm(addresses)), ignore_index=True) - - df = cc_postprocessor(df) - df = df.rename(columns={ - 'name': 'tokenName', - # Replace only if addr_key is not 'id' - 'address': self.addr_key if self.addr_key != 'id' else 'address' - }) - - self._update_data(df, force) \ No newline at end of file diff --git a/cache_scripts/common/common.py b/cache_scripts/common/common.py deleted file mode 100644 index 83954e4e..00000000 --- a/cache_scripts/common/common.py +++ /dev/null @@ -1,268 +0,0 @@ -from abc import ABC, abstractmethod -from pathlib import Path -from typing import List, Dict, Iterable -import logging -import sys -import json -from datetime import datetime, timezone -import traceback -import pkgutil - -from tenacity import retry, retry_if_exception_type, wait_exponential, stop_after_attempt -import pandas as pd -from tqdm import tqdm -from gql.transport.exceptions import TransportQueryError - -from .api_requester import GQLRequester -from ..metadata import RunnerMetadata, Block -from .. import config -from ... import cache_scripts - -# To be able to obtain endpoints.json -ENDPOINTS: Dict = json.loads(pkgutil.get_data(cache_scripts.__name__, 'endpoints.json')) - -def solve_decimals(df: pd.DataFrame) -> pd.DataFrame: - """ Adds the balanceFloat column to the dataframe - - This column is a precalculated value of tokenBalance / 10 ** tokenDecimals as float - """ - dkey, bkey, fkey = 'decimals', 'balance', 'balanceFloat' - - df[dkey] = df[dkey].astype(int) - df[fkey] = df[bkey].astype(float) / 10 ** df[dkey] - - return df - -class Collector(ABC): - INDEX = ['network', 'id'] - - def __init__(self, name:str, runner): - self.name: str = name - self.runner = runner - - @property - def data_path(self) -> Path: - return self.runner.basedir / (self.name + '.arr') - - @property - def long_name(self) -> str: - return f"{self.runner.name}/{self.name}" - - @property - def collectorid(self) -> str: - return self.long_name - - @property - def df(self) -> pd.DataFrame: - return pd.DataFrame() - - def verify(self) -> bool: - """ - Checks if the Collector is in a valid state. This check is run for every - collector before starting to get data. Can be ignored with --no-verify - """ - return True - - def _update_data(self, df: pd.DataFrame, force: bool = False) -> pd.DataFrame: - """ Updates the dataframe in `self.data_path` with the new data. - """ - if df.empty: - logging.warning("Empty dataframe, not updating file") - return - - if not self.data_path.is_file(): - df.reset_index(drop=True).to_feather(self.data_path) - return - - prev_df: pd.DataFrame = pd.read_feather(self.data_path) - - # If force is selected, we delete the ones of the same network only - if force: - prev_df = prev_df[prev_df["network"] != self.network] - - prev_df = prev_df.set_index(self.INDEX, verify_integrity=True, drop=True) - df = df.set_index(self.INDEX, verify_integrity=True, drop=True) - - # Updating data - combined = df.combine_first(prev_df).reset_index() - combined.to_feather(self.data_path) - return combined - - @abstractmethod - def run(self, force=False, **kwargs) -> None: - return - -class NetworkCollector(Collector): - """ Collector runnable in a specific network and to a block number """ - def __init__(self, name: str, runner, network: str='mainnet'): - super().__init__(name, runner) - self.network = network - - @property - def collectorid(self) -> str: - return '-'.join([super().collectorid, self.network]) - -class UpdatableCollector(Collector): # Flag class - pass - -class Runner(ABC): - def __init__(self, dw: Path = Path()): - self.__dw: Path = dw - - def set_dw(self, dw) -> Path: - self.__dw = dw - - @property - def basedir(self) -> Path: - return self.__dw / self.name - - @property - def collectors(self) -> List[Collector]: - return [] - - def run(self, **kwargs): - raise NotImplementedError - -class NetworkRunner(Runner, ABC): - def __init__(self, dw = None): - super().__init__(dw) - self.networks = {n for n,v in ENDPOINTS.items() if self.name in v and not n.startswith('_')} - - def filterCollectors(self, - networks: Iterable[str] = [], - names: Iterable[str] = [], - long_names: Iterable[str] = [] - ) -> Iterable[Collector]: - result = self.collectors - - if config.only_updatable: - result = filter(lambda c: isinstance(c, UpdatableCollector), result) - - # networks ^ (names v long_names) - if networks: - # GraphQLCollector => c.network in networks - # a => b : not(a) or b - result = filter(lambda c: not isinstance(c, NetworkCollector) or c.network in networks, result) - - if names or long_names: - result = (c for c in result if c.name in names or c.long_name in long_names) - - return result - - def filterCollector(self, - collector_id: str = None, - network: str = None, - name: str = None, - long_name: str = None, - ) -> Collector: - if collector_id: - return next((c for c in self.collectors if c.collectorid == collector_id), None) - - return next(self.filterCollectors( - networks=[network] if network else [], - names=[name] if name else [], - long_names=[long_name] if long_name else [] - ), None) - - @staticmethod - @retry(retry=retry_if_exception_type(TransportQueryError), wait=wait_exponential(max=10), stop=stop_after_attempt(3)) - def validated_block(network: str, prev_block: Block = None) -> Block: - requester = GQLRequester(ENDPOINTS[network]["_blocks"]) - ds = requester.get_schema() - - number_gte = prev_block.number if prev_block else 0 - - args = { - "first": 1, - "skip": config.SKIP_INVALID_BLOCKS, - "orderBy": "number", - "orderDirection": "desc", - "where": { - "number_gte": number_gte - } - } - - if config.block_datetime: - del args["skip"] - del args["where"]["number_gte"] - args["where"]["timestamp_lte"] = int(config.block_datetime.timestamp()) - - response = requester.request(ds.Query.blocks(**args).select( - ds.Block.id, - ds.Block.number, - ds.Block.timestamp - ))["blocks"] - - if len(response) == 0: - logging.warning(f"Blocks query returned no response with args {args}") - return prev_block - - return Block(response[0]) - - @staticmethod - def _verifyCollectors(tocheck: Iterable[Collector]): - verified = [] - for c in tqdm(list(tocheck), desc="Verifying"): - try: - if c.verify(): - verified.append(c) - else: - print(f"Verified returned false for {c.collectorid} (view logs the see why)") - except Exception: - print(f"Won't run {c.collectorid}", file=sys.stderr) - traceback.print_exc() - return verified - - def run(self, networks: List[str] = [], force=False, collectors=None): - self.basedir.mkdir(parents=True, exist_ok=True) - - print("Verifying collectors") - verified = self._verifyCollectors(self.filterCollectors( - networks=networks, - long_names=collectors - )) - if not verified: - # Nothing to do - return - - with RunnerMetadata(self) as metadata: - print(f'--- Updating {self.name} datawarehouse ---') - blocks: dict[str, Block] = {} - for c in verified: - try: - if isinstance(c, NetworkCollector): - if c.network not in blocks: - # Getting a block more recent than the one in the metadata (just to narrow down the search) - print("Requesting a block number...", end='\r') - blocks[c.network] = self.validated_block(c.network, None if force else metadata[c.collectorid].block) - print(f"Using block {blocks[c.network].id} for {c.network} (ts: {blocks[c.network].timestamp.isoformat()})") - - print(f"Running collector {c.long_name} ({c.network})") - olderBlock = blocks[c.network] < metadata[c.collectorid].block - if not force and olderBlock: - print("Warning: Forcing because requesting an older block") - logging.warning("Forcing because using an older block") - - # Running the collector - c.run( - force=force or olderBlock, - block=blocks[c.network], - prev_block=metadata[c.collectorid].block, - ) - - # Updating the block in the metadata - metadata[c.collectorid].block = blocks[c.network] - else: - print(f"Running collector {c.long_name}") - c.run( - force=force, - ) - - metadata[c.collectorid].last_update = datetime.now(timezone.utc) - except Exception as e: - metadata.errors[c.collectorid] = e.__str__() - if config.ignore_errors: - print(traceback.format_exc()) - else: - raise e - print(f'--- {self.name}\'s datawarehouse updated ---') \ No newline at end of file diff --git a/cache_scripts/common/cryptocompare.py b/cache_scripts/common/cryptocompare.py deleted file mode 100644 index 492b8e85..00000000 --- a/cache_scripts/common/cryptocompare.py +++ /dev/null @@ -1,62 +0,0 @@ -import pandas as pd -import numpy as np - -from .api_requester import CryptoCompareRequester - -from .. import config -from .common import Collector, NetworkRunner - -import logging - -EMPTY_KEY_MSG = \ -"""\ -Empty CryptoCompare API key. You can obtain one from https://www.cryptocompare.com/cryptopian/api-keys -You can set the API key using --cc-api-key argument or the DAOA_CC_API_KEY env variable. -""" - -def cc_postprocessor(df: pd.DataFrame) -> pd.DataFrame: - ccrequester = CryptoCompareRequester(api_key=config.cc_api_key) - - tokenSymbols = df['symbol'].drop_duplicates() - - # TODO: Get only the ones with available symbols (relaxedValidation=False) - df_fiat = pd.DataFrame.from_dict(ccrequester.get_symbols_price(tokenSymbols, relaxedValidation=True), orient='index') - - def _apply_values(row): - if row['symbol'] in df_fiat.index: - row['usdValue'] = row['balanceFloat'] * df_fiat.loc[row['symbol'], 'USD'] - row['ethValue'] = row['balanceFloat'] * df_fiat.loc[row['symbol'], 'ETH'] - row['eurValue'] = row['balanceFloat'] * df_fiat.loc[row['symbol'], 'EUR'] - else: - row['usdValue'] = np.NaN - row['ethValue'] = np.NaN - row['eurValue'] = np.NaN - - return row - - df = df.apply(_apply_values, axis='columns') - - return df - -class CCPricesCollector(Collector): - def __init__(self, runner: NetworkRunner, name: str='tokenPrices'): - super().__init__(name, runner) - self.requester = CryptoCompareRequester(api_key=config.cc_api_key) - - def verify(self) -> bool: - if not self.requester.api_key: - logging.warning(EMPTY_KEY_MSG) - return False - - return super().verify() - - @property - def base(self): - return self.runner.filterCollector(name='tokenBalances') - - def run(self, force=False, block=None): - tokenSymbols = pd.read_feather(self.base.data_path, columns=['symbol']).drop_duplicates()['symbol'] - # TODO: Get only coins with available info (relaxedValidation=False) - - df = pd.DataFrame.from_dict(self.requester.get_symbols_price(tokenSymbols, relaxedValidation=True), orient='index') - df.reset_index().to_feather(self.data_path) \ No newline at end of file diff --git a/cache_scripts/common/graphql.py b/cache_scripts/common/graphql.py deleted file mode 100644 index 36d257fe..00000000 --- a/cache_scripts/common/graphql.py +++ /dev/null @@ -1,143 +0,0 @@ -import logging -from typing import Callable, List, Dict -from abc import ABC, abstractmethod - -import pandas as pd - -from gql.dsl import DSLField - -from .common import ENDPOINTS, NetworkCollector, NetworkRunner, Runner, UpdatableCollector -from .api_requester import GQLRequester -from ..metadata import Block - -def add_where(d, **kwargs): - """ - Adds the values specified in kwargs to the where inside d - Example: `**add_where(kwargs, deleted=False)` - """ - if "where" in d: - d["where"] |= kwargs - else: - d["where"] = kwargs - - return d - -def partial_query(q, w) -> DSLField: - def wrapper(**kwargs): - return q(**add_where(kwargs, **w)) - return wrapper - -def checkSubgraphHealth(endpoint: str, network: str = None): - subgraphName = '/'.join(endpoint.split('/')[-2:]) - - requester = GQLRequester(endpoint=ENDPOINTS['_theGraph']['index-node'], introspection=False) - q = f""" - {{ - indexingStatusForCurrentVersion(subgraphName: "{subgraphName}") {{ - health, - synced, - node, - chains {{ - network - }} - }} - }} - """ - - r = requester.request_single(q) - - if not r['synced']: - logging.info(f"Subgraph {endpoint} is not synced") - - if r['health'].lower() != 'healthy': - logging.error(f"Subgraph {endpoint} is not healthy") - return False - - subgraph_network = r['chains'][0]['network'] - if network and subgraph_network != network: - logging.error(f"Subgraph {endpoint} expected network {network} but got {subgraph_network}") - return False - - return True - -class GraphQLCollector(NetworkCollector, UpdatableCollector): - def __init__(self, name: str, runner: Runner, endpoint: str, result_key: str = None, index: str = None, network: str='mainnet', pbar_enabled: bool=True): - super().__init__(name, runner, network) - self.endpoint: str = endpoint - self.index = index if index else 'id' - self.result_key = result_key if result_key else name - self.postprocessors: Callable = [] - - self.requester: GQLRequester = GQLRequester(endpoint=self.endpoint, pbar_enabled=pbar_enabled) - - def postprocessor(self, f: Callable[[pd.DataFrame], pd.DataFrame]): - self.postprocessors.append(f) - return f - - @property - def schema(self): - return self.requester.get_schema() - - @abstractmethod - def query(self, **kwargs) -> DSLField: - raise NotImplementedError - - @property - def df(self) -> pd.DataFrame: - if not self.data_path.is_file(): - return pd.DataFrame() - - df = pd.read_feather(self.data_path) - if self.network: - df = df[df['network'] == self.network] - - return df - - def transform_to_df(self, data: List[Dict], skip_post: bool=False) -> pd.DataFrame: - df = pd.DataFrame.from_dict(pd.json_normalize(data)) - - # For compatibility reasons we change from . to snake case - def dotsToSnakeCase(str: str) -> str: - splitted = str.split('.') - return splitted[0] + ''.join(x[0].upper()+x[1:] for x in splitted[1:]) - - df = df.rename(columns=dotsToSnakeCase) - df['network'] = self.network - - if not skip_post: - for post in self.postprocessors: - logging.debug(f"Running postprocessor {post.__name__}") - df = post(df) - if df is None: - raise ValueError(f"The postprocessor {post.__name__} returned None") - - return df - - def verify(self) -> bool: - # Checking if the queryBuilder doesn't raise any errors - self.query() - - # Checking the health of the subgraph - return checkSubgraphHealth(self.endpoint) - - def query_cb(self, prev_block: Block = None): - if prev_block: - return partial_query(self.query, {'_change_block': {'number_gte': prev_block.number}}) - else: - return self.query - - def run(self, force=False, block: Block = None, prev_block: Block = None): - logging.info(f"Running GraphQLCollector with block: {block}, prev_block: {prev_block}") - if block is None: - block = Block() - if prev_block is None or force: - prev_block = Block() - - data = self.requester.n_requests(query=self.query_cb(prev_block), block_hash=block.id) - - # transform to df - df: pd.DataFrame = self.transform_to_df(data) - self._update_data(df, force) - -class GraphQLRunner(NetworkRunner, ABC): - pass \ No newline at end of file diff --git a/cache_scripts/config.py b/cache_scripts/config.py deleted file mode 100644 index 7e4756c4..00000000 --- a/cache_scripts/config.py +++ /dev/null @@ -1,28 +0,0 @@ -from pathlib import Path -import os - -from . import __version__ - -CACHE_SCRIPTS_VERSION = __version__ - -# https://letsexchange.io/blog/what-is-block-confirmation-on-ethereum-and-how-many-confirmations-are-required/ -# Number of blocks to skip to only consult confirmed blocks -SKIP_INVALID_BLOCKS = 250 -DEFAULT_DATAWAREHOUSE = Path(os.getenv('DAOA_DW_PATH', 'datawarehouse')) - -# LOGGING CONFIG -LOGGING_BACKUP_COUNT = os.getenv('DAOA_LOGGING_BACKUP_COUNT', 3) -LOGGING_MAX_MB = os.getenv('DAOA_LOGGING_MAX_MB', 100) - -__args = None - -def populate_args(args): - global __args - __args = args - - -def __getattr__(name): - """ - Called when no function has been defined. Defaults to search argsparser. - """ - return __args.__getattribute__(name) diff --git a/cache_scripts/daohaus/__init__.py b/cache_scripts/daohaus/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cache_scripts/daohaus/runner.py b/cache_scripts/daohaus/runner.py deleted file mode 100644 index 15baa497..00000000 --- a/cache_scripts/daohaus/runner.py +++ /dev/null @@ -1,222 +0,0 @@ -""" - Descp: Daohaus Runner and Collectors - - Created on: 13-nov-2021 - - Copyright 2021 David Davó - -""" -import requests -import requests_cache -from typing import List -from datetime import timedelta - -import pandas as pd -from tqdm import tqdm -from gql.dsl import DSLField - -from .. import config -from ..common.common import solve_decimals -from ..common.cryptocompare import cc_postprocessor - -from ..common import ENDPOINTS, Collector -from ..common.graphql import GraphQLCollector, GraphQLRunner, add_where - -DATA_ENDPOINT: str = "https://data.daohaus.club/dao/{id}" - -class MembersCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('members', runner, network=network, endpoint=ENDPOINTS[network]['daohaus']) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.members(**kwargs).select( - ds.Member.id, - ds.Member.createdAt, - ds.Member.molochAddress, - ds.Member.memberAddress, - ds.Member.shares, - ds.Member.loot, - ds.Member.exists, - ds.Member.didRagequit - ) - -class MolochesCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('moloches', runner, network=network, endpoint=ENDPOINTS[network]['daohaus_stats']) - - @self.postprocessor - def moloch_id(df: pd.DataFrame) -> pd.DataFrame: - df['molochAddress'] = df['id'] - return df - - @self.postprocessor - def moloch_names(df: pd.DataFrame) -> pd.DataFrame: - df = df.rename(columns={"title":"name"}) - - if config.skip_daohaus_names: - return df - - cached = requests_cache.CachedSession(self.data_path.parent / '.names_cache', - use_cache_dir=False, - expire_after=timedelta(days=30) - ) - tqdm.pandas(desc="Getting moloch names") - df["name"] = df.progress_apply(lambda x:self._request_moloch_name(cached, x['id']), axis=1) - - return df - - @staticmethod - def _request_moloch_name(req: requests.Session, moloch_id: str): - response = req.get(DATA_ENDPOINT.format(id=moloch_id)) - - o = response.json() - if isinstance(o, list) and o and "name" in o[0]: - return o[0]["name"] - else: - return None - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.moloches(**add_where(kwargs, deleted=False)).select( - ds.Moloch.id, - ds.Moloch.title, - ds.Moloch.version, - ds.Moloch.summoner, - ds.Moloch.summoningTime, - ds.Moloch.timestamp, - ds.Moloch.proposalCount, - ds.Moloch.memberCount, - ds.Moloch.voteCount, - ds.Moloch.rageQuitCount, - ds.Moloch.totalGas - ) - -class ProposalsCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('proposals', runner, network=network, endpoint=ENDPOINTS[network]["daohaus"]) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.proposals(**kwargs).select( - ds.Proposal.id, - ds.Proposal.createdAt, - ds.Proposal.proposalId, - ds.Proposal.molochAddress, - ds.Proposal.memberAddress, - ds.Proposal.proposer, - ds.Proposal.sponsor, - ds.Proposal.sharesRequested, - ds.Proposal.lootRequested, - ds.Proposal.tributeOffered, - ds.Proposal.paymentRequested, - ds.Proposal.yesVotes, - ds.Proposal.noVotes, - ds.Proposal.sponsored, - ds.Proposal.sponsoredAt, - ds.Proposal.processed, - ds.Proposal.processedAt, - ds.Proposal.didPass, - ds.Proposal.yesShares, - ds.Proposal.noShares - ) - -class RageQuitCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('rageQuits', runner, network=network, endpoint=ENDPOINTS[network]["daohaus"]) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.rageQuits(**kwargs).select( - ds.RageQuit.id, - ds.RageQuit.createdAt, - ds.RageQuit.molochAddress, - ds.RageQuit.memberAddress, - ds.RageQuit.shares, - ds.RageQuit.loot - ) - -class TokenBalancesCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('tokenBalances', runner, network=network, endpoint=ENDPOINTS[network]["daohaus"]) - - @self.postprocessor - def change_col_names(df: pd.DataFrame) -> pd.DataFrame: - return df.rename(columns={ - 'molochId': 'molochAddress', - 'tokenTokenAddress': 'tokenAddress', - 'tokenDecimals': 'decimals', - 'tokenBalance': 'balance', - 'tokenSymbol': 'symbol' - }) - - @self.postprocessor - def coalesce_bank_type(df: pd.DataFrame) -> pd.DataFrame: - bank_idx = ['guildBank', 'memberBank', 'ecrowBank'] - - df['bank'] = df[bank_idx].idxmax(1) - df['bank'] = df['bank'].str.lower() - df['bank'] = df['bank'].str.replace('bank', '') - df = df.drop(columns=bank_idx) - - return df - - self.postprocessors.append(solve_decimals) - self.postprocessors.append(cc_postprocessor) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.tokenBalances(**add_where(kwargs, guildBank=True, tokenBalance_gt=0)).select( - ds.TokenBalance.id, - ds.TokenBalance.moloch.select( - ds.Moloch.id - ), - ds.TokenBalance.token.select( - ds.Token.tokenAddress, - ds.Token.symbol, - ds.Token.decimals - ), - ds.TokenBalance.guildBank, - ds.TokenBalance.memberBank, - ds.TokenBalance.ecrowBank, - ds.TokenBalance.tokenBalance - ) - -class VoteCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('votes', runner, network=network, endpoint=ENDPOINTS[network]["daohaus"]) - - @self.postprocessor - def changeColumnNames(df: pd.DataFrame) -> pd.DataFrame: - return df.rename(columns={"proposalId":"proposalAddress"}) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.votes(**kwargs).select( - ds.Vote.id, - ds.Vote.createdAt, - ds.Vote.proposal.select(ds.Proposal.id), - ds.Vote.molochAddress, - ds.Vote.memberAddress, - ds.Vote.uintVote - ) - -class DaohausRunner(GraphQLRunner): - name: str = 'daohaus' - - def __init__(self): - super().__init__() - self._collectors: List[Collector] = [] - for n in self.networks: - self._collectors.extend([ - MembersCollector(self, n), - MolochesCollector(self, n), - ProposalsCollector(self, n), - RageQuitCollector(self, n), - TokenBalancesCollector(self, n), - VoteCollector(self, n) - ]) - - @property - def collectors(self) -> List[Collector]: - return self._collectors \ No newline at end of file diff --git a/cache_scripts/daostack/__init__.py b/cache_scripts/daostack/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cache_scripts/daostack/runner.py b/cache_scripts/daostack/runner.py deleted file mode 100644 index 4d38af24..00000000 --- a/cache_scripts/daostack/runner.py +++ /dev/null @@ -1,287 +0,0 @@ -""" - Descp: Daostack Runner and Collectors - - Created on: 15-nov-2021 - - Copyright 2021 David Davó - -""" -from typing import List, Callable - -import pandas as pd -from gql.dsl import DSLField - -from .. import config -from ..common.blockscout import BlockscoutBallancesCollector -from ..common.cryptocompare import CCPricesCollector - -from ..common import ENDPOINTS, Collector -from ..common.graphql import GraphQLCollector, GraphQLRunner, add_where - -def _changeProposalColumnNames(df: pd.DataFrame) -> pd.DataFrame: - df = df.rename(columns={ - 'daoId': 'dao', - 'proposalId': 'proposal' - }) - return df - -def _remove_phantom_daos_wr(daoc: 'DaosCollector') -> Callable[[pd.DataFrame], pd.DataFrame]: - def _remove_phantom_daos(df: pd.DataFrame) -> pd.DataFrame: - if df.empty or daoc.df.empty: - return df - - return df[df.dao.isin(daoc.df.dao)] - - return _remove_phantom_daos - -class BalancesCollector(BlockscoutBallancesCollector): - def __init__(self, runner, base, network: str): - super().__init__(runner, base=base, network=network, addr_key='dao') - -class DaosCollector(GraphQLCollector): - def __init__(self, runner, network: str): - super().__init__('daos', runner, network=network, endpoint=ENDPOINTS[network]['daostack']) - - @self.postprocessor - def changeColumnNames(df: pd.DataFrame) -> pd.DataFrame: - df = df.rename(columns={ - 'nativeTokenId':'nativeToken', - 'nativeReputationId':'nativeReputation'}) - return df - - @self.postprocessor - def clone_id(df: pd.DataFrame) -> pd.DataFrame: - if df.empty: - return df - - df['dao'] = df['id'] - return df - - def query(self, **kwargs) -> DSLField: - ds = self.schema - - where = { 'register': 'registered' } - if config.daostack_all: - where.pop('register') - - return ds.Query.daos(**add_where(kwargs, **where)).select( - ds.DAO.id, - ds.DAO.name, - ds.DAO.register, - ds.DAO.nativeToken.select(ds.Token.id), - ds.DAO.nativeReputation.select(ds.Rep.id) - ) - -class ProposalsCollector(GraphQLCollector): - def __init__(self, runner, network: str, daoC: DaosCollector): - super().__init__('proposals', runner, network=network, endpoint=ENDPOINTS[network]['daostack']) - - @self.postprocessor - def changeColumnNames(df: pd.DataFrame) -> pd.DataFrame: - return df.rename(columns={ - 'daoId': 'dao', - }).rename(columns=self._stripGenesis) - - @self.postprocessor - def deleteColums(df: pd.DataFrame) -> pd.DataFrame: - return df.drop(columns=['competition'], errors='ignore') - - self.postprocessors.append(_remove_phantom_daos_wr(daoC)) - - @staticmethod - def _stripGenesis(s: str): - tostrip='genesisProtocolParams' - - if s and len(s) > 1 and s.startswith(tostrip): - s = s[len(tostrip):] - return s[0].lower() + s[1:] - else: - return s - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.proposals(**kwargs).select( - ds.Proposal.id, - ds.Proposal.proposer, - # enum ProposalState { None, ExpiredInQueue, Executed, Queued, PreBoosted, Boosted, QuietEndingPeriod} - ds.Proposal.stage, - ds.Proposal.createdAt, - ds.Proposal.preBoostedAt, - ds.Proposal.boostedAt, - ds.Proposal.quietEndingPeriodBeganAt, - ds.Proposal.closingAt, - ds.Proposal.preBoostedClosingAt, - ds.Proposal.executedAt, - ds.Proposal.totalRepWhenExecuted, - ds.Proposal.totalRepWhenCreated, - ds.Proposal.executionState, - ds.Proposal.expiresInQueueAt, - ds.Proposal.votesFor, - ds.Proposal.votesAgainst, - ds.Proposal.winningOutcome, - ds.Proposal.stakesFor, - ds.Proposal.stakesAgainst, - ds.Proposal.title, - ds.Proposal.description, - ds.Proposal.url, - ds.Proposal.confidence, - ds.Proposal.confidenceThreshold, - ds.Proposal.genesisProtocolParams.select( - ds.GenesisProtocolParam.queuedVoteRequiredPercentage, - ds.GenesisProtocolParam.queuedVotePeriodLimit, - ds.GenesisProtocolParam.boostedVotePeriodLimit, - # Used for Holografic Consensus threshold - ds.GenesisProtocolParam.thresholdConst, - ds.GenesisProtocolParam.minimumDaoBounty, - ds.GenesisProtocolParam.daoBountyConst, - ), - ds.Proposal.dao.select(ds.DAO.id), - ds.Proposal.competition.select(ds.CompetitionProposal.id) - ) - -class ReputationHoldersCollector(GraphQLCollector): - def __init__(self, runner, network: str, daoC: DaosCollector): - super().__init__('reputationHolders', runner, network=network, endpoint=ENDPOINTS[network]['daostack']) - self.postprocessor(_changeProposalColumnNames) - - self.postprocessors.append(_remove_phantom_daos_wr(daoC)) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.reputationHolders(**kwargs).select( - ds.ReputationHolder.id, - ds.ReputationHolder.contract, - ds.ReputationHolder.address, - ds.ReputationHolder.balance, - ds.ReputationHolder.createdAt, - ds.ReputationHolder.dao.select(ds.DAO.id) - ) - -class StakesCollector(GraphQLCollector): - def __init__(self, runner, network: str, daoC: DaosCollector): - super().__init__('stakes', runner, network=network, endpoint=ENDPOINTS[network]['daostack']) - self.postprocessor(_changeProposalColumnNames) - - self.postprocessors.append(_remove_phantom_daos_wr(daoC)) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.proposalStakes(**kwargs).select( - ds.ProposalStake.id, - ds.ProposalStake.createdAt, - ds.ProposalStake.staker, - ds.ProposalStake.outcome, - ds.ProposalStake.amount, - ds.ProposalStake.dao.select(ds.DAO.id), - ds.ProposalStake.proposal.select(ds.Proposal.id) - ) - -class TokenPricesCollector(CCPricesCollector): - pass - -class VotesCollector(GraphQLCollector): - def __init__(self, runner, network: str, daoC: DaosCollector): - super().__init__('votes', runner, network=network, endpoint=ENDPOINTS[network]['daostack']) - self.postprocessor(_changeProposalColumnNames) - - self.postprocessors.append(_remove_phantom_daos_wr(daoC)) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.proposalVotes(**kwargs).select( - ds.ProposalVote.id, - ds.ProposalVote.createdAt, - ds.ProposalVote.voter, - ds.ProposalVote.outcome, - ds.ProposalVote.reputation, - ds.ProposalVote.dao.select(ds.DAO.id), - ds.ProposalVote.proposal.select(ds.Proposal.id) - ) - -class CommonRepEventCollector(GraphQLCollector): - def __init__(self, name, runner, base, network: str): - super().__init__(name, runner, network=network, endpoint=ENDPOINTS[network]['daostack']) - self.base = base - - @self.postprocessor - def add_dao_id(df: pd.DataFrame) -> pd.DataFrame: - """ Using the contract info, appends the DAO id. - Used by ReputationMintsCollector and ReputationBurnsCollector - """ - # Skip postprocessor if empty - if df.empty: - return df - - l_index = ['network', 'contract'] - r_index = ['network', 'nativeReputation'] - - wants = ['dao'] - prev_cols = list(df.columns) - - # Add the DAO field to the dataframe - df = df.merge(self.base.df[r_index + wants], - how='left', - left_on=l_index, - right_on=r_index, - ) - - # Get only the dao field - df = df[prev_cols + wants] - - return df - - self.postprocessors.append(_remove_phantom_daos_wr(self.base)) - -class ReputationMintsCollector(CommonRepEventCollector): - def __init__(self, *args, **kwargs): - super().__init__('reputationMints', *args, **kwargs) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.reputationMints(**add_where(kwargs, amount_not=0)).select( - ds.ReputationMint.id, - # ds.ReputationMint.txHash, # Not used - ds.ReputationMint.contract, - ds.ReputationMint.address, - ds.ReputationMint.amount, - ds.ReputationMint.createdAt - ) - -class ReputationBurnsCollector(CommonRepEventCollector): - def __init__(self, *args, **kwargs): - super().__init__('reputationBurns', *args, **kwargs) - - def query(self, **kwargs) -> DSLField: - ds = self.schema - return ds.Query.reputationBurns(**add_where(kwargs, amount_not=0)).select( - ds.ReputationBurn.id, - # ds.ReputationBurn.txHash, # Not used - ds.ReputationBurn.contract, - ds.ReputationBurn.address, - ds.ReputationBurn.amount, - ds.ReputationBurn.createdAt - ) - -class DaostackRunner(GraphQLRunner): - name: str = 'daostack' - - def __init__(self): - super().__init__() - self._collectors: List[Collector] = [] - for n in self.networks: - dc = DaosCollector(self, n) - - self._collectors.extend([ - dc, - ProposalsCollector(self, n, dc), - ReputationHoldersCollector(self, n, dc), - StakesCollector(self, n, dc), - VotesCollector(self, n, dc), - BalancesCollector(self, dc, n), - ReputationMintsCollector(self, dc, n), - ReputationBurnsCollector(self, dc, n), - ]) - - @property - def collectors(self) -> List[Collector]: - return self._collectors diff --git a/cache_scripts/endpoints.json b/cache_scripts/endpoints.json deleted file mode 100644 index ae68c11c..00000000 --- a/cache_scripts/endpoints.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "mainnet": { - "_blocks": "https://api.thegraph.com/subgraphs/name/blocklytics/ethereum-blocks", - "blockscout": "https://blockscout.com/eth/mainnet/api", - "daostack": "https://api.thegraph.com/subgraphs/name/grasia/daostack", - "daohaus": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus", - "daohaus_stats": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-stats", - "aragon": "https://api.thegraph.com/subgraphs/name/aragon/aragon-mainnet", - "aragon_tokens": "https://api.thegraph.com/subgraphs/name/grasia/aragon-tokens-mainnet", - "aragon_voting": "https://api.thegraph.com/subgraphs/name/grasia/aragon-voting-mainnet", - "aragon_finance": "https://api.thegraph.com/subgraphs/name/grasia/aragon-finance-mainnet" - }, - "xdai": { - "_blocks": "https://api.thegraph.com/subgraphs/name/1hive/xdai-blocks", - "blockscout": "https://blockscout.com/xdai/mainnet/api", - "daostack": "https://api.thegraph.com/subgraphs/name/grasia/daostack-xdai", - "daohaus": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-xdai", - "daohaus_stats": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-stats-xdai", - "aragon": "https://api.thegraph.com/subgraphs/name/1hive/aragon-xdai", - "aragon_tokens": "https://api.thegraph.com/subgraphs/name/grasia/aragon-tokens-xdai", - "aragon_voting": "https://api.thegraph.com/subgraphs/name/grasia/aragon-voting-xdai", - "aragon_finance": "https://api.thegraph.com/subgraphs/name/grasia/aragon-finance-xdai" - }, - "polygon": { - "_blocks": "https://api.thegraph.com/subgraphs/name/grasia/matic-blocks", - "daohaus": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-matic", - "daohaus_stats": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-stats-matic" - }, - "arbitrum": { - "_blocks": "https://api.thegraph.com/subgraphs/name/grasia/arbitrum-blocks", - "daohaus": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-arbitrum", - "daohaus_stats": "https://api.thegraph.com/subgraphs/name/odyssy-automaton/daohaus-stats-arbitrum" - }, - "_theGraph": { - "index-node": "https://api.thegraph.com/index-node/graphql" - } -} diff --git a/cache_scripts/main.py b/cache_scripts/main.py deleted file mode 100755 index b20f1bd0..00000000 --- a/cache_scripts/main.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 -from typing import Dict - -from datetime import datetime -from pathlib import Path -import portalocker as pl -import os -import tempfile -import shutil -from sys import stderr - -import logging -from logging.handlers import RotatingFileHandler - -from .aragon.runner import AragonRunner -from .daohaus.runner import DaohausRunner -from .daostack.runner import DaostackRunner -from .common import Runner, ENDPOINTS -from .argparser import CacheScriptsArgParser -from . import config - -LOG_FILE_FORMAT = "[%(levelname)s] - %(asctime)s - %(name)s - : %(message)s in %(pathname)s:%(lineno)d" -LOG_STREAM_FORMAT = "%(levelname)s: %(message)s" - -AVAILABLE_PLATFORMS: Dict[str, Runner] = { - AragonRunner.name: AragonRunner, - DaohausRunner.name: DaohausRunner, - DaostackRunner.name: DaostackRunner -} - -# Get available networks from Runners -AVAILABLE_NETWORKS = {n for n in ENDPOINTS.keys() if not n.startswith('_')} - -def _call_platform(platform: str, datawarehouse: Path, force: bool=False, networks=None, collectors=None): - p = AVAILABLE_PLATFORMS[platform]() - p.set_dw(datawarehouse) - p.run(networks=networks, force=force, collectors=collectors) - -def _is_good_version(datawarehouse: Path) -> bool: - versionfile = datawarehouse / 'version.txt' - if not versionfile.is_file(): - return False - - with open(versionfile, 'r') as vf: - return vf.readline().startswith(config.CACHE_SCRIPTS_VERSION) - -def main_aux(datawarehouse: Path): - if config.delete_force or not _is_good_version(datawarehouse): - if not config.delete_force: - print(f"datawarehouse version is not version {config.CACHE_SCRIPTS_VERSION}, upgrading") - - # We skip the dotfiles like .lock - for p in datawarehouse.glob('[!.]*'): - if p.is_dir(): - shutil.rmtree(p) - else: - p.unlink() - - logger = logging.getLogger() - logger.propagate = True - filehandler = RotatingFileHandler( - filename=config.datawarehouse / 'cache_scripts.log', - maxBytes=config.LOGGING_MAX_MB * 2**20, - backupCount=config.LOGGING_BACKUP_COUNT, - ) - - filehandler.setFormatter(logging.Formatter(LOG_FILE_FORMAT)) - logger.addHandler(filehandler) - logger.setLevel(level=logging.DEBUG if config.debug else logging.INFO) - - # Log errors to STDERR - streamhandler = logging.StreamHandler(stderr) - streamhandler.setLevel(logging.WARNING if config.debug else logging.ERROR) - streamhandler.setFormatter(logging.Formatter(LOG_STREAM_FORMAT)) - logger.addHandler(streamhandler) - - # The default config is every platform - if not config.platforms: - config.platforms = AVAILABLE_PLATFORcache_scriptsMS.keys() - - # Now calling the platform and deleting if needed - for p in config.platforms: - _call_platform(p, datawarehouse, config.force, config.networks, config.collectors) - - # write date - data_date: str = str(datetime.now().isoformat()) - - if config.block_datetime: - data_date = config.block_datetime.isoformat() - - with open(datawarehouse / 'update_date.txt', 'w') as f: - f.write(data_date) - - with open(datawarehouse / 'version.txt', 'w') as f: - f.write(config.CACHE_SCRIPTS_VERSION) - -def main_lock(datawarehouse: Path): - datawarehouse.mkdir(exist_ok=True) - - # Lock for the datawarehouse (also used by the dash) - p_lock: Path = datawarehouse / '.lock' - - # Exclusive lock for the chache-scripts (no two cache-scripts running) - cs_lock: Path = datawarehouse / '.cs.lock' - - try: - with pl.Lock(cs_lock, 'w', timeout=1) as lock, \ - tempfile.TemporaryDirectory(prefix="datawarehouse_") as tmp_dw: - - # Writing pid and dir name to lock (debugging) - tmp_dw = Path(tmp_dw) - print(os.getpid(), file=lock) - print(tmp_dw, file=lock) - lock.flush() - - ignore = shutil.ignore_patterns('*.log', '.lock*') - - # We want to copy the dw, so we open it as readers - p_lock.touch(exist_ok=True) - with pl.Lock(p_lock, 'r', timeout=1, flags=pl.LOCK_SH | pl.LOCK_NB): - shutil.copytree(datawarehouse, tmp_dw, dirs_exist_ok=True, ignore=ignore) - - main_aux(datawarehouse=tmp_dw) - - with pl.Lock(p_lock, 'w', timeout=10): - shutil.copytree(tmp_dw, datawarehouse, dirs_exist_ok=True, ignore=ignore) - - # Removing pid from lock - lock.truncate(0) - except pl.LockException: - with open(cs_lock, 'r') as f: - pid = int(f.readline()) - - print(f"The cache_scripts are already being run with pid {pid}", file=stderr) - exit(1) - -def main(): - parser = CacheScriptsArgParser( - available_platforms=list(AVAILABLE_PLATFORMS.keys()), - available_networks=AVAILABLE_NETWORKS) - - config.populate_args(parser.parse_args()) - - if config.display_version: - print(config.CACHE_SCRIPTS_VERSION) - exit(0) - - main_lock(config.datawarehouse) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/cache_scripts/metadata.py b/cache_scripts/metadata.py deleted file mode 100644 index 22aeffca..00000000 --- a/cache_scripts/metadata.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - Descp: Metadata definitions - - Created on: 23-nov-2021 - - Copyright 2021 David Davó - -""" -from json.encoder import JSONEncoder -from typing import Dict -import json -from functools import total_ordering -from datetime import datetime, timezone - -from . import config - -@total_ordering -class Block: - def __init__(self, init=None): - self.number = 0 - self.id = None - self.timestamp = datetime.min - - if isinstance(init, dict): - self.number = int(init["number"]) if "number" in init else self.number - self.id = init["id"] if "id" in init else self.id - - if "timestamp" in init: - if init["timestamp"].isdigit(): - self.timestamp = datetime.fromtimestamp(int(init["timestamp"])) - else: - self.timestamp = datetime.fromisoformat(init["timestamp"]) - - if self.timestamp.tzinfo is None: - self.timestamp = self.timestamp.replace(tzinfo=timezone.utc) - - def __eq__(self, other): - if isinstance(other, Block): - # Both shouldn't be null - return not self.id and not other.id and self.id == other.id - else: - return False - - def __lt__(self, other): - return self.number and self.number < other.number - - def toDict(self): - return { - "number": self.number, - "id": self.id, - "timestamp": self.timestamp.isoformat() - } - - def __str__(self): - return self.toDict().__str__() - -class CollectorMetaData: - def __init__(self, c: str, d = None): - self.block = Block() - self._collector: str = c - self.last_update: datetime = datetime.now(timezone.utc) - - if d: - self.block = Block(d["block"]) if "block" in d else None - self.last_update = datetime.fromisoformat(d["last_update"]) - - if self.last_update.tzinfo is None: - self.last_update = self.last_update.replace(tzinfo=timezone.utc) - - def toDict(self): - return { - "block": self.block, - "last_update": self.last_update.isoformat() - } - - def __eq__(self, other): - if isinstance(other, CollectorMetaData): - return self._collector == other._collector and self.block == other.block - else: - return False - -class MetadataEncoder(JSONEncoder): - def default(self, o): - if hasattr(o, 'toDict'): - return o.toDict() - else: - return super().default(o) - -class RunnerMetadata: - def __init__(self, runner): - self._path = runner.basedir / 'metadata.json' - self.collectorMetaData: Dict[str, CollectorMetaData] = {} - self.errors: Dict[str, str] = {} - self._setPrev() - - def _setPrev(self): - self._prev = (self.errors.copy(), self.collectorMetaData.copy()) - - @property - def dirty(self) -> bool: - return (self.errors, self.collectorMetaData) != self._prev - - def __getitem__(self, key): - if key not in self.collectorMetaData: - self.collectorMetaData[key] = CollectorMetaData(key) - return self.collectorMetaData[key] - - def __setitem__(self, key, val): - self.collectorMetaData[key] = val - self.ifdump() - - def __delitem__(self, key): - del self.collectorMetaData[key] - - def load(self): - with open(self._path, 'r') as f: - j = json.load(f) - self.collectorMetaData = {k:CollectorMetaData(k,v) for k,v in j["metadata"].items()} - self.errors = j["errors"] - self._setPrev() - - def dump(self): - with open(self._path, 'w') as f: - json.dump({ - "metadata": self.collectorMetaData, - "errors": self.errors - }, f, - indent=2 if config.debug else None, - cls=MetadataEncoder) - - def ifdump(self): - if self.dirty: - self.dump() - - def __enter__(self): - if self._path.is_file(): - self.load() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.dump() diff --git a/setup.cfg b/setup.cfg index 1d0d8080..6626c885 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Scientific/Engineering :: Visualization Topic :: Sociology Typing :: Typed @@ -31,6 +32,7 @@ classifiers = # packages and package_dir in setup.py python_requires = >= 3.8 install_requires = + dao-scripts == 1.1.9 # Waiting for plotly/dash#2251 to be fixed dash >= 2.5.0, <2.6.0 dash-bootstrap-components >= 1.1.0 @@ -64,7 +66,6 @@ dao_analyzer_components = [options.entry_points] console_scripts = - daoa-cache-scripts = dao_analyzer.cache_scripts.main:main daoa-server = dao_analyzer.web.app:main [options.extras_require] @@ -118,7 +119,7 @@ max-complexity = 10 max-line-length = 100 [tox:tox] -envlist = py{38,39,310} +envlist = py{38,39,310,311} [testenv] deps = .[dev] @@ -138,3 +139,4 @@ python = 3.8: py38 3.9: py39 3.10: py310 + 3.11: py311 diff --git a/setup.py b/setup.py index b5e53966..25adacc7 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,6 @@ def version_dev(version): package_dir = { 'dao_analyzer.web': 'dao_analyzer/web', - 'dao_analyzer.cache_scripts': 'cache_scripts', # Unfortunately, this package can't be in the same namespace as the others # see https://github.com/plotly/dash/issues/2236 'dao_analyzer_components': 'dao_analyzer_components/dao_analyzer_components',