From 34b7d09c76661af517d4360a3e03dff598b6d22f Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 16 Nov 2023 13:26:26 -0800 Subject: [PATCH] Support for hyperdrive v0.3.0. Adds running fuzzbots container with crash reporting. (#96) * Splitting up competition to blocktime and testnet * Adding fuzzbot container * Binding crash report output dir to local machine from container * Bump images * Updating readme * Renaming elfpy to agent0 env var * Updating comment * Bumping agent0 version (with new repo name) * Bump FE tag --------- Co-authored-by: Ryan Goree --- .gitignore | 3 +- README.md | 20 +++++++-- docker-compose.anvil.yaml | 4 +- docker-compose.blocktime.yaml | 6 +++ docker-compose.data.yaml | 10 ++--- docker-compose.fuzz-bot.yaml | 12 ++++++ docker-compose.testnet.yaml | 3 -- env/env.images | 8 ++-- env/env.time | 2 +- setup_env.sh | 81 +++++++++++++++++++++++++---------- 10 files changed, 106 insertions(+), 43 deletions(-) create mode 100644 docker-compose.blocktime.yaml create mode 100644 docker-compose.fuzz-bot.yaml diff --git a/.gitignore b/.gitignore index 4c842e4..ec65770 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env -.DS_Store \ No newline at end of file +.DS_Store +.crash_report \ No newline at end of file diff --git a/README.md b/README.md index a636f41..9d00a7b 100644 --- a/README.md +++ b/README.md @@ -38,17 +38,22 @@ We use an environment file, `.env`, to choose which containers to build together To select an environment, run `sh setup_env.sh` with one or more of the following flags: - `--anvil` : Spin up an Anvil node, deploy Hyperdrive to it, and serve artifacts on an nginx server. -- `--data` : Runs the data framework, queries the chain, writes to postgres, and deploys the dashboard. +- `--blocktime` : Sets the anvil node to run in blocktime mode. +- `--testnet` : Uses the testnet hyperdrive image with restricted mint access. - `--frontend` : Build the frontend container. +- `--data` : Runs the data framework, queries the chain, writes to postgres, and deploys the dashboard. - `--postgres` : Launches a local postgres server for the data pipeline. - `--ports` : Expose docker images to your machine, as specified in `env/env.ports`. - `--fund-accounts` : Fund accounts from `/accounts/balances.json`. +- `--dynamic-rate` : Yield source will have a dynamic variable rate. +- `--fuzzbot` : Runs fuzzbots on the chain. We also support shortcuts for common combinations. The most inclusive tag used will take priority. -- `--all` : Fund accounts and enable all components: anvil, data, postgres, bots, frontend, and ports. -- `--competition`: Fund accounts and enable anvil, bots and ports. Use this for a trading competition deployment. -- `--develop` : Fund accounts and enable anvil, data, postgres, bots and ports. Suitable for local development work. +- `--all` : Fund accounts and enable all components: anvil, data, postgres, frontend, and ports. +- `--competition` : Fund accounts and enable anvil on block time with testnet image, data (without postgres), and ports. Use this for a trading competition deployment. +- `--develop` : Fund accounts and enable anvil, data, and ports. Suitable for local development work. +- `--fuzz` : Enable anvil on block time, data, and ports. Runs fuzzbots and outputs crash reports. See [Fuzz Agents](#fuzz-agents) for more details. You can also change the tags in `env/env.images` to modify which docker image you build from. @@ -67,3 +72,10 @@ See live logs with `docker logs CONTAINER_NAME -f`. ## Tearing down the app Run `docker compose down -v`. The `-v` ensures that storage volumes are deleted. + +## Fuzz agents + +We provide a method for launching [agents making random trades](https://github.com/delvtech/elf-simulations/blob/main/lib/agent0/bin/fuzz_bots.py) on Hyperdrive along with the rest of the +infrastructure by providing the `--fuzz` argument to `setup_env.sh`. These agents also support detailed +crash reports in the event that an error is detected, located in the `.crash_report` directory. By default, +the chain will be paused when a crash happens, freezing the anvil state for further analysis. diff --git a/docker-compose.anvil.yaml b/docker-compose.anvil.yaml index 6c02fc1..24ced46 100644 --- a/docker-compose.anvil.yaml +++ b/docker-compose.anvil.yaml @@ -21,7 +21,7 @@ services: timestamp_interval: image: curlimages/curl:latest profiles: - - "competition" + - "blocktime" command: | /bin/sh -c "sleep 5; curl -X POST -H \"Content-Type: application/json\" --data '{\"jsonrpc\":\"2.0\", \"method\":\"anvil_setBlockTimestampInterval\", \"params\":[${BLOCK_TIMESTAMP_INTERVAL}], \"id\":1}' ${RPC_URI}" @@ -40,7 +40,7 @@ services: - artifacts:/var/www/artifacts/ checkpoint-bot: - image: ${ELFPY_IMAGE} + image: ${AGENT0_IMAGE} command: | /bin/sh -c "sleep 1; python lib/agent0/bin/checkpoint_bot.py" env_file: diff --git a/docker-compose.blocktime.yaml b/docker-compose.blocktime.yaml new file mode 100644 index 0000000..1dd434c --- /dev/null +++ b/docker-compose.blocktime.yaml @@ -0,0 +1,6 @@ +version: "3.9" +services: + ethereum: + # This file overwrites docker-compose.anvil.yaml + command: | + 'anvil --load-state ./data --block-time ${BLOCK_TIME} --host 0.0.0.0 --code-size-limit 9999999999 --chain-id ${CHAIN_ID}' diff --git a/docker-compose.data.yaml b/docker-compose.data.yaml index 2c5e95e..0101295 100644 --- a/docker-compose.data.yaml +++ b/docker-compose.data.yaml @@ -2,7 +2,7 @@ version: "3.9" services: data: - image: ${ELFPY_IMAGE} + image: ${AGENT0_IMAGE} profiles: - "data" working_dir: /app/ @@ -11,7 +11,7 @@ services: - .env analysis: - image: ${ELFPY_IMAGE} + image: ${AGENT0_IMAGE} profiles: - "data" working_dir: /app/ @@ -20,7 +20,7 @@ services: - .env database-api-server: - image: ${ELFPY_IMAGE} + image: ${AGENT0_IMAGE} profiles: - "data" working_dir: /app/ @@ -29,7 +29,7 @@ services: - .env dashboard: - image: ${ELFPY_IMAGE} + image: ${AGENT0_IMAGE} profiles: - "data" working_dir: /app/ @@ -48,7 +48,7 @@ services: # Set initial username mappings init-usernames: - image: ${ELFPY_IMAGE} + image: ${AGENT0_IMAGE} profiles: - "data" working_dir: /app/ diff --git a/docker-compose.fuzz-bot.yaml b/docker-compose.fuzz-bot.yaml new file mode 100644 index 0000000..8324cb9 --- /dev/null +++ b/docker-compose.fuzz-bot.yaml @@ -0,0 +1,12 @@ +version: "3.9" +services: + fuzz: + image: ${AGENT0_IMAGE} + profiles: + - "fuzz" + working_dir: /app/ + command: /bin/sh -c "DEVELOP=true python lib/agent0/bin/fuzz_bots.py" + env_file: + - .env + volumes: + - .crash_report:/app/.crash_report diff --git a/docker-compose.testnet.yaml b/docker-compose.testnet.yaml index 5a94034..6f37839 100644 --- a/docker-compose.testnet.yaml +++ b/docker-compose.testnet.yaml @@ -3,6 +3,3 @@ services: ethereum: # This file overwrites docker-compose.anvil.yaml image: ${TESTNET_IMAGE} - # We run anvil with additional arguments for competition mode - command: | - 'anvil --load-state ./data --block-time 12 --host 0.0.0.0 --code-size-limit 9999999999 --chain-id ${CHAIN_ID}' diff --git a/env/env.images b/env/env.images index 7276e1e..9b74392 100644 --- a/env/env.images +++ b/env/env.images @@ -5,8 +5,8 @@ # - edge = The newest image regardless of stability # Anvil -DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:0.0.16 -TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:0.0.16 +DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:0.3.0 +TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:0.3.0 # Infra ARTIFACTS_IMAGE=ghcr.io/delvtech/infra/artifacts:0.0.6 @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.6 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.6 # Elfpy (bots) -ELFPY_IMAGE=ghcr.io/delvtech/elf-simulations/elf-simulations:0.8.1 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.9.0 # Frontend -FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-monorepo/hyperdrive-monorepo:0.8.0 +FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-monorepo/hyperdrive-monorepo:0.13.0 diff --git a/env/env.time b/env/env.time index 9232020..7a912a0 100644 --- a/env/env.time +++ b/env/env.time @@ -1,4 +1,4 @@ -# The chain time options that should be appended if the --competition option is used. +# The chain time options that should be appended if the --blocktime option is used. # How long in real time each block should be mined BLOCK_TIME=12 # How much time to advance per mined block diff --git a/setup_env.sh b/setup_env.sh index fd85c96..de626dd 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -7,28 +7,35 @@ if [[ $# -eq 0 ]] || [[ "$1" == "--help" ]]; then echo "Usage: ./setup_env.sh [flags]" echo "Flags:" + echo " --all : Fund accounts and enable all components: anvil, data, postgres, frontend, and ports." + echo " --competition : Fund accounts and enable anvil on block time with testnet image, data (without postgres), and ports. Use this for a trading competition deployment." + echo " --develop : Fund accounts and enable anvil, data, and ports. Suitable for local development work." + echo " --fuzz : Enable anvil on block time, data, and ports. Runs fuzzbots and outputs crash reports." echo " --anvil : Spin up an Anvil node, deploy Hyperdrive to it, and serve artifacts on an nginx server." - echo " --data : Runs the data framework, querying the chain and writing to postgres." + echo " --blocktime : Sets the anvil node to run in blocktime mode." + echo " --testnet : Uses the testnet hyperdrive image with restricted mint access." echo " --frontend : Build the frontend container." + echo " --data : Runs the data framework, querying the chain and writing to postgres." + echo " --postgres : Runs a postgres db container for storing data." echo " --ports : Expose docker images to your machine, as specified in env/env.ports." echo " --fund-accounts : Fund accounts from /accounts/balances.json." - echo " --all : Fund accounts and enable all components: anvil, bots, frontend, and ports." - echo " --competition : Fund accounts and enable anvil, bots and ports. Use this for a trading competition deployment." - echo " --develop : Fund accounts and enable anvil, bots and ports. Suitable for local development work." echo " --dynamic-rate : Yield source will have a dynamic variable rate." + echo " --fuzzbot : Runs fuzzbots on the chain." exit 0 fi # Parse all of the arguments ## Initialize variables ANVIL=false -PORTS=false +BLOCKTIME=false +TESTNET=false FRONTEND=false -POSTGRES=false DATA=false +POSTGRES=false +PORTS=false FUND_ACCOUNTS=false -COMPETITION=false DYNAMIC_RATE=false +FUZZBOT=false ## Loop through the arguments while [[ $# -gt 0 ]]; do @@ -36,18 +43,19 @@ while [[ $# -gt 0 ]]; do --all) ANVIL=true FRONTEND=true + DATA=true POSTGRES=true PORTS=true - DATA=true FUND_ACCOUNTS=true DYNAMIC_RATE=true ;; --competition) ANVIL=true - PORTS=true + BLOCKTIME=true + TESTNET=true DATA=true + PORTS=true FUND_ACCOUNTS=true - COMPETITION=true DYNAMIC_RATE=true ;; --develop) @@ -57,27 +65,45 @@ while [[ $# -gt 0 ]]; do DATA=true FUND_ACCOUNTS=true ;; + --fuzz) + ANVIL=true + BLOCKTIME=true + DATA=true + POSTGRES=true + PORTS=true + DYNAMIC_RATE=true + FUZZBOT=true + ;; --anvil) ANVIL=true ;; + --blocktime) + BLOCKTIME=true + ;; + --testnet) + TESTNET=true + ;; + --frontend) + FRONTEND=true + ;; --data) DATA=true ;; - --dynamic-rate) - DYNAMIC_RATE=true - ;; --postgres) POSTGRES=true ;; - --frontend) - FRONTEND=true - ;; --ports) PORTS=true ;; --fund-accounts) FUND_ACCOUNTS=true ;; + --dynamic-rate) + DYNAMIC_RATE=true + ;; + --fuzzbot) + FUZZBOT=true + ;; *) echo "Unknown flag: $1" ;; @@ -97,12 +123,17 @@ echo "# Environment for Docker compose" >>.env anvil_compose="docker-compose.anvil.yaml" data_compose="docker-compose.data.yaml" rate_bot_compose="docker-compose.rate-bot.yaml" +fuzz_bot_compose="docker-compose.fuzz-bot.yaml" frontend_compose="docker-compose.frontend.yaml" postgres_compose="docker-compose.postgres.yaml" testnet_compose="docker-compose.testnet.yaml" +blocktime_compose="docker-compose.blocktime.yaml" ports_compose="docker-compose.ports.yaml" -full_compose_files="COMPOSE_FILE=$anvil_compose:$data_compose:$frontend_compose:$postgres_compose:$rate_bot_compose:" -if $COMPETITION; then +full_compose_files="COMPOSE_FILE=$anvil_compose:$data_compose:$frontend_compose:$postgres_compose:$rate_bot_compose:$fuzz_bot_compose:" +if $BLOCKTIME; then + full_compose_files+="$blocktime_compose:" +fi +if $TESTNET; then full_compose_files+="$testnet_compose:" fi if $PORTS; then @@ -124,8 +155,9 @@ frontend_profile="frontend" postgres_profile="postgres" data_profile="data" fund_accounts_profile="fund-accounts" -competition_profile="competition" +blocktime_profile="blocktime" dynamic_rate_profile="dynamic-rate" +fuzz_bot_profile="fuzz" full_compose_profiles="COMPOSE_PROFILES=" if $FRONTEND; then full_compose_profiles+="$frontend_profile," @@ -139,12 +171,15 @@ fi if $FUND_ACCOUNTS; then full_compose_profiles+="$fund_accounts_profile," fi -if $COMPETITION; then - full_compose_profiles+="$competition_profile," +if $BLOCKTIME; then + full_compose_profiles+="$blocktime_profile," fi if $DYNAMIC_RATE; then full_compose_profiles+="$dynamic_rate_profile," fi +if $FUZZBOT; then + full_compose_profiles+="$fuzz_bot_profile," +fi # Check if "," is at the end of the string if [[ $full_compose_profiles == *"," ]]; then # Remove "," from the end of the string @@ -185,8 +220,8 @@ if $POSTGRES || $DATA; then cat env/env.postgres >> .env fi -# optionally add an env.time to .env file if --competition -if $COMPETITION; then +# optionally add an env.time to .env file if --blocktime +if $BLOCKTIME; then echo "" >>.env cat env/env.time >>.env fi