diff --git a/README-ci-failures.md b/README-ci-failures.md index 6e4f678c8b9..595b083f84b 100644 --- a/README-ci-failures.md +++ b/README-ci-failures.md @@ -46,7 +46,7 @@ In this case, you should: [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain) where they have been automatically uploaded. Look for your branch name and commit hash in the second column. You should find several images suffixed - with `-bullseye` and `-focal` (Debian versions). + with `-bullseye` or `-focal` (Debian versions). 3. For each such image, retrieve its full name and hash by hovering its link and clicking the `Copy full image name` tooltip that appears (or retrieve it on the image's page). diff --git a/README-dev.md b/README-dev.md index 9e0478ae2aa..61617d1c4bf 100644 --- a/README-dev.md +++ b/README-dev.md @@ -261,7 +261,7 @@ Debian package can be build locally by using below commands: ``` make build ``` -2. Build debian for mina-devet (example for ubuntu 18.04): +2. Build debian for mina-devnet (example for ubuntu 18.04): ``` ./scripts/debian/build.sh daemon_devnet ``` diff --git a/automation/services/watchdog/Dockerfile b/automation/services/watchdog/Dockerfile index 43e6e00f825..05e207350f6 100644 --- a/automation/services/watchdog/Dockerfile +++ b/automation/services/watchdog/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.16-slim-buster +FROM python:3.9.16-slim-bullseye ARG GCLOUDSDK_DOWNLOAD_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-296.0.1-linux-x86_64.tar.gz" ARG WATCHMAN_DEB_URL="http://ftp.us.debian.org/debian/pool/main/w/watchman/watchman_4.9.0-5+b1_amd64.deb" diff --git a/buildkite/Makefile b/buildkite/Makefile index eff56f84029..dceeec07ab4 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -32,4 +32,4 @@ check_dups: dump_pipelines check_names: dump_pipelines python3 scripts/dhall/checker.py --root "$(TMP)" names -all: check_syntax lint format check_deps check_dirty check_dups check_names \ No newline at end of file +all: check_syntax lint format check_deps check_dirty check_dups check_names diff --git a/buildkite/scripts/bench/install.sh b/buildkite/scripts/bench/install.sh index ea06e6ee02b..b9e0292dae0 100644 --- a/buildkite/scripts/bench/install.sh +++ b/buildkite/scripts/bench/install.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index 6808c61e9e3..241de3c15d6 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -10,7 +10,7 @@ function get_shas { function image_tag { SHA=$1 - IMAGE_TAG="$SHA-bullseye-berkeley" + IMAGE_TAG="$SHA-bullseye-devnet" } function download-docker { @@ -22,17 +22,25 @@ function download-docker { function try_docker_shas { DOCKER_SHAS=$1 GOT_DOCKER=0 - for sha in $DOCKER_SHAS; do - download-docker $sha - if [ $? -eq 0 ] ; then - GOT_DOCKER=1 - image_tag $sha - break - else - echo "No docker available for SHA=$sha" - fi + + set +e + download-docker $sha + + if [ $? -eq 0 ] ; then + GOT_DOCKER=1 + image_tag $sha + break + else + echo "No docker available for SHA=$sha" + fi + set -e done + + if [[ $GOT_DOCKER == 0 ]]; then + echo "docker cannot be found for given shas: $DOCKER_SHAS" + exit 1 + fi } function image_id { diff --git a/buildkite/scripts/connect-to-berkeley.sh b/buildkite/scripts/connect-to-berkeley.sh deleted file mode 100755 index 1b2aa7cfe82..00000000000 --- a/buildkite/scripts/connect-to-berkeley.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - -# Don't prompt for answers during apt-get install -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl - -TESTNET_VERSION_NAME="berkeley" -TESTNET_NAME="testworld-2-0" - -git config --global --add safe.directory /workdir - -source buildkite/scripts/export-git-env-vars.sh - -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" - -# Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: - -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key -# Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ - -# Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background - -# Attempt to connect to the GraphQL client every 30s for up to 12 minutes -num_status_retries=24 -for ((i=1;i<=$num_status_retries;i++)); do - sleep 30s - set +e - mina client status - status_exit_code=$? - set -e - if [ $status_exit_code -eq 0 ]; then - break - elif [ $i -eq $num_status_retries ]; then - exit $status_exit_code - fi -done - -# Check that the daemon has connected to peers and is still up after 2 mins -sleep 2m -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi - diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 3cfd7a4aa7b..cba27c93759 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -7,53 +7,46 @@ if [[ $# -ne 3 ]]; then exit 1 fi -TESTNET_VERSION_NAME="berkeley" TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl +sudo apt-get update +sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" +source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1 # Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: +sudo rm ~/.mina-config/.mina-lock ||: + +sudo mkdir -p /root/libp2p-keys/ -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ +sudo chmod -R 0700 /root/libp2p-keys/ +# Pre-generated random password for this quick test +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \ + TESTNET_NAME=$TESTNET_NAME \ + bash -c "mina daemon \ + --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \ + --libp2p-keypair \"/root/libp2p-keys/key\" \ + --seed &" # -background # Attempt to connect to the GraphQL client every 10s for up to 8 minutes num_status_retries=24 for ((i=1;i<=$num_status_retries;i++)); do sleep $WAIT_BETWEEN_POLLING_GRAPHQL set +e - mina client status + sudo mina client status status_exit_code=$? set -e if [ $status_exit_code -eq 0 ]; then @@ -63,13 +56,18 @@ for ((i=1;i<=$num_status_retries;i++)); do fi done -# Check that the daemon has connected to peers and is still up after 2 mins -sleep $WAIT_AFTER_FINAL_CHECK -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi +peer_retries=10 +for ((i=1;i<=$peer_retries;i++)); do + peer_count=$(sudo mina advanced get-peers | wc -l) + sudo mina client status + + if [ "$peer_count" -gt 0 ]; then + echo "Found some peers" + exit 0; + else + echo "No peers found" + fi + sleep $WAIT_AFTER_FINAL_CHECK +done +exit 1; \ No newline at end of file diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index 75848a7d23c..dfe1c425367 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -34,7 +34,7 @@ else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do case $i in - mina-berkeley*|mina-devnet|mina-mainnet) + mina-devnet*|mina-mainnet) # Downaload mina-logproc too source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER ;; diff --git a/buildkite/scripts/debian/promote.sh b/buildkite/scripts/debian/promote.sh index 491e77f562e..0e869728d9b 100755 --- a/buildkite/scripts/debian/promote.sh +++ b/buildkite/scripts/debian/promote.sh @@ -12,6 +12,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -s|--from-component) FROM_COMPONENT="$2"; shift;; -t|--to-component) TO_COMPONENT="$2"; shift;; --new-name) NEW_NAME="$2"; shift;; + --repo-key) REPO_KEY="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -28,6 +29,7 @@ function usage() { echo " -s, --from-component The source channel in which package currently resides" echo " -t, --to-component The target channel for package (unstable, alpha, beta etc.)" echo " -c, --codename The Debian codename (bullseye, focal etc.)" + echo " --repo-key The Debian target repo key" echo "" echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly" exit 1 @@ -40,6 +42,7 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi; if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi; if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi; +if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi; # check for AWS Creds if [ -z "$AWS_ACCESS_KEY_ID" ]; then @@ -65,5 +68,6 @@ else --new-version $NEW_VERSION \ --suite $FROM_COMPONENT \ --new-suite $TO_COMPONENT \ - --new-name $NEW_NAME + --new-name $NEW_NAME \ + --sign $REPO_KEY fi \ No newline at end of file diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index 8b96a974591..75ddad3c1da 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -15,8 +15,17 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi +if [ -z "${SIGN:-}" ]; then + SIGN_ARG="" +else + sudo chown -R opam ~/.gnupg/ + gpg --batch --yes --import /var/secrets/debian/key.gpg + SIGN_ARG="--sign $SIGN" +fi + source scripts/debian/publish.sh \ --names "${DOWNLOAD_FOLDER}/mina-*.deb" \ --release $MINA_DEB_RELEASE \ --version $MINA_DEB_VERSION \ - --codename $MINA_DEB_CODENAME + --codename $MINA_DEB_CODENAME \ + $SIGN_ARG diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh index 5c1d402e215..c803937d3d0 100755 --- a/buildkite/scripts/dump-mina-type-shapes.sh +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index fd8d48aca49..452eaec2acd 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then send_zkapp_transactions & fi -next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) +mina client status --json + +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} +sleep $sleep_time # Mina Rosetta Checks (spec construction data perf) echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" diff --git a/buildkite/scripts/run-test-executive-cloud.sh b/buildkite/scripts/run-test-executive-cloud.sh index 884b8222cbc..8a3df280401 100755 --- a/buildkite/scripts/run-test-executive-cloud.sh +++ b/buildkite/scripts/run-test-executive-cloud.sh @@ -2,7 +2,7 @@ set -o pipefail -x TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 09c37c2b206..8512b93df0f 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -26,7 +26,7 @@ cleanup TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then @@ -42,7 +42,7 @@ rm -f /etc/apt/sources.list.d/hashicorp.list apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 191c24e197b..c7d02b1a5fa 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index 713749f642f..5821d199a5e 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/single-node-tests.sh b/buildkite/scripts/single-node-tests.sh index 93b224c5ba8..dcd6752ae23 100755 --- a/buildkite/scripts/single-node-tests.sh +++ b/buildkite/scripts/single-node-tests.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1 +source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1 export MINA_LIBP2P_PASS="naughty blue worm" export MINA_PRIVKEY_PASS="naughty blue worm" diff --git a/buildkite/scripts/tests/ledger_test_apply.sh b/buildkite/scripts/tests/ledger_test_apply.sh index d72c20154b7..ce8304244a6 100755 --- a/buildkite/scripts/tests/ledger_test_apply.sh +++ b/buildkite/scripts/tests/ledger_test_apply.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1 +source buildkite/scripts/debian/install.sh "mina-devnet-instrumented" 1 echo "removing magic config files" sudo rm -f /var/lib/coda/config_* diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index fb1ea8c9d89..8af413c2436 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then exit 1 fi -TESTNET_NAME="${TESTNET_NAME:-berkeley}" +TESTNET_NAME="${TESTNET_NAME:-devnet}" # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 3d2431943b6..48427e6738d 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -10,6 +10,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall +let Network = ../../Constants/Network.dhall + let Profiles = ../../Constants/Profiles.dhall let Command = ../../Command/Base.dhall @@ -42,6 +44,7 @@ let Spec = , dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard , additionalDirtyWhen = [] : List SelectFiles.Type , yellowThreshold = 0.1 diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index ba4bba1b974..91bf6ee8bea 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -4,9 +4,9 @@ let Command = ./Base.dhall let Size = ./Size.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let RunInToolchain = ./RunInToolchain.dhall -let Cmd = ../Lib/Cmds.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type in { step = \(dependsOn : List Command.TaggedKey.Type) @@ -17,12 +17,9 @@ in { step = -> Command.build Command.Config::{ , commands = - [ Cmd.runInDocker - Cmd.Docker::{ - , image = (../Constants/ContainerImages.dhall).ubuntu2004 - } + RunInToolchain.runInToolchain + ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" - ] , label = "Connect to ${testnet}" , soft_fail = Some soft_fail , key = "connect-to-${testnet}" diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 1199d461332..917a50fb621 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -10,8 +10,6 @@ let Size = ./Size.dhall let Profiles = ../Constants/Profiles.dhall -let Artifacts = ../Constants/Artifacts.dhall - let BuildFlags = ../Constants/BuildFlags.dhall let Cmd = ../Lib/Cmds.dhall @@ -20,57 +18,81 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianVersions = ../Constants/DebianVersions.dhall + +let Network = ../Constants/Network.dhall + +let Artifacts = ../Constants/Artifacts.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type , network : Text - , service : Text + , service : Artifacts.Type , version : Text , branch : Text , repo : Text , no_cache : Bool - , deb_codename : Text + , deb_codename : DebianVersions.DebVersion , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , step_key : Text + , step_key_suffix : Text , if : Optional B/If } , default = { deps = [] : List Command.TaggedKey.Type - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" - , service = Artifacts.dockerName Artifacts.Type.Daemon + , service = Artifacts.Type.Daemon , branch = "\\\${BUILDKITE_BRANCH}" , repo = "\\\${BUILDKITE_REPO}" - , deb_codename = "bullseye" + , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test , no_cache = False - , step_key = "daemon-standard-docker-image" + , step_key_suffix = "-docker-image" , if = None B/If } } +let stepKey = + \(spec : ReleaseSpec.Type) + -> "${Artifacts.lowerName + spec.service}${Profiles.toLabelSegment + spec.deb_profile}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" + +let stepLabel = + \(spec : ReleaseSpec.Type) + -> "Docker: ${Artifacts.capitalName + spec.service} ${spec.network} ${DebianVersions.capitalName + spec.deb_codename} ${Profiles.toSuffixUppercase + spec.deb_profile} ${BuildFlags.toSuffixUppercase + spec.build_flags}" + let generateStep = \(spec : ReleaseSpec.Type) - -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + -> let exportMinaDebCmd = + "export MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.deb_codename}" let maybeCacheOption = if spec.no_cache then "--no-cache" else "" let buildDockerCmd = "./scripts/docker/build.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --network ${spec.network}" ++ " --version ${spec.version}" ++ " --branch ${spec.branch}" ++ " ${maybeCacheOption} " - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" @@ -81,10 +103,11 @@ let generateStep = let releaseDockerCmd = "./scripts/docker/release.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName @@ -121,12 +144,16 @@ let generateStep = in Command.build Command.Config::{ , commands = commands - , label = "Docker: ${spec.step_key}" - , key = spec.step_key + , label = "${stepLabel spec}" + , key = "${stepKey spec}" , target = Size.XLarge , docker_login = Some DockerLogin::{=} , depends_on = spec.deps , if = spec.if } -in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } +in { generateStep = generateStep + , ReleaseSpec = ReleaseSpec + , stepKey = stepKey + , stepLabel = stepLabel + } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 183826d81b4..9e9af392e52 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -30,14 +30,14 @@ let DebianVersions = ../Constants/DebianVersions.dhall let DebianRepo = ../Constants/DebianRepo.dhall -let Artifacts = ../Constants/Artifacts.dhall - let Profiles = ../Constants/Profiles.dhall let Toolchain = ../Constants/Toolchain.dhall let Network = ../Constants/Network.dhall +let Artifacts = ../Constants/Artifacts.dhall + let Spec = { Type = { codename : DebianVersions.DebVersion @@ -98,6 +98,17 @@ let pipeline "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" + let dockerSpec = + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = Artifacts.Type.Daemon + , network = network_name + , deb_codename = debVersion + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + } + in Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = [ S.everything ] @@ -155,20 +166,7 @@ let pipeline , key = "publish-hardfork-deb-pkg" , target = Size.Small } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - , deps = - [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Daemon - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" - } + , DockerImage.generateStep dockerSpec , Command.build Command.Config::{ , commands = @@ -182,10 +180,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -202,10 +197,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -214,28 +206,20 @@ let pipeline DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Archive + , service = Artifacts.Type.Archive , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_codename = debVersion , deb_profile = profile , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" } , DockerImage.generateStep DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Rosetta + , service = Artifacts.Type.Rosetta , network = network_name , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , deb_codename = debVersion } ] } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 0f7e43f1256..7f582d8b0b7 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -40,12 +40,13 @@ let MinaBuildSpec = , artifacts : List Artifacts.Type , debVersion : DebianVersions.DebVersion , profile : Profiles.Type - , networks : List Network.Type + , network : Network.Type , buildFlags : BuildFlags.Type , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type , tags : List PipelineTag.Type , channel : DebianChannel.Type + , debianRepo : DebianRepo.Type } , default = { prefix = "MinaArtifact" @@ -53,14 +54,39 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Berkeley ] + , network = Network.Type.Devnet , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] , channel = DebianChannel.Type.Unstable + , debianRepo = DebianRepo.Type.PackagesO1Test } } +let labelSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion} ${Network.capitalName + spec.network} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let nameSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion}${Network.capitalName + spec.network}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let keySuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}" + let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -77,22 +103,18 @@ let build_artifacts , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - , Network.foldMinaBuildMainnetEnv spec.networks + , Network.buildMainnetEnv spec.network ] # BuildFlags.buildEnvs spec.buildFlags ) "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts - spec.networks}" + spec.network}" # [ Cmd.run "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName spec.debVersion}" ] - , label = - "Build Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile} ${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , label = "Debian: Build ${labelSuffix spec}" , key = "build-deb-pkg" , target = Size.XLarge , retries = @@ -112,12 +134,15 @@ let publish_to_debian_repo = Toolchain.select spec.toolchainSelectMode spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] + ( [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName + spec.channel}" + ] + # DebianRepo.keyIdEnvList spec.debianRepo + ) "./buildkite/scripts/debian/publish.sh" , label = "Publish Mina for ${DebianVersions.capitalName @@ -139,123 +164,97 @@ let docker_step DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags step_dep_name in merge { Daemon = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Daemon - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Daemon + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , LogProc = [] : List DockerImage.ReleaseSpec.Type + , Toolchain = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Toolchain + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] + , ItnOrchestrator = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.ItnOrchestrator + , network = Network.lowerName Network.Type.Devnet + , deb_repo = DebianRepo.Type.Local + } + ] + , Leaderboard = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Leaderboard + } + ] , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-batch-txn" - , network = "berkeley" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.BatchTxn + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "batch-txn-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-archive" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.Archive + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Rosetta = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Rosetta - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "rosetta-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Rosetta + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , deb_repo = DebianRepo.Type.Local + } + ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-zkapp-test-transaction" + , service = Artifacts.Type.ZkappTestTransaction , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + , deb_codename = spec.debVersion } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-test-suite" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.FunctionalTestSuite + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , step_key = - "functional_test_suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - , network = "berkeley" } ] } @@ -298,11 +297,7 @@ let pipelineBuilder , spec = JobSpec::{ , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , name = "${spec.prefix}${nameSuffix spec}" , tags = spec.tags , mode = spec.mode } @@ -320,6 +315,7 @@ let onlyDebianPipeline ( DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags "build" @@ -335,4 +331,6 @@ in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline , publishToDebian = publish_to_debian_repo , MinaBuildSpec = MinaBuildSpec + , labelSuffix = labelSuffix + , keySuffix = keySuffix } diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall new file mode 100644 index 00000000000..4989004d394 --- /dev/null +++ b/buildkite/src/Command/PromotePackage.dhall @@ -0,0 +1,305 @@ +let B = ../External/Buildkite.dhall + +let B/If = B.definitions/commandStep/properties/if/Type + +let Prelude = ../External/Prelude.dhall + +let List/map = Prelude.List.map + +let Package = ../Constants/DebianPackage.dhall + +let Network = ../Constants/Network.dhall + +let PipelineMode = ../Pipeline/Mode.dhall + +let PipelineTag = ../Pipeline/Tag.dhall + +let Pipeline = ../Pipeline/Dsl.dhall + +let JobSpec = ../Pipeline/JobSpec.dhall + +let DebianChannel = ../Constants/DebianChannel.dhall + +let Profiles = ../Constants/Profiles.dhall + +let Artifact = ../Constants/Artifacts.dhall + +let DebianVersions = ../Constants/DebianVersions.dhall + +let Toolchain = ../Constants/Toolchain.dhall + +let Command = ./Base.dhall + +let Size = ./Size.dhall + +let Cmd = ../Lib/Cmds.dhall + +let PromoteDebianSpec = + { Type = + { deps : List Command.TaggedKey.Type + , package : Package.Type + , version : Text + , new_version : Text + , architecture : Text + , network : Network.Type + , codename : DebianVersions.DebVersion + , from_channel : DebianChannel.Type + , to_channel : DebianChannel.Type + , profile : Profiles.Type + , remove_profile_from_name : Bool + , step_key : Text + , if : Optional B/If + } + , default = + { deps = [] : List Command.TaggedKey.Type + , package = Package.Type.LogProc + , version = "" + , new_version = "" + , architecture = "amd64" + , network = Network.Type.Devnet + , codename = DebianVersions.DebVersion.Bullseye + , from_channel = DebianChannel.Type.Unstable + , to_channel = DebianChannel.Type.NightlyDevelop + , profile = Profiles.Type.Standard + , remove_profile_from_name = False + , step_key = "promote-debian-package" + , if = None B/If + } + } + +let PromoteDockerSpec = + { Type = + { deps : List Command.TaggedKey.Type + , name : Artifact.Type + , version : Text + , profile : Profiles.Type + , codename : DebianVersions.DebVersion + , new_tag : Text + , network : Network.Type + , step_key : Text + , if : Optional B/If + , publish : Bool + , remove_profile_from_name : Bool + } + , default = + { deps = [] : List Command.TaggedKey.Type + , name = Artifact.Type.Daemon + , version = "" + , new_tag = "" + , step_key = "promote-docker" + , profile = Profiles.Type.Standard + , network = Network.Type.Devnet + , codename = DebianVersions.DebVersion.Bullseye + , if = None B/If + , publish = False + , remove_profile_from_name = False + } + } + +let promoteDebianStep = + \(spec : PromoteDebianSpec.Type) + -> let package_name + : Text + = Package.debianName spec.package spec.profile spec.network + + let new_name = + if spec.remove_profile_from_name + + then "--new-name ${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else "" + + in Command.build + Command.Config::{ + , commands = + Toolchain.runner + DebianVersions.DebVersion.Bullseye + [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" ] + "./buildkite/scripts/promote-deb.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName + spec.codename} --from-component ${DebianChannel.lowerName + spec.from_channel} --to-component ${DebianChannel.lowerName + spec.to_channel} ${new_name}" + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDebianVerificationStep = + \(spec : PromoteDebianSpec.Type) + -> let name = + if spec.remove_profile_from_name + + then "${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else Package.debianName spec.package spec.profile spec.network + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName + spec.codename} --channel ${DebianChannel.lowerName + spec.to_channel}" + ] + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerStep = + \(spec : PromoteDockerSpec.Type) + -> let old_tag = + Artifact.dockerTag + spec.name + spec.version + spec.codename + spec.profile + spec.network + False + + let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let publish = if spec.publish then "-p" else "" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName + spec.name} --version ${old_tag} --tag ${new_tag} ${publish}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerVerificationStep = + \(spec : PromoteDockerSpec.Type) + -> let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let repo = + if spec.publish + + then "docker.io/minaprotocol" + + else "gcr.io/o1labs-192920" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "docker pull ${repo}/${Artifact.dockerName + spec.name}:${new_tag}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promotePipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + (\(spec : PromoteDebianSpec.Type) -> promoteDebianStep spec) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + (\(spec : PromoteDockerSpec.Type) -> promoteDockerStep spec) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "PromotePackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +let verifyPipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + ( \(spec : PromoteDebianSpec.Type) + -> promoteDebianVerificationStep spec + ) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + ( \(spec : PromoteDockerSpec.Type) + -> promoteDockerVerificationStep spec + ) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "VerifyPackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +in { promoteDebianStep = promoteDebianStep + , promoteDockerStep = promoteDockerStep + , promoteDebianVerificationStep = promoteDebianVerificationStep + , promoteDockerVerificationStep = promoteDockerVerificationStep + , promotePipeline = promotePipeline + , verifyPipeline = verifyPipeline + , PromoteDebianSpec = PromoteDebianSpec + , PromoteDockerSpec = PromoteDockerSpec + } diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index a5437e6a6c1..cc31e92dcbe 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -40,7 +40,7 @@ let PromoteDebianSpec = , version = "\\\\\$MINA_DEB_VERSION" , new_version = "\\\\\$MINA_DEB_VERSION" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index cbc60627dc8..7e56225d5ef 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -41,7 +41,7 @@ let PromoteDockerSpec = , new_tags = [] : List Text , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index 9a62c3b8209..0335aeeced1 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -37,11 +37,11 @@ let Spec = } , default = { dockerType = Dockers.Type.Bullseye - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , mode = PipelineMode.Type.Stable , additionalDirtyWhen = [] : List S.Type , softFail = B/SoftFail.Boolean False - , timeout = 900 + , timeout = 1000 } } @@ -67,10 +67,9 @@ let command , target = Size.XLarge , soft_fail = Some spec.softFail , depends_on = - Dockers.dependsOnStep + Dockers.dependsOn spec.dockerType - "MinaArtifactMainnet" - (Some spec.network) + spec.network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index b6279112a23..1c10cae985e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -4,12 +4,12 @@ let P = Prelude let Text/concatMap = P.Text.concatMap +let Cmd = ../Lib/Cmds.dhall + let Optional/map = Prelude.Optional.map let Optional/default = Prelude.Optional.default -let Cmd = ../Lib/Cmds.dhall - let ContainerImages = ../Constants/ContainerImages.dhall let Artifacts = ../Constants/Artifacts.dhall diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index b41f52cf28c..673b14c3808 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -18,6 +18,9 @@ let Artifact | Rosetta | ZkappTestTransaction | FunctionalTestSuite + | Toolchain + | ItnOrchestrator + | Leaderboard > let AllButTests = @@ -33,7 +36,13 @@ let AllButTests = let Main = [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] -let All = AllButTests # [ Artifact.FunctionalTestSuite ] +let All = + AllButTests + # [ Artifact.FunctionalTestSuite + , Artifact.Toolchain + , Artifact.ItnOrchestrator + , Artifact.Leaderboard + ] let capitalName = \(artifact : Artifact) @@ -46,6 +55,9 @@ let capitalName = , Rosetta = "Rosetta" , ZkappTestTransaction = "ZkappTestTransaction" , FunctionalTestSuite = "FunctionalTestSuite" + , Toolchain = "Toolchain" + , ItnOrchestrator = "ItnOrchestrator" + , Leaderboard = "Leaderboard" } artifact @@ -60,6 +72,9 @@ let lowerName = , Rosetta = "rosetta" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "toolchain" + , ItnOrchestrator = "itnOrchestrator" + , Leaderboard = "leaderboard" } artifact @@ -74,6 +89,9 @@ let dockerName = , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" , FunctionalTestSuite = "mina-test-suite" + , Toolchain = "mina-toolchain" + , ItnOrchestrator = "itn-orchestrator" + , Leaderboard = "leaderboard" } artifact @@ -89,38 +107,32 @@ let toDebianName = , Rosetta = "rosetta_${Network.lowerName network}" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "" + , ItnOrchestrator = "" + , Leaderboard = "" } artifact let toDebianNames = \(artifacts : List Artifact) - -> \(networks : List Network.Type) + -> \(network : Network.Type) -> let list_of_list_of_debians = Prelude.List.map Artifact (List Text) ( \(a : Artifact) -> merge - { Daemon = - Prelude.List.map - Network.Type - Text - (\(n : Network.Type) -> toDebianName a n) - networks + { Daemon = [ toDebianName a network ] , Archive = [ "archive" ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] - , Rosetta = - Prelude.List.map - Network.Type - Text - ( \(n : Network.Type) - -> "rosetta_${Network.lowerName n}" - ) - networks + , Rosetta = [ toDebianName a network ] , ZkappTestTransaction = [ "zkapp_test_transaction" ] , FunctionalTestSuite = [ "functional_test_suite" ] + , Toolchain = [] : List Text + , ItnOrchestrator = [] : List Text + , Leaderboard = [] : List Text } a ) @@ -165,6 +177,9 @@ let dockerTag = "${version_and_codename}-${Network.lowerName network}" , ZkappTestTransaction = "${version_and_codename}" , FunctionalTestSuite = "${version_and_codename}" + , Toolchain = "${version_and_codename}" + , ItnOrchestrator = "${version_and_codename}" + , Leaderboard = "${version_and_codename}" } artifact diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index b42dfd5ecae..463af9f56a4 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -4,11 +4,11 @@ -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI { toolchainBase = "codaprotocol/ci-toolchain-base:v3" , minaToolchainBullseye = - "gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2" + "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d" , minaToolchainBookworm = - "gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2" + "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d" , minaToolchain = - "gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2" + "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d" , elixirToolchain = "elixir:1.10-alpine" , nodeToolchain = "node:14.13.1-stretch-slim" , ubuntu2004 = "ubuntu:20.04" diff --git a/buildkite/src/Constants/DebianRepo.dhall b/buildkite/src/Constants/DebianRepo.dhall index 98239c16858..c994563ca20 100644 --- a/buildkite/src/Constants/DebianRepo.dhall +++ b/buildkite/src/Constants/DebianRepo.dhall @@ -1,3 +1,11 @@ +let Prelude = ../External/Prelude.dhall + +let Optional/map = Prelude.Optional.map + +let Optional/default = Prelude.Optional.default + +let Optional/toList = Prelude.Optional.toList + let DebianRepo : Type = < Local | PackagesO1Test > @@ -10,4 +18,94 @@ let address = } repo -in { Type = DebianRepo, address = address } +let bucket = + \(repo : DebianRepo) + -> merge + { Local = None Text, PackagesO1Test = Some "packages.o1test.net" } + repo + +let bucket_or_default = + \(repo : DebianRepo) + -> let maybeBucket = + Optional/map + Text + Text + (\(bucket : Text) -> bucket) + (bucket repo) + + in Optional/default Text "" maybeBucket + +let bucketArg = + \(repo : DebianRepo) + -> let maybeBucket = + Optional/map + Text + Text + (\(bucket : Text) -> "--bucket " ++ bucket) + (bucket repo) + + in Optional/default Text "" maybeBucket + +let keyId = + \(repo : DebianRepo) + -> merge { Local = None Text, PackagesO1Test = None Text } repo + +let keyAddress = + \(repo : DebianRepo) + -> merge { Local = None Text, PackagesO1Test = None Text } repo + +let keyAddressArg = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(key : Text) -> "--key-path " ++ key) + (keyAddress repo) + + in Optional/default Text "" maybeKey + +let keyArg = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "--sign " ++ repo) + (keyId repo) + + in Optional/default Text "" maybeKey + +let keyIdEnvList = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "SIGN=" ++ repo) + (keyId repo) + + in Optional/toList Text maybeKey + +let bucketEnv = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "BUCKET=" ++ repo) + (bucket repo) + + in Optional/default Text "" maybeKey + +in { Type = DebianRepo + , keyIdEnvList = keyIdEnvList + , keyAddressArg = keyAddressArg + , address = address + , bucket = bucket + , bucket_or_default = bucket_or_default + , bucketArg = bucketArg + , bucketEnv = bucketEnv + , keyId = keyId + , keyArg = keyArg + } diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 138a0533803..e62660a0316 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,6 +4,8 @@ let Optional/default = Prelude.Optional.default let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall @@ -33,6 +35,7 @@ let lowerName = let dependsOnStep = \(prefix : Optional Text) -> \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) @@ -40,47 +43,28 @@ let dependsOnStep = let prefix = Optional/default Text "MinaArtifact" prefix + let name = + "${prefix}${capitalName + debVersion}${Network.capitalName + network}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + in merge - { Bookworm = - [ { name = - "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Bullseye = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Jammy = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Focal = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] + { Bookworm = [ { name = name, key = "${step}-deb-pkg" } ] + , Bullseye = [ { name = name, key = "${step}-deb-pkg" } ] + , Jammy = [ { name = name, key = "${step}-deb-pkg" } ] + , Focal = [ { name = name, key = "${step}-deb-pkg" } ] } debVersion let dependsOn = \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> dependsOnStep (None Text) debVersion + network profile BuildFlags.Type.None "build" @@ -94,10 +78,8 @@ let minimalDirtyWhen = , S.exactly "buildkite/src/Command/ReplayerTest" "dhall" , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") , S.strictlyStart (S.contains "dockerfiles/stages") - , S.exactly "scripts/debian/build" "sh" - , S.exactly "scripts/debian/builder-helpers" "sh" - , S.exactly "scripts/docker/release" "sh" - , S.exactly "scripts/docker/build" "sh" + , S.strictlyStart (S.contains "scripts/debian") + , S.strictlyStart (S.contains "scripts/docker") , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" @@ -111,7 +93,7 @@ let bullseyeDirtyWhen = [ S.strictlyStart (S.contains "src") , S.strictlyStart (S.contains "automation") , S.strictly (S.contains "Makefile") - , S.exactly "buildkite/scripts/connect-to-berkeley" "sh" + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" , S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index b700603b17d..eb2dfeb9600 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,9 +1,3 @@ -let Prelude = ../External/Prelude.dhall - -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Profiles = ./Profiles.dhall let Artifacts = ./Artifacts.dhall @@ -37,44 +31,45 @@ let lowerName = let dependsOnStep = \(docker : Docker) -> \(prefix : Text) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> let profileSuffix = Profiles.toSuffixUppercase profile + -> let network = "${Network.capitalName network}" - let suffix = "docker-image" - - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network + let profileSuffix = "${Profiles.toSuffixUppercase profile}" - let networkOrDefault = Optional/default Text "" maybeNetwork + let suffix = "docker-image" - let key = - "${Artifacts.lowerName - binary}${networkOrDefault}-${lowerName docker}-${suffix}" + let key = "${Artifacts.lowerName binary}-${suffix}" in merge { Bookworm = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Bullseye = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Jammy = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] , Focal = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] @@ -83,7 +78,7 @@ let dependsOnStep = let dependsOn = \(docker : Docker) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) -> dependsOnStep docker "MinaArtifact" network profile binary diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 92462c87ae3..86094577017 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -4,23 +4,18 @@ let List/any = Prelude.List.any let Network : Type - = < Devnet | Mainnet | Berkeley > + = < Devnet | Mainnet > let capitalName = \(network : Network) - -> merge - { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } - network + -> merge { Devnet = "Devnet", Mainnet = "Mainnet" } network let lowerName = \(network : Network) - -> merge - { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } - network + -> merge { Devnet = "devnet", Mainnet = "mainnet" } network let requiresMainnetBuild = - \(network : Network) - -> merge { Devnet = True, Mainnet = True, Berkeley = False } network + \(network : Network) -> merge { Devnet = False, Mainnet = True } network let foldMinaBuildMainnetEnv = \(networks : List Network) @@ -30,9 +25,28 @@ let foldMinaBuildMainnetEnv = else "MINA_BUILD_MAINNET=false" +let buildMainnetEnv = + \(network : Network) + -> if requiresMainnetBuild network + + then "MINA_BUILD_MAINNET=true" + + else "MINA_BUILD_MAINNET=false" + +let foldNames = + \(networks : List Network) + -> Prelude.List.fold + Network + networks + Text + (\(x : Network) -> \(y : Text) -> "${capitalName x}" ++ y) + "" + in { Type = Network , capitalName = capitalName , lowerName = lowerName , requiresMainnetBuild = requiresMainnetBuild , foldMinaBuildMainnetEnv = foldMinaBuildMainnetEnv + , buildMainnetEnv = buildMainnetEnv + , foldNames = foldNames } diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 660027486f0..a2bb91b05a8 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -73,19 +73,6 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } - , Command.build - Command.Config::{ - , commands = - [ Cmd.run "scripts/merged-to-proof-systems.sh berkeley" ] - , label = - "[proof-systems] Check merges cleanly into proof-systems berkeley branch" - , key = "merged-to-proof-systems-berkeley" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Multi - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } - } , Command.build Command.Config::{ , commands = diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall deleted file mode 100644 index 5c88037f8d1..00000000000 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ /dev/null @@ -1,46 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let DebianRepo = ../../Constants/DebianRepo.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "itn-orchestrator" - , step_key = "itn-orchestrator-docker-image" - , network = "berkeley" - , deb_repo = DebianRepo.Type.Local - , deps = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") - , S.strictlyStart (S.contains "src/app/itn_orchestrator") - ] - , path = "Release" - , name = "ItnOrchestratorArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = [ DockerImage.generateStep spec ] - } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall similarity index 88% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index aa4037d8626..18633217d3c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -21,13 +21,12 @@ in Pipeline.build , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ] - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall similarity index 89% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 82d9d49a532..76829c42836 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -24,13 +24,12 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction ] , debVersion = DebianVersions.DebVersion.Focal - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 86e483e1e31..357da774d05 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -27,10 +31,9 @@ in Pipeline.build , steps = [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "bullseye" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Bullseye , no_cache = True - , step_key = "toolchain-bullseye-docker-image" } in DockerImage.generateStep toolchainBullseyeSpec diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall index ab0030f99f4..2ffe0472892 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -25,14 +29,13 @@ in Pipeline.build , tags = [ PipelineTag.Type.Toolchain ] } , steps = - [ let toolchainSpec = + [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "focal" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Focal , no_cache = True - , step_key = "toolchain-focal-docker-image" } - in DockerImage.generateStep toolchainSpec + in DockerImage.generateStep toolchainBullseyeSpec ] } diff --git a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall b/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall deleted file mode 100644 index b44d2b6ea96..00000000000 --- a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Command = ../../Command/Base.dhall - -let Docker = ../../Command/Docker/Type.dhall - -let Size = ../../Command/Size.dhall - -let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/check-compatibility" "sh" - , S.exactly "buildkite/src/Jobs/Test/BerkeleyCompatibility" "dhall" - ] - , path = "Test" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - , name = "BerkeleyCompatibility" - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run "buildkite/scripts/check-compatibility.sh berkeley" ] - , label = "Test: berkeley compatibilty test" - , key = "berkeley-compatibilty-test" - , target = Size.XLarge - , docker = None Docker.Type - , depends_on = dependsOn - , timeout_in_minutes = Some +60 - } - ] - } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index dee29949cd5..bef4ebb1e47 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -12,9 +12,12 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall similarity index 74% rename from buildkite/src/Jobs/Test/ConnectToBerkeley.dhall rename to buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 312e63f9435..526ee022143 100644 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -14,18 +14,15 @@ let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall let Network = ../../Constants/Network.dhall -let Dockers = ../../Constants/DockerVersions.dhall - let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard - Artifacts.Type.Daemon in Pipeline.build Pipeline.Config::{ @@ -33,17 +30,17 @@ in Pipeline.build , dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" + , S.exactly "buildkite/src/Jobs/Test/ConnectToDevnet" "dhall" , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" ] , path = "Test" - , name = "ConnectToBerkeley" + , name = "ConnectToDevnet" , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } , steps = [ ConnectToTestnet.step dependsOn - "berkeley" + "${Network.lowerName Network.Type.Devnet}" "40s" "2m" (B/SoftFail.Boolean True) diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 99132224a5b..8bcecf6c27f 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -14,11 +14,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Lightnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall index c0a4b2c26cc..9cbab00b41b 100644 --- a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall +++ b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall @@ -18,6 +18,8 @@ let Profiles = ../../Constants/Profiles.dhall let BuildFlags = ../../Constants/BuildFlags.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -26,12 +28,14 @@ let dependsOn = DebianVersions.dependsOnStep (None Text) DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard BuildFlags.Type.Instrumented "build" # DebianVersions.dependsOnStep (None Text) DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard BuildFlags.Type.None "build" diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index a0414c85a4f..67cbfc24f21 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -10,16 +10,16 @@ let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall let Profiles = ../../Constants/Profiles.dhall +let Dockers = ../../Constants/DockerVersions.dhall + let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall -let Dockers = ../../Constants/DockerVersions.dhall - let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.FunctionalTestSuite diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index f9caa4f30a7..45b7840f749 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -19,7 +19,7 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.FunctionalTestSuite diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 21397b34be7..21bfdcb7498 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -18,14 +18,16 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let RunWithPostgres = ../../Command/RunWithPostgres.dhall let Network = ../../Constants/Network.dhall -let RunWithPostgres = ../../Command/RunWithPostgres.dhall +let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,12 +57,13 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Berkeley) + (Some Network.Type.Devnet) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] @@ -71,7 +74,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 9fb8d631225..ab0731cd45e 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -20,12 +20,14 @@ let Network = ../../Constants/Network.dhall let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall - let Dockers = ../../Constants/DockerVersions.dhall +let Artifacts = ../../Constants/Artifacts.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,18 +57,20 @@ in Pipeline.build , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-full.sh" ] , label = "Rosetta integration tests Bullseye Long" , key = "rosetta-integration-tests-bullseye-long" , soft_fail = Some (B/SoftFail.Boolean True) + , timeout_in_minutes = Some +90 , target = Size.Small , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index 9bfa98eb8a5..599b946a2ad 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -16,11 +16,14 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall +let Network = ../../Constants/Network.dhall + let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 74959dd325a..e5bbae08182 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -14,6 +14,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,9 +23,11 @@ let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Lightnet # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 240902fe732..8f09d7ebc18 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 77bd48b49ff..742f708aa09 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index 1368e8e70bc..8e55d8e7067 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -18,7 +18,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Standard let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> B/SoftFail -> Command.Type diff --git a/buildkite/src/Jobs/Test/ZkappMetrics.dhall b/buildkite/src/Jobs/Test/ZkappMetrics.dhall index 90117a87a34..e03be0b2b0e 100644 --- a/buildkite/src/Jobs/Test/ZkappMetrics.dhall +++ b/buildkite/src/Jobs/Test/ZkappMetrics.dhall @@ -12,6 +12,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,6 +23,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/README.md b/buildkite/src/README.md index d020eb0d2c2..372da8d329c 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -1,6 +1,6 @@ # Buildkite CI -This folder contains all dhall code which is an backbone for our CI related code for buildkite. +This folder contains all dhall code which is a backbone for our CI related code for buildkite. # Structure @@ -53,7 +53,7 @@ User defined value which describe current pipeline chunk of jobs to be executed. - coverage gathering - which gathers coverage artifacts and uploads it to coveralls.io To reach above pipeline configuration below configuration can be provided: -(non important attributes where omitted) +(non-important attributes where omitted) ``` steps: - commands: @@ -195,7 +195,7 @@ Pipeline with create 6 jobs for each Docker and Debian component separately. #### Promoting dockers form gcr to dockerhub -We want only to move dockers from gcr to dockerhub without changing version. Current implementation of pipeline is not user friendly so we need to still define `FROM_VERSION` and `TO_VERSION`. They should be equal. +We want only to move dockers from gcr to dockerhub without changing version. Current implementation of pipeline is not user-friendly so we need to still define `FROM_VERSION` and `TO_VERSION`. They should be equal. ``` - "DOCKERS=Archive,Daemon,Rosetta" @@ -204,4 +204,4 @@ We want only to move dockers from gcr to dockerhub without changing version. Cur - "NEW_VERSION=3.0.0-dc6bf78" - "CODENAMES=Focal,Bullseye" - "PUBLISH=1" -``` \ No newline at end of file +``` diff --git a/rfcs/0016-transition-frontier-persistence.md b/rfcs/0016-transition-frontier-persistence.md index 8a719a907a4..81a227202d4 100644 --- a/rfcs/0016-transition-frontier-persistence.md +++ b/rfcs/0016-transition-frontier-persistence.md @@ -8,7 +8,7 @@ This RFC proposes a new system for persisting the transition frontier's state to ## Motivation [motivation]: #motivation -The Transition Frontier is too large of a data structure to just blindly serialize and write to disk. Under non optimal network scenarios, we expect the upper bound of the data structure to be >100Gb. Even if the structure were smaller, we cannot write the structure out to disk every time we mutate it as the speed of the transition frontier data structure is critical to the systems ability to prevent DDoS attacks. Therefore, a more robust and effecient system is required to persist the Transition Frontier to disk without negatively effecting the speed of operations on the in memory copy of the Transition Frontier. +The Transition Frontier is too large of a data structure to just blindly serialize and write to disk. Under non-optimal network scenarios, we expect the upper bound of the data structure to be >100Gb. Even if the structure were smaller, we cannot write the structure out to disk every time we mutate it as the speed of the transition frontier data structure is critical to the systems ability to prevent DDoS attacks. Therefore, a more robust and effecient system is required to persist the Transition Frontier to disk without negatively effecting the speed of operations on the in memory copy of the Transition Frontier. ## Detailed design [detailed-design]: #detailed-design @@ -30,7 +30,7 @@ As actions are performed on the Transition Frontier, diffs are emitted and store Having two different mechanisms for writing the same data can be tricky as there can be bugs in one of the two mechanisms that would cause the data structures to become desynchronized. In order to help prevent this, we can introduce an incremental hash on top of the Transition Frontier which can be updated upon each diff application. This hash will give a direct and easy way to compare the structural equality of the two data structures. Being incremental, however, also means that the order of diff application needs to be the same across both data structures, so care needs to be taken with that ordering. Therefore, in a sense, this hash will represent the structure and content of the data structure, as well as the order in which actions were taken to get there. We only care about the former in our case, and the latter is just a consequence of the hash being incremental. -In order to calculate this hash correctly, we need to introduce a new concept to a diff, which is that of a diff mutant. Each diff represents some mutation to perform on the Transition Frontier, however not every diff will contain the enough information by itself to encapsulate the state of the data structure after the mutation occurs. For example, setting a balance on an account in two implementations of the data structure does not guarantee that the accounts in each a equal as there are other fields on the account besides that. This is where the concept of a diff mutant comes in. The mutant of a diff is the set of all modified values in the data structure after the diff has been applied. Using this, we can create a proper incremental diff which will truly ensure our data structures are in sync. +In order to calculate this hash correctly, we need to introduce a new concept to a diff, which is that of a diff mutant. Each diff represents some mutation to perform on the Transition Frontier, however not every diff will contain the enough information by itself to encapsulate the state of the data structure after the mutation occurs. For example, setting a balance on an account in two implementations of the data structure does not guarantee that the accounts in each an equal as there are other fields on the account besides that. This is where the concept of a diff mutant comes in. The mutant of a diff is the set of all modified values in the data structure after the diff has been applied. Using this, we can create a proper incremental diff which will truly ensure our data structures are in sync. These hashes will be Sha256 as there is no reason to use the Pedersen hashing algorithm we use in the rest of our code since none of this information needs to be snarked. The formula for calculating a new hash `h'` given an old hash `h` and a diff `diff` is as follows: `h' = sha256 h diff (Diff.mutant diff)`. diff --git a/rfcs/0020-transition-frontier-extensions-2.md b/rfcs/0020-transition-frontier-extensions-2.md index c495a2fe2fb..15df4da40f4 100644 --- a/rfcs/0020-transition-frontier-extensions-2.md +++ b/rfcs/0020-transition-frontier-extensions-2.md @@ -28,7 +28,7 @@ See [#1585](https://github.com/CodaProtocol/coda/pull/1585) for early discussion ### Extensions Redefined -A Transition Frontier Extension is an stateful, incremental view on the state of a Transiton Frontier. When a Transition Frontier is initialized, all of its extensions are also initialized using the Transition Frontier's root. Every mutation performed is represented as a list of diffs, and when the Transition Frontier updates, each Extension is notified of this list of diffs synchronously. Transition Frontier Extensions will notify the Transition Frontier if there was a update to the Extension's view when handling the diffs. If an Extension's view is updated, then a synchronous event is broadcast internally with the new view of that Extension. A Transition Frontier Extension has access to the Transition Frontier so that it can query and calculate information it requires when it handles diffs. +A Transition Frontier Extension is a stateful, incremental view on the state of a Transiton Frontier. When a Transition Frontier is initialized, all of its extensions are also initialized using the Transition Frontier's root. Every mutation performed is represented as a list of diffs, and when the Transition Frontier updates, each Extension is notified of this list of diffs synchronously. Transition Frontier Extensions will notify the Transition Frontier if there was a update to the Extension's view when handling the diffs. If an Extension's view is updated, then a synchronous event is broadcast internally with the new view of that Extension. A Transition Frontier Extension has access to the Transition Frontier so that it can query and calculate information it requires when it handles diffs. ### Extension Guidelines diff --git a/scripts/debian/aptly.sh b/scripts/debian/aptly.sh index 2c661e5a821..408bda8f105 100755 --- a/scripts/debian/aptly.sh +++ b/scripts/debian/aptly.sh @@ -12,7 +12,6 @@ CLEAR='\033[0m' RED='\033[0;31m' # global variables -declare CLI_VERSION='1.0.0'; declare CLI_NAME='aptly.sh'; declare PS4='debug($LINENO) ${FUNCNAME[0]:+${FUNCNAME[0]}}(): '; @@ -34,28 +33,28 @@ function start_aptly() { local __background=$3 local __clean=$4 local __component=$5 - local __repo="$__distribution-$__component" + local __repo="${__distribution}"-"${__component}" local __port=$6 - if [ $__clean = 1 ]; then + if [ "${__clean}" = 1 ]; then rm -rf ~/.aptly fi - - aptly repo create -component $__component -distribution $__distribution $__repo - aptly repo add $__repo $__debs + aptly repo create -component "${__component}" -distribution "${__distribution}" "${__repo}" - aptly snapshot create $__component from repo $__repo + aptly repo add "${__repo}" "${__debs}" - aptly publish snapshot -distribution=$__distribution -skip-signing $__component + aptly snapshot create "${__component}" from repo "${__repo}" - if [ $__background = 1 ]; then - aptly serve -listen localhost:$__port & - else - aptly serve -listen localhost:$__port + aptly publish snapshot -distribution="${__distribution}" -skip-signing "${__component}" + + if [ "${__background}" = 1 ]; then + aptly serve -listen localhost:"${__port}" & + else + aptly serve -listen localhost:"${__port}" fi - + } @@ -68,7 +67,7 @@ function start_help(){ echo "" echo "Parameters:" echo "" - echo " -b, --background The Docker name (mina-devnet, mina-archive etc.)" + echo " -b, --background The Docker name (mina, mina-archive etc.)" echo " -c, --codename The Codename for debian repository" echo " -d, --debians The Debian(s) to be available in aptly. Supports regular expression" echo " -m, --component The Component for debian repository. For example: unstable" @@ -91,7 +90,7 @@ function start(){ local __clean=0 local __component="unstable" local __port=$PORT - + while [ ${#} -gt 0 ]; do error_message="Error: a value is needed for '$1'"; @@ -99,7 +98,7 @@ function start(){ -h | --help ) start_help; ;; - -b | --background ) + -b | --background ) __background=1 shift; ;; @@ -130,13 +129,13 @@ function start(){ ;; esac done - - start_aptly $__distribution \ - $__debs \ - $__background \ - $__clean \ - $__component \ - $__port + + start_aptly "${__distribution}" \ + "${__debs}" \ + "${__background}" \ + "${__clean}" \ + "${__component}" \ + "${__port}" } @@ -156,9 +155,9 @@ function stop_help(){ } function stop(){ - + local __clean=0 - + while [ ${#} -gt 0 ]; do case $1 in -h | --help ) @@ -175,9 +174,9 @@ function stop(){ ;; esac done - + pkill aptly - if [ $__clean = 1 ]; then + if [ "${__clean}" = 1 ]; then rm -rf ~/.aptly fi } @@ -215,4 +214,4 @@ function main(){ esac } -main "$@"; \ No newline at end of file +main "$@"; diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index 1c1fff73126..81b8c2a01e4 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -8,16 +8,16 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # In case of running this script on detached head, script has difficulties in finding out # what is the current branch. -if [[ -n "$BRANCH_NAME" ]]; then - BRANCH_NAME="$BRANCH_NAME" source ${SCRIPTPATH}/../export-git-env-vars.sh +if [[ -n "${BRANCH_NAME}" ]]; then + BRANCH_NAME="${BRANCH_NAME}" source "${SCRIPTPATH}"/../export-git-env-vars.sh else - source ${SCRIPTPATH}/../export-git-env-vars.sh -fi + source "${SCRIPTPATH}"/../export-git-env-vars.sh +fi echo "after export" -source ${SCRIPTPATH}/builder-helpers.sh - +source "${SCRIPTPATH}"/builder-helpers.sh + if [ $# -eq 0 ] then echo "No arguments supplied. Building all known debian packages" @@ -25,7 +25,6 @@ if [ $# -eq 0 ] build_keypair_deb build_archive_deb build_batch_txn_deb - build_daemon_berkeley_deb build_daemon_mainnet_deb build_daemon_devnet_deb build_rosetta_berkeley_deb @@ -35,15 +34,15 @@ if [ $# -eq 0 ] build_functional_test_suite_deb build_zkapp_test_transaction_deb - else + else for i in "$@"; do - if [[ $(type -t "build_${i}_deb") == function ]] - then + if [[ $(type -t "build_${i}_deb") == function ]] + then echo "Building $i debian package" "build_${i}_deb" - else + else echo "invalid debian package name '$i'" exit 1 fi done -fi \ No newline at end of file +fi diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index af9a92b1c1e..a3d31fecff5 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -10,7 +10,7 @@ MINA_DEB_RELEASE=${MINA_DEB_RELEASE:-"unstable"} # Helper script to include when building deb archives. -echo "--- Setting up the envrionment to build debian packages..." +echo "--- Setting up the environment to build debian packages..." SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" cd "${SCRIPTPATH}/../../_build" @@ -46,13 +46,13 @@ esac # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" + MINA_DEB_NAME="mina-devnet" DEB_SUFFIX="" ;; *) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" + MINA_DEB_NAME="mina-devnet-${_SUFFIX}" DEB_SUFFIX="-${_SUFFIX}" ;; esac @@ -63,7 +63,7 @@ if [[ -v DUNE_INSTRUMENT_WITH ]]; then INSTRUMENTED_SUFFIX=instrumented MINA_DEB_NAME="${MINA_DEB_NAME}-${INSTRUMENTED_SUFFIX}" DEB_SUFFIX="${DEB_SUFFIX}-${INSTRUMENTED_SUFFIX}" -fi +fi BUILDDIR="deb_build" @@ -123,9 +123,9 @@ build_deb() { # Build the package echo "------------------------------------------------------------" - fakeroot dpkg-deb --build "${BUILDDIR}" ${1}_${MINA_DEB_VERSION}.deb + fakeroot dpkg-deb --build "${BUILDDIR}" "${1}"_"${MINA_DEB_VERSION}".deb echo "build_deb outputs:" - ls -lh ${1}_*.deb + ls -lh "${1}"_*.deb echo "deleting BUILDDIR ${BUILDDIR}" rm -rf "${BUILDDIR}" @@ -137,9 +137,9 @@ copy_common_daemon_configs() { echo "------------------------------------------------------------" echo "copy_common_daemon_configs inputs:" - echo "Network Name: ${1} (like mainnet, devnet, berkeley)" + echo "Network Name: ${1} (like mainnet, devnet)" echo "Signature Type: ${2} (mainnet or testnet)" - echo "Seed List URL path: ${3} (like seed-lists/berkeley_seeds.txt)" + echo "Seed List URL path: ${3} (like seed-lists/devnet_seeds.txt)" # Copy shared binaries cp ../src/app/libp2p_helper/result/bin/libp2p_helper "${BUILDDIR}/usr/local/bin/coda-libp2p_helper" @@ -148,7 +148,7 @@ copy_common_daemon_configs() { cp ./default/src/app/validate_keypair/validate_keypair.exe "${BUILDDIR}/usr/local/bin/mina-validate-keypair" # Copy signature-based Binaries (based on signature type $2 passed into the function) - cp ./default/src/app/cli/src/mina_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina" + cp ./default/src/app/cli/src/mina_"${2}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina" # Copy over Build Configs (based on $2) mkdir -p "${BUILDDIR}/etc/coda/build_config" @@ -161,9 +161,8 @@ copy_common_daemon_configs() { # Include all useful genesis ledgers cp ../genesis_ledgers/mainnet.json "${BUILDDIR}/var/lib/coda/mainnet.json" cp ../genesis_ledgers/devnet.json "${BUILDDIR}/var/lib/coda/devnet.json" - cp ../genesis_ledgers/berkeley.json "${BUILDDIR}/var/lib/coda/berkeley.json" # Set the default configuration based on Network name ($1) - cp ../genesis_ledgers/${1}.json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" + cp ../genesis_ledgers/"${1}".json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" cp ../scripts/hardfork/create_runtime_config.sh "${BUILDDIR}/usr/local/bin/mina-hf-create-runtime-config" cp ../scripts/mina-verify-packaged-fork-config "${BUILDDIR}/usr/local/bin/mina-verify-packaged-fork-config" # Update the mina.service with a new default PEERS_URL based on Seed List URL $3 @@ -239,6 +238,10 @@ build_batch_txn_deb() { build_functional_test_suite_deb() { create_control_file mina-test-suite "${SHARED_DEPS}" 'Test suite apps for mina.' + mkdir -p "${BUILDDIR}/etc/mina/test/archive" + + cp -r ../src/test/archive/* "${BUILDDIR}"/etc/mina/test/archive/ + # Binaries cp ./default/src/test/command_line_tests/command_line_tests.exe "${BUILDDIR}/usr/local/bin/mina-command-line-tests" cp ./default/src/app/heap_usage/heap_usage.exe "${BUILDDIR}/usr/local/bin/mina-heap-usage" @@ -251,11 +254,11 @@ build_functional_test_suite_deb() { ##################################### END TEST SUITE PACKAGE ####################################### function copy_common_rosetta_configs () { - - # Copy rosetta-based Binaries - cp ./default/src/app/rosetta/rosetta_${1}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" - cp ./default/src/app/rosetta/ocaml-signer/signer_${1}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" - + + # Copy rosetta-based Binaries + cp ./default/src/app/rosetta/rosetta_"${1}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" + cp ./default/src/app/rosetta/ocaml-signer/signer_"${1}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" + mkdir -p "${BUILDDIR}/etc/mina/rosetta" mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" mkdir -p "${BUILDDIR}/etc/mina/rosetta/scripts" @@ -265,52 +268,38 @@ function copy_common_rosetta_configs () { cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ./default/src/app/rosetta/indexer_test/indexer_test.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-indexer-test" - + } ##################################### ROSETTA MAINNET PACKAGE ####################################### build_rosetta_mainnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building mainnet rosetta deb" create_control_file mina-rosetta-mainnet "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" copy_common_rosetta_configs "mainnet" - + build_deb mina-rosetta-mainnet } ##################################### ROSETTA MAINNET PACKAGE ####################################### build_rosetta_devnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building devnet rosetta deb" create_control_file mina-rosetta-devnet "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" copy_common_rosetta_configs "testnet" - - build_deb mina-rosetta-devnet -} - -##################################### ROSETTA BERKELEY PACKAGE ####################################### -build_rosetta_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building rosetta berkeley deb" - create_control_file mina-rosetta-berkeley "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" - - copy_common_rosetta_configs "testnet" - - build_deb mina-rosetta-berkeley + build_deb mina-rosetta-devnet } - ##################################### MAINNET PACKAGE ####################################### build_daemon_mainnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building mainnet deb without keys:" @@ -324,32 +313,17 @@ build_daemon_mainnet_deb() { ##################################### DEVNET PACKAGE ####################################### build_daemon_devnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building testnet signatures deb without keys:" - create_control_file mina-devnet "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' "${SUGGESTED_DEPS}" + create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' copy_common_daemon_configs devnet testnet 'seed-lists/devnet_seeds.txt' - build_deb mina-devnet -} -##################################### END DEVNET PACKAGE ####################################### - -##################################### BERKELEY PACKAGE ####################################### -build_daemon_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building Mina Berkeley testnet signatures deb without keys:" - - create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon' - - copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt' - build_deb "${MINA_DEB_NAME}" - } -##################################### END BERKELEY PACKAGE ####################################### +##################################### END DEVNET PACKAGE ####################################### ##################################### ARCHIVE PACKAGE ########################################## build_archive_deb () { @@ -365,13 +339,13 @@ build_archive_deb () { cp ./default/src/app/archive/archive.exe "${BUILDDIR}/usr/local/bin/mina-archive" cp ./default/src/app/archive_blocks/archive_blocks.exe "${BUILDDIR}/usr/local/bin/mina-archive-blocks" cp ./default/src/app/extract_blocks/extract_blocks.exe "${BUILDDIR}/usr/local/bin/mina-extract-blocks" - + mkdir -p "${BUILDDIR}/etc/mina/archive" cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/usr/local/bin/mina-missing-blocks-guardian" - + cp ./default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe "${BUILDDIR}/usr/local/bin/mina-missing-blocks-auditor" cp ./default/src/app/replayer/replayer.exe "${BUILDDIR}/usr/local/bin/mina-replayer" - + cp ../src/app/archive/create_schema.sql "${BUILDDIR}/etc/mina/archive" cp ../src/app/archive/drop_tables.sql "${BUILDDIR}/etc/mina/archive" @@ -389,6 +363,6 @@ build_zkapp_test_transaction_deb () { # Binaries cp ./default/src/app/zkapp_test_transaction/zkapp_test_transaction.exe "${BUILDDIR}/usr/local/bin/mina-zkapp-test-transaction" - build_deb mina-zkapp-test-transaction + build_deb mina-zkapp-test-transaction } ##################################### END ZKAPP TEST TXN PACKAGE ####################################### diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 624a82d4f33..0b5c669ab40 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -11,6 +11,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -r|--release) DEB_RELEASE="$2"; shift;; -v|--version) DEB_VERSION="$2"; shift;; -c|--codename) DEB_CODENAME="$2"; shift;; + -s|--sign) SIGN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -23,6 +24,7 @@ function usage() { echo " -r, --release The Debian release" echo " -v, --version The Debian version" echo " -c, --codename The Debian codename" + echo " -s, --sign The Debian key id used for sign" echo "" echo "Example: $0 --name mina-archive --release unstable --version 2.0.0-rc1-48efea4 --codename bullseye " exit 1 @@ -34,6 +36,12 @@ if [[ -z "$DEB_CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$DEB_RELEASE" ]]; then usage "Release is not set!"; fi; +if [[ -z "${SIGN:-}" ]]; then + SIGN_ARG="" +else + SIGN_ARG="--sign=$SIGN" +fi + BUCKET_ARG="--bucket=packages.o1test.net" S3_REGION_ARG="--s3-region=us-west-2" # utility for publishing deb repo with commons options @@ -47,20 +55,24 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ --fail-if-exists \ --lock \ --preserve-versions \ - --cache-control=max-age=120" + --cache-control=max-age=120 \ + $SIGN_ARG" + +if [[ -z "${PASSPHRASE:-}" ]]; then + GPG_OPTS=() +else + GPG_OPTS=("--gpg-options=\"--batch" "--pinentry-mode=loopback" "--yes") +fi + + echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}" # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. for _ in {1..10}; do ( - ${DEBS3_UPLOAD} \ - --component "${DEB_RELEASE}" \ - --codename "${DEB_CODENAME}" \ - "${DEB_NAMES}" + ${DEBS3_UPLOAD} --component "${DEB_RELEASE}" --codename "${DEB_CODENAME}" "${GPG_OPTS[@]}" "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done -debs=() - for deb in $DEB_NAMES do # extracting name from debian package path. E.g: diff --git a/scripts/debian/reversion.sh b/scripts/debian/reversion.sh index 5fec028f490..94cb3e27aa8 100755 --- a/scripts/debian/reversion.sh +++ b/scripts/debian/reversion.sh @@ -3,6 +3,7 @@ set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m' +BUCKET=packages.o1test.net while [[ "$#" -gt 0 ]]; do case $1 in -d|--deb) DEB="$2"; shift;; @@ -14,6 +15,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --new-version) NEW_VERSION="$2"; shift;; --suite) SUITE="$2"; shift;; --new-suite) NEW_SUITE="$2"; shift;; + --sign) SIGN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -30,6 +32,7 @@ function usage() { echo " --new-version The New Debian version" echo " --suite The Current Debian suite" echo " --new-suite The New Debian suite" + echo " --sign The Public Key id, which is used to sign package. Key must be stored locally" echo "" echo "Example: $0 --deb mina-archive --version 2.0.0-rc1-48efea4 --new-version 2.0.0-rc1 --codename bullseye --release unstable --new-release umt" exit 1 @@ -39,18 +42,21 @@ if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$DEB; fi; if [[ -z "$NEW_RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi; if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$VERSION; fi; if [[ -z "$NEW_SUITE" ]]; then NEW_SUITE=$SUITE; fi; - if [[ -z "$DEB" ]]; then NEW_NAME=$DEB; fi; if [[ -z "$RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi; if [[ -z "$VERSION" ]]; then NEW_VERSION=$VERSION; fi; if [[ -z "$SUITE" ]]; then NEW_SUITE=$SUITE; fi; - +if [[ -z "$SIGN" ]]; then + SIGN_ARG="" +else + SIGN_ARG="--sign $SIGN" +fi function rebuild_deb() { rm -f "${DEB}_${VERSION}.deb" rm -rf "${NEW_NAME}_${NEW_VERSION}" - wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb + wget https://s3.us-west-2.amazonaws.com/${BUCKET}/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb dpkg-deb -R "${DEB}_${VERSION}.deb" "${NEW_NAME}_${NEW_VERSION}" sed -i 's/Version: '"${VERSION}"'/Version: '"${NEW_VERSION}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" sed -i 's/Package: '"${DEB}"'/Package: '"${NEW_NAME}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" @@ -59,5 +65,4 @@ function rebuild_deb() { } rebuild_deb - -source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} +source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version "${NEW_VERSION}" --codename "${CODENAME}" --release "${NEW_RELEASE}" --bucket ${BUCKET} ${SIGN_ARG} diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index c61d3636a7e..b0837a142a6 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -4,16 +4,18 @@ set -eox pipefail CHANNEL=umt-mainnet VERSION=3.0.0-f872d85 CODENAME=bullseye +BUCKET=packages.o1test.net while [[ "$#" -gt 0 ]]; do case $1 in -c|--channel) CHANNEL="$2"; shift;; -v|--version) VERSION="$2"; shift;; -p|--package) PACKAGE="$2"; shift;; -m|--codename) CODENAME="$2"; shift;; + -b|--bucket) BUCKET="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done -if [ -z $PACKAGE ]; then +if [ -z "${PACKAGE}" ]; then echo "No package defined. exiting.."; exit 1; fi @@ -29,7 +31,7 @@ SCRIPT=' set -x \ && echo installing mina \ && apt-get update > /dev/null \ && apt-get install -y lsb-release ca-certificates > /dev/null \ - && echo "deb [trusted=yes] http://packages.o1test.net '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \ + && echo "deb [trusted=yes] https://'$BUCKET' '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \ && apt-get update > /dev/null \ && apt list -a '$PACKAGE' \ && apt-get install -y --allow-downgrades '$PACKAGE'='$VERSION' \ diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 34f8f62a0eb..4c400e7e396 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -47,6 +47,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --deb-profile) DEB_PROFILE="$2"; shift;; --deb-repo) INPUT_REPO="$2"; shift;; --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; + --deb-repo-key) DEB_REPO_KEY="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done diff --git a/src/app/archive/cli/dune b/src/app/archive/cli/dune index 69f4c7aa20c..2a815358764 100644 --- a/src/app/archive/cli/dune +++ b/src/app/archive/cli/dune @@ -2,22 +2,23 @@ (name archive_cli) (public_name archive.cli) (libraries - ;;opam libraries - async_kernel - caqti - archive_lib - core - async - caqti-async - core_kernel - async.async_command - ;;local libraries - genesis_constants - mina_runtime_config - cli_lib - logger - block_time - mina_version - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_mina))) + ;;opam libraries + async_kernel + caqti + archive_lib + core + async + caqti-async + core_kernel + async.async_command + ;;local libraries + genesis_constants + mina_runtime_config + cli_lib + logger + block_time + mina_version) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane ppx_mina))) diff --git a/src/app/archive/dune b/src/app/archive/dune index 27c2526bfc1..29b24b628ab 100644 --- a/src/app/archive/dune +++ b/src/app/archive/dune @@ -4,9 +4,18 @@ (public_name archive) (modules archive) (modes native) - (libraries archive_cli async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package archive) @@ -14,9 +23,19 @@ (public_name archive-testnet) (modules archive_testnet_signatures) (modes native) - (libraries archive_cli mina_signature_kind.testnet async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + mina_signature_kind.testnet + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package archive) @@ -24,6 +43,16 @@ (public_name archive-mainnet) (modules archive_mainnet_signatures) (modes native) - (libraries archive_cli mina_signature_kind.mainnet async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + mina_signature_kind.mainnet + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/archive/lib/dune b/src/app/archive/lib/dune index b24273b9de8..3752fa03a60 100644 --- a/src/app/archive/lib/dune +++ b/src/app/archive/lib/dune @@ -2,80 +2,88 @@ (name archive_lib) (public_name archive_lib) (libraries - ;;opam libraries - sexplib0 - async_unix - base64 - async_rpc_kernel - base.caml - base.base_internalhash_types - bin_prot.shape - async_kernel - caqti-driver-postgresql - core - async - core_kernel - caqti - caqti-async - async.async_rpc - integers - ppx_inline_test.config - uri - ;;local libraries - bounded_types - mina_wire_types - kimchi_backend - child_processes - precomputed_values - coda_genesis_ledger - consensus.vrf - mina_runtime_config - sgn - mina_base.util - kimchi_pasta - kimchi_pasta.basic - pickles - random_oracle - pipe_lib - block_time - currency - staged_ledger - data_hash_lib - transition_frontier_base - genesis_ledger_helper - one_or_two - mina_caqti - mina_generators - zkapp_command_builder - mina_base - mina_ledger - mina_block - transition_frontier - logger - mina_metrics - signature_lib - mina_numbers - unsigned_extended - mina_state - pickles_types - pickles.backend - with_hash - mina_base.import - consensus - genesis_constants - genesis_ledger_helper.lib - verifier - quickcheck_lib - snark_params - mina_transaction - protocol_version - staged_ledger_diff - error_json - ppx_deriving_yojson.runtime - ppx_version.runtime - o1trace - ) - (inline_tests (flags -verbose -show-counts)) + ;;opam libraries + sexplib0 + async_unix + base64 + async_rpc_kernel + base.caml + base.base_internalhash_types + bin_prot.shape + async_kernel + caqti-driver-postgresql + core + async + core_kernel + caqti + caqti-async + async.async_rpc + integers + ppx_inline_test.config + uri + ;;local libraries + bounded_types + mina_wire_types + kimchi_backend + child_processes + precomputed_values + coda_genesis_ledger + consensus.vrf + mina_runtime_config + sgn + mina_base.util + kimchi_pasta + kimchi_pasta.basic + pickles + random_oracle + pipe_lib + block_time + currency + staged_ledger + data_hash_lib + transition_frontier_base + genesis_ledger_helper + one_or_two + mina_caqti + mina_generators + zkapp_command_builder + mina_base + mina_ledger + mina_block + transition_frontier + logger + mina_metrics + signature_lib + mina_numbers + unsigned_extended + mina_state + pickles_types + pickles.backend + with_hash + mina_base.import + consensus + genesis_constants + genesis_ledger_helper.lib + verifier + quickcheck_lib + snark_params + mina_transaction + protocol_version + staged_ledger_diff + error_json + ppx_deriving_yojson.runtime + ppx_version.runtime + o1trace) + (inline_tests + (flags -verbose -show-counts)) (modes native) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf ppx_deriving_yojson h_list.ppx))) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_jane + ppx_custom_printf + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/archive_blocks/dune b/src/app/archive_blocks/dune index 48f0db5aabd..152986ce769 100644 --- a/src/app/archive_blocks/dune +++ b/src/app/archive_blocks/dune @@ -3,27 +3,28 @@ (name archive_blocks) (public_name archive_blocks) (libraries - ;; opam libraries - async_unix - core - result - async_kernel - uri - stdio - caqti-driver-postgresql - caqti - async - core_kernel - caqti-async - base - base.caml - async.async_command - ;; local libraries - logger - mina_block - bounded_types - genesis_constants - archive_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + async_unix + core + result + async_kernel + uri + stdio + caqti-driver-postgresql + caqti + async + core_kernel + caqti-async + base + base.caml + async.async_command + ;; local libraries + logger + mina_block + bounded_types + genesis_constants + archive_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) diff --git a/src/app/batch_txn_tool/dune b/src/app/batch_txn_tool/dune index f55ebfdf875..8b957922d9f 100644 --- a/src/app/batch_txn_tool/dune +++ b/src/app/batch_txn_tool/dune @@ -1,37 +1,34 @@ (executable (name batch_txn_tool) (libraries - ;; opam libraries - async.async_command - integers - base - uri - core_kernel - core - async - async_kernel - yojson - async_unix - ;; local libraries - currency - mina_base.import - graphql_lib - signature_lib - mina_base - secrets - mina_numbers - snark_params - unsigned_extended - mina_compile_config - mina_wire_types - integration_test_lib - logger - bounded_types - ) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + async.async_command + integers + base + uri + core_kernel + core + async + async_kernel + yojson + async_unix + ;; local libraries + currency + mina_base.import + graphql_lib + signature_lib + mina_base + secrets + mina_numbers + snark_params + unsigned_extended + mina_compile_config + mina_wire_types + integration_test_lib + logger + bounded_types) + (instrumentation + (backend bisect_ppx)) (preprocessor_deps ../../../graphql_schema.json) - (preprocess (pps - ppx_version - ppx_let - ppx_mina -))) + (preprocess + (pps ppx_version ppx_let ppx_mina))) diff --git a/src/app/benchmarks/benchmarks.ml b/src/app/benchmarks/benchmarks.ml index bc09c3c7b8f..0396bee99c6 100644 --- a/src/app/benchmarks/benchmarks.ml +++ b/src/app/benchmarks/benchmarks.ml @@ -3,11 +3,11 @@ (* you can control which libraries are benchmarked using the environment variable BENCHMARK_LIBRARIES, which is either "all" or a comma-delimited list of libraries; if the variable is not present, or empty, run benchmarks for all libraries - *) +*) open Core_kernel -let available_libraries = ["vrf_lib_tests"; "mina_base"] +let available_libraries = [ "vrf_lib_tests"; "mina_base" ] let run_benchmarks_in_lib libname = printf "Running inline tests in library \"%s\"\n%!" libname ; diff --git a/src/app/benchmarks/dune b/src/app/benchmarks/dune index 23918c24e7c..4052d497da6 100644 --- a/src/app/benchmarks/dune +++ b/src/app/benchmarks/dune @@ -3,8 +3,17 @@ (executable (name benchmarks) (public_name main) - (libraries core_bench.inline_benchmarks vrf_lib_tests mina_base core_kernel core base bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (libraries + core_bench.inline_benchmarks + vrf_lib_tests + mina_base + core_kernel + core + base + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (link_flags -linkall) (modes native)) diff --git a/src/app/best_tip_merger/dune b/src/app/best_tip_merger/dune index 2b8ab0c9e03..93c2ff1087b 100644 --- a/src/app/best_tip_merger/dune +++ b/src/app/best_tip_merger/dune @@ -1,50 +1,50 @@ (executable - (package best_tip_merger) - (name best_tip_merger) - (public_name best_tip_merger) - (modes native) - (libraries - ;; opam libraries - async.async_command - async_kernel - stdio - base.caml - yojson - lib - core_kernel - async - ppx_deriving_yojson.runtime - core - sexplib0 - result - base.base_internalhash_types - async_unix - ;; local libraries - visualization - mina_numbers - mina_state - transition_frontier_extensions - mina_base - logger.file_system - logger - cli_lib - transition_frontier - data_hash_lib - rose_tree - consensus - mina_wire_types - pickles.backend - snark_params - pickles - kimchi_pasta - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_jane - ppx_version - ppx_compare)) - ) + (package best_tip_merger) + (name best_tip_merger) + (public_name best_tip_merger) + (modes native) + (libraries + ;; opam libraries + async.async_command + async_kernel + stdio + base.caml + yojson + lib + core_kernel + async + ppx_deriving_yojson.runtime + core + sexplib0 + result + base.base_internalhash_types + async_unix + ;; local libraries + visualization + mina_numbers + mina_state + transition_frontier_extensions + mina_base + logger.file_system + logger + cli_lib + transition_frontier + data_hash_lib + rose_tree + consensus + mina_wire_types + pickles.backend + snark_params + pickles + kimchi_pasta + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_jane + ppx_version + ppx_compare))) diff --git a/src/app/cli/src/cli_entrypoint/dune b/src/app/cli/src/cli_entrypoint/dune index 9c321cf212c..cefa0dc0d61 100644 --- a/src/app/cli/src/cli_entrypoint/dune +++ b/src/app/cli/src/cli_entrypoint/dune @@ -5,77 +5,84 @@ (public_name cli.mina_cli_entrypoint) (modes native) (libraries - ;;opam libraries - stdio - async_kernel - async_unix - base - core_kernel - core - bin_prot - bin_prot.shape - init - async - core.daemon - base.caml - uri - async.async_command - result - sexplib0 - ;;local libraries - verifier - ledger_proof - snark_worker - mina_block - pipe_lib - gossip_net - data_hash_lib - mina_ledger - precomputed_values - mina_networking - cache_dir - parallel - secrets - logger - mina_lib - currency - signature_lib - mina_base - mina_base.import - error_json - genesis_ledger_helper - consensus - child_processes - node_addrs_and_ports - mina_plugins - logger.file_system - cli_lib - genesis_constants - blake2 - mina_metrics - transaction_witness - snark_work_lib - transaction_snark - mina_compile_config - node_error_service - mina_user_error - file_system - mina_version - mina_net2 - block_time - mina_runtime_config - trust_system - protocol_version - coda_genesis_proof - with_hash - block_producer - prover - blockchain_snark - snarky.backendless - o1trace - ppx_version.runtime - internal_tracing - itn_logger - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_here ppx_let ppx_sexp_conv ppx_deriving_yojson))) + ;;opam libraries + stdio + async_kernel + async_unix + base + core_kernel + core + bin_prot + bin_prot.shape + init + async + core.daemon + base.caml + uri + async.async_command + result + sexplib0 + ;;local libraries + verifier + ledger_proof + snark_worker + mina_block + pipe_lib + gossip_net + data_hash_lib + mina_ledger + precomputed_values + mina_networking + cache_dir + parallel + secrets + logger + mina_lib + currency + signature_lib + mina_base + mina_base.import + error_json + genesis_ledger_helper + consensus + child_processes + node_addrs_and_ports + mina_plugins + logger.file_system + cli_lib + genesis_constants + blake2 + mina_metrics + transaction_witness + snark_work_lib + transaction_snark + mina_compile_config + node_error_service + mina_user_error + file_system + mina_version + mina_net2 + block_time + mina_runtime_config + trust_system + protocol_version + coda_genesis_proof + with_hash + block_producer + prover + blockchain_snark + snarky.backendless + o1trace + ppx_version.runtime + internal_tracing + itn_logger) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_here + ppx_let + ppx_sexp_conv + ppx_deriving_yojson))) diff --git a/src/app/cli/src/dune b/src/app/cli/src/dune index ce6717a5cb4..d88c44698a4 100644 --- a/src/app/cli/src/dune +++ b/src/app/cli/src/dune @@ -7,8 +7,10 @@ (modules mina) (modes native) (libraries mina_cli_entrypoint) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) (executable @@ -18,8 +20,10 @@ (modules mina_testnet_signatures) (modes native) (libraries mina_cli_entrypoint mina_signature_kind.testnet) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) (executable @@ -29,6 +33,8 @@ (modules mina_mainnet_signatures) (modes native) (libraries mina_cli_entrypoint mina_signature_kind.mainnet) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) diff --git a/src/app/cli/src/init/dune b/src/app/cli/src/init/dune index aaa7df52dba..00a4dee6b0a 100644 --- a/src/app/cli/src/init/dune +++ b/src/app/cli/src/init/dune @@ -1,135 +1,158 @@ (rule (targets assets.ml) - (deps (source_tree assets)) - (action (run %{bin:ocaml-crunch} -m plain assets -o assets.ml))) + (deps + (source_tree assets)) + (action + (run %{bin:ocaml-crunch} -m plain assets -o assets.ml))) (library (name init) (public_name init) (libraries - ;;opam libraries - stdio - integers - astring - async_ssl - sexplib0 - async.async_rpc - result - graphql - async_kernel - async_unix - uri - core_kernel - core_kernel.uuid - core - core.uuid - base - async - cohttp - graphql_parser - async.async_command - async_rpc_kernel - base.caml - base.base_internalhash_types - graphql-cohttp - cohttp-async - graphql-async - mirage-crypto-ec - base_quickcheck - ;;local libraries - bounded_types - snark_profiler_lib - archive_lib - mina_wire_types - perf_histograms - one_or_two - block_time - child_processes - mina_net2 - snark_worker - allocation_functor - blockchain_snark - genesis_ledger_helper.lib - transaction_inclusion_status - error_json - network_pool - user_command_input - network_peer - logger - itn_logger - coda_genesis_proof - data_hash_lib - currency - genesis_constants - bash_colors - graphql_lib - o1trace_webkit_event - generated_graphql_queries - itn_crypto - mina_transaction - mina_generators - mina_user_error - mina_ledger - mina_block - mina_lib - mina_base - cli_lib - mina_graphql - mina_state - consensus - mina_version - o1trace - protocol_version - node_error_service - genesis_ledger_helper - signature_lib - mina_base.import - mina_numbers - precomputed_values - with_hash - transaction_snark - transaction_snark_tests - transaction_protocol_state - test_util - file_system - daemon_rpcs - trust_system - secrets - participating_state - staged_ledger - mina_commands - mina_runtime_config - coda_genesis_ledger - mina_networking - sgn - unsigned_extended - mina_metrics - group_map - snark_work_lib - mina_compile_config - mina_transaction_logic - verifier - parallel - random_oracle - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - pickles_types - pickles - pickles.backend - snark_params - mina_signature_kind - string_sign - zkapp_command_builder - internal_tracing - transaction_snark_scan_state - ) - (instrumentation (backend bisect_ppx)) - (preprocessor_deps ../../../../../graphql_schema.json - ../../../../graphql-ppx-config.inc) - (preprocess (pps ppx_deriving_yojson ppx_mina ppx_version - ppx_base ppx_assert ppx_bench ppx_bin_prot ppx_custom_printf - ppx_fields_conv ppx_fixed_literal ppx_here ppx_inline_test ppx_let - ppx_module_timer ppx_optional ppx_pipebang ppx_sexp_message - ppx_sexp_value ppx_string ppx_typerep_conv ppx_variants_conv - graphql_ppx -- %{read-lines:../../../../graphql-ppx-config.inc}))) + ;;opam libraries + stdio + integers + astring + async_ssl + sexplib0 + async.async_rpc + result + graphql + async_kernel + async_unix + uri + core_kernel + core_kernel.uuid + core + core.uuid + base + async + cohttp + graphql_parser + async.async_command + async_rpc_kernel + base.caml + base.base_internalhash_types + graphql-cohttp + cohttp-async + graphql-async + mirage-crypto-ec + base_quickcheck + ;;local libraries + bounded_types + snark_profiler_lib + archive_lib + mina_wire_types + perf_histograms + one_or_two + block_time + child_processes + mina_net2 + snark_worker + allocation_functor + blockchain_snark + genesis_ledger_helper.lib + transaction_inclusion_status + error_json + network_pool + user_command_input + network_peer + logger + itn_logger + coda_genesis_proof + data_hash_lib + currency + genesis_constants + bash_colors + graphql_lib + o1trace_webkit_event + generated_graphql_queries + itn_crypto + mina_transaction + mina_generators + mina_user_error + mina_ledger + mina_block + mina_lib + mina_base + cli_lib + mina_graphql + mina_state + consensus + mina_version + o1trace + protocol_version + node_error_service + genesis_ledger_helper + signature_lib + mina_base.import + mina_numbers + precomputed_values + with_hash + transaction_snark + transaction_snark_tests + transaction_protocol_state + test_util + file_system + daemon_rpcs + trust_system + secrets + participating_state + staged_ledger + mina_commands + mina_runtime_config + coda_genesis_ledger + mina_networking + sgn + unsigned_extended + mina_metrics + group_map + snark_work_lib + mina_compile_config + mina_transaction_logic + verifier + parallel + random_oracle + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + pickles_types + pickles + pickles.backend + snark_params + mina_signature_kind + string_sign + zkapp_command_builder + internal_tracing + transaction_snark_scan_state) + (instrumentation + (backend bisect_ppx)) + (preprocessor_deps + ../../../../../graphql_schema.json + ../../../../graphql-ppx-config.inc) + (preprocess + (pps + ppx_deriving_yojson + ppx_mina + ppx_version + ppx_base + ppx_assert + ppx_bench + ppx_bin_prot + ppx_custom_printf + ppx_fields_conv + ppx_fixed_literal + ppx_here + ppx_inline_test + ppx_let + ppx_module_timer + ppx_optional + ppx_pipebang + ppx_sexp_message + ppx_sexp_value + ppx_string + ppx_typerep_conv + ppx_variants_conv + graphql_ppx + -- + %{read-lines:../../../../graphql-ppx-config.inc}))) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index af2363a72c9..90b52ac1f3a 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,49 +1,50 @@ (executable (name delegation_verify) (libraries - core_kernel - async - async_kernel - async_unix - core - stdio - base - base.caml - ppx_deriving_yojson.runtime - yojson - base64 - integers - async.async_command - sexplib0 - sexplib - hex - ; mina libs - signature_lib - mina_block - transaction_snark - blockchain_snark - mina_base - genesis_constants - uptime_service - currency - ledger_proof - mina_base.import - mina_state - mina_wire_types - consensus - data_hash_lib - mina_numbers - snark_params - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - pickles - pickles_types - pickles.backend - genesis_ledger_helper - mina_runtime_config - precomputed_values - logger - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) + core_kernel + async + async_kernel + async_unix + core + stdio + base + base.caml + ppx_deriving_yojson.runtime + yojson + base64 + integers + async.async_command + sexplib0 + sexplib + hex + ; mina libs + signature_lib + mina_block + transaction_snark + blockchain_snark + mina_base + genesis_constants + uptime_service + currency + ledger_proof + mina_base.import + mina_state + mina_wire_types + consensus + data_hash_lib + mina_numbers + snark_params + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + genesis_ledger_helper + mina_runtime_config + precomputed_values + logger) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/disk_caching_stats/dune b/src/app/disk_caching_stats/dune index 57292875db8..6a8db8e8632 100644 --- a/src/app/disk_caching_stats/dune +++ b/src/app/disk_caching_stats/dune @@ -1,11 +1,45 @@ (executable (name disk_caching_stats) - (libraries pickles pickles_types pickles.backend snark_params crypto_params network_pool mina_base - signature_lib one_or_two currency ledger_proof mina_state mina_base.import mina_wire_types - mina_numbers random_oracle random_oracle_input kimchi_pasta.basic data_hash_lib with_hash - kimchi_pasta mina_ledger transaction_snark_scan_state mina_transaction_logic transaction_snark - sgn snark_profiler_lib genesis_constants digestif bigarray-compat - ; -- - core core_kernel base base.caml bin_prot splittable_random sexplib0) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_jane ppx_version))) + (libraries + pickles + pickles_types + pickles.backend + snark_params + crypto_params + network_pool + mina_base + signature_lib + one_or_two + currency + ledger_proof + mina_state + mina_base.import + mina_wire_types + mina_numbers + random_oracle + random_oracle_input + kimchi_pasta.basic + data_hash_lib + with_hash + kimchi_pasta + mina_ledger + transaction_snark_scan_state + mina_transaction_logic + transaction_snark + sgn + snark_profiler_lib + genesis_constants + digestif + bigarray-compat + ; -- + core + core_kernel + base + base.caml + bin_prot + splittable_random + sexplib0) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_jane ppx_version))) diff --git a/src/app/dump_blocks/dump_blocks.ml b/src/app/dump_blocks/dump_blocks.ml index f1ca08be9d4..8ecc364d923 100644 --- a/src/app/dump_blocks/dump_blocks.ml +++ b/src/app/dump_blocks/dump_blocks.ml @@ -8,10 +8,7 @@ open Full_frontier.For_tests Choice of the actual codec module depends on more than one CLI argument, which is why it must be delayed until all the CLI arguments are parsed. This choice is made by mk_io function below. *) -type 'a io = - { encoding : 'a - ; filename : string - } +type 'a io = { encoding : 'a; filename : string } type 'a codec = (module Encoding.S with type t = 'a) @@ -41,15 +38,20 @@ let encoded_block = Command.Arg_type.create (fun s -> match String.split ~on:':' s with | [ encoding; filename ] -> - { encoding= + { encoding = ( match String.lowercase encoding with - | "sexp" -> Encoding.Sexp - | "json" -> Json - | "bin" | "binary" -> Binary - | _ -> failwith "invalid encoding" ) + | "sexp" -> + Encoding.Sexp + | "json" -> + Json + | "bin" | "binary" -> + Binary + | _ -> + failwith "invalid encoding" ) ; filename } - | _ -> failwith "invalid format" ) + | _ -> + failwith "invalid format" ) let () = let open Core_kernel in @@ -62,18 +64,14 @@ let precomputed_values = Lazy.force Precomputed_values.for_unit_tests let constraint_constants = precomputed_values.constraint_constants -let output_block : type a. a -> a codec io -> unit - = fun content { encoding; filename } -> +let output_block : type a. a -> a codec io -> unit = + fun content { encoding; filename } -> let module Enc : Encoding.S with type t = a = (val encoding) in let channel = - match filename with - | "-" -> Out_channel.stdout - | s -> Out_channel.create s + match filename with "-" -> Out_channel.stdout | s -> Out_channel.create s in - fprintf channel "%s" (Enc.to_string content); - match filename with - | "-" -> () - | _ -> Out_channel.close channel + fprintf channel "%s" (Enc.to_string content) ; + match filename with "-" -> () | _ -> Out_channel.close channel (* This executable outputs random block to stderr in sexp and json The output is useful for src/lib/mina_block tests when the sexp/json representation changes. *) @@ -91,75 +89,67 @@ let f (type a) ?parent (outputs : a codec io list) make_breadcrumb = List.iter outputs ~f:(fun output -> let module Enc = (val output.encoding) in let content = - Enc.of_breadcrumb - ?with_parent_statehash:parent - breadcrumb + Enc.of_breadcrumb ?with_parent_statehash:parent breadcrumb in - eprintf !"Randomly generated block, %s: %s\n" + eprintf + !"Randomly generated block, %s: %s\n" Enc.name (match output.filename with "-" -> "" | s -> s) ; - output_block content output;); + output_block content output ) ; clean_up_persistent_root ~frontier ) let default_outputs = - [ { encoding= Encoding.Sexp; filename= "-" } - ; { encoding= Encoding.Json; filename= "-" } + [ { encoding = Encoding.Sexp; filename = "-" } + ; { encoding = Encoding.Json; filename = "-" } ] let command = Command.basic ~summary:"Transcribe between block formats or generate a random block" (let%map_open.Command outputs = - flag - "-o" - (listed encoded_block) - ~doc:"output :; formats are: sexp / json / bin; a dash (-) denotes stdout (example: -o json:-)" + flag "-o" (listed encoded_block) + ~doc: + "output :; formats are: sexp / json / bin; a dash (-) \ + denotes stdout (example: -o json:-)" and input = - flag - "-i" - (optional encoded_block) - ~doc:"input :; formats are: sexp / json / bin; a dash (-) denotes stdin (example: -i json:-)" + flag "-i" (optional encoded_block) + ~doc: + "input :; formats are: sexp / json / bin; a dash (-) \ + denotes stdin (example: -i json:-)" and full = - flag - "--full" - no_arg - ~doc:"use full blocks, not just precomputed values" + flag "--full" no_arg ~doc:"use full blocks, not just precomputed values" and parent = - flag - "--parent" - (optional string) - ~aliases:[ "--parent-statehash" ] + flag "--parent" (optional string) ~aliases:[ "--parent-statehash" ] ~doc:"statehash of the parent block" in - let outs = match outputs with - | [] -> default_outputs - | _ -> outputs - in + let outs = match outputs with [] -> default_outputs | _ -> outputs in let conf = if full then - Conf ( Option.map input ~f:(mk_io Block) - , List.map ~f:(mk_io Encoding.Block) outs) + Conf + ( Option.map input ~f:(mk_io Block) + , List.map ~f:(mk_io Encoding.Block) outs ) else - Conf ( Option.map input ~f:(mk_io Precomputed) - , List.map ~f:(mk_io Encoding.Precomputed) outs) + Conf + ( Option.map input ~f:(mk_io Precomputed) + , List.map ~f:(mk_io Encoding.Precomputed) outs ) in fun () -> - match conf with - | Conf (None, outs) -> - let verifier = verifier () in - Core_kernel.Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:1 ~f:(f ?parent outs) - | Conf (Some { encoding; filename }, outs)-> - let module Enc = (val encoding) in - let input = - match filename with - | "-" -> In_channel.stdin - | s -> In_channel.create s - in - let contents = - In_channel.input_all input - |> Enc.of_string - in - List.iter outs ~f:(output_block contents)) + match conf with + | Conf (None, outs) -> + let verifier = verifier () in + Core_kernel.Quickcheck.test + (gen_breadcrumb ~verifier ()) + ~trials:1 ~f:(f ?parent outs) + | Conf (Some { encoding; filename }, outs) -> + let module Enc = (val encoding) in + let input = + match filename with + | "-" -> + In_channel.stdin + | s -> + In_channel.create s + in + let contents = In_channel.input_all input |> Enc.of_string in + List.iter outs ~f:(output_block contents) ) let () = Command_unix.run command - diff --git a/src/app/dump_blocks/dune b/src/app/dump_blocks/dune index 1e499797f02..4c403c5a92d 100644 --- a/src/app/dump_blocks/dune +++ b/src/app/dump_blocks/dune @@ -2,27 +2,29 @@ (name dump_blocks) (public_name dump_blocks) (libraries - async - async_kernel - async_unix - base - block_time - bounded_types - core - core_kernel - core_unix.command_unix - logger - mina_base - mina_block - mina_state - mina_wire_types - ppx_inline_test.runner.lib - precomputed_values - sexplib0 - transition_frontier - transition_frontier_base - transition_frontier_full_frontier - yojson) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_custom_printf)) + async + async_kernel + async_unix + base + block_time + bounded_types + core + core_kernel + core_unix.command_unix + logger + mina_base + mina_block + mina_state + mina_wire_types + ppx_inline_test.runner.lib + precomputed_values + sexplib0 + transition_frontier + transition_frontier_base + transition_frontier_full_frontier + yojson) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_custom_printf)) (modes native)) diff --git a/src/app/dump_blocks/encoding.ml b/src/app/dump_blocks/encoding.ml index 1a246a1a2b9..dba55a5b52a 100644 --- a/src/app/dump_blocks/encoding.ml +++ b/src/app/dump_blocks/encoding.ml @@ -12,52 +12,49 @@ let block_of_breadcrumb ?with_parent_statehash breadcrumb = let open Mina_block in let block = Frontier_base.Breadcrumb.block breadcrumb in match with_parent_statehash with - | None -> - block - | Some hash -> - let previous_state_hash = Mina_base.State_hash.of_base58_check_exn hash in - let h = header block in - let state = Header.protocol_state h in - let header = - Header.create - ~protocol_state:{ state with previous_state_hash } - ~protocol_state_proof:(Header.protocol_state_proof h) - ~delta_block_chain_proof:(Header.delta_block_chain_proof h) - ?proposed_protocol_version_opt:(Header.proposed_protocol_version_opt h) - ~current_protocol_version:(Header.current_protocol_version h) - () - in - Mina_block.create ~header ~body:(body block) + | None -> + block + | Some hash -> + let previous_state_hash = Mina_base.State_hash.of_base58_check_exn hash in + let h = header block in + let state = Header.protocol_state h in + let header = + Header.create + ~protocol_state:{ state with previous_state_hash } + ~protocol_state_proof:(Header.protocol_state_proof h) + ~delta_block_chain_proof:(Header.delta_block_chain_proof h) + ?proposed_protocol_version_opt: + (Header.proposed_protocol_version_opt h) + ~current_protocol_version:(Header.current_protocol_version h) + () + in + Mina_block.create ~header ~body:(body block) module type S = sig type t val name : string - val of_breadcrumb : ?with_parent_statehash:string -> Frontier_base.Breadcrumb.t -> t + + val of_breadcrumb : + ?with_parent_statehash:string -> Frontier_base.Breadcrumb.t -> t val to_string : t -> string + val of_string : string -> t end -module Sexp_block : S with type t = Mina_block.t = struct +module Sexp_block : S with type t = Mina_block.t = struct type t = Mina_block.t let name = "sexp" let of_breadcrumb = block_of_breadcrumb + let to_string b = Mina_block.sexp_of_t b |> Sexp.to_string |> append_newline - let to_string b = - Mina_block.sexp_of_t b - |> Sexp.to_string - |> append_newline - - let of_string s = - Sexp.of_string s - |> Mina_block.t_of_sexp + let of_string s = Sexp.of_string s |> Mina_block.t_of_sexp end - module Binary_block : S with type t = Mina_block.t = struct type t = Mina_block.t @@ -84,16 +81,14 @@ let constraint_constants = precomputed_values.constraint_constants let precomputed_of_breadcrumb ?with_parent_statehash breadcrumb = let open Frontier_base in let block = block_of_breadcrumb ?with_parent_statehash breadcrumb in - let staged_ledger = - Transition_frontier.Breadcrumb.staged_ledger breadcrumb - in + let staged_ledger = Transition_frontier.Breadcrumb.staged_ledger breadcrumb in let scheduled_time = Mina_block.(Header.protocol_state @@ header block) |> Mina_state.Protocol_state.blockchain_state |> Mina_state.Blockchain_state.timestamp in - Mina_block.Precomputed.of_block ~logger ~constraint_constants - ~staged_ledger ~scheduled_time + Mina_block.Precomputed.of_block ~logger ~constraint_constants ~staged_ledger + ~scheduled_time (Breadcrumb.block_with_hash breadcrumb) module Sexp_precomputed : S with type t = Mina_block.Precomputed.t = struct @@ -104,16 +99,12 @@ module Sexp_precomputed : S with type t = Mina_block.Precomputed.t = struct let of_breadcrumb = precomputed_of_breadcrumb let to_string b = - Mina_block.Precomputed.sexp_of_t b - |> Sexp.to_string - |> append_newline + Mina_block.Precomputed.sexp_of_t b |> Sexp.to_string |> append_newline - let of_string s = - Sexp.of_string s - |> Mina_block.Precomputed.t_of_sexp + let of_string s = Sexp.of_string s |> Mina_block.Precomputed.t_of_sexp end -module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct +module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct type t = Mina_block.Precomputed.t let name = "json" @@ -122,12 +113,10 @@ module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct let to_string b = Mina_block.Precomputed.to_yojson b - |> Yojson.Safe.to_string - |> append_newline + |> Yojson.Safe.to_string |> append_newline let of_string s = - Yojson.Safe.from_string s - |> Mina_block.Precomputed.of_yojson + Yojson.Safe.from_string s |> Mina_block.Precomputed.of_yojson |> Result.ok_or_failwith end @@ -138,9 +127,7 @@ module Binary_precomputed : S with type t = Mina_block.Precomputed.t = struct let of_breadcrumb = precomputed_of_breadcrumb - let to_string = - Binable.to_string (module Mina_block.Precomputed.Stable.Latest) + let to_string = Binable.to_string (module Mina_block.Precomputed.Stable.Latest) - let of_string = - Binable.of_string (module Mina_block.Precomputed.Stable.Latest) + let of_string = Binable.of_string (module Mina_block.Precomputed.Stable.Latest) end diff --git a/src/app/extract_blocks/dune b/src/app/extract_blocks/dune index a41ea72fc19..1b31d5f0471 100644 --- a/src/app/extract_blocks/dune +++ b/src/app/extract_blocks/dune @@ -3,47 +3,55 @@ (name extract_blocks) (public_name extract_blocks) (libraries - ;; opam libraries - async_unix - core - base - base64 - integers - caqti-async - caqti-driver-postgresql - async - core_kernel - caqti - async_kernel - uri - async.async_command - ;; local libraries - consensus.vrf - mina_wire_types - bounded_types - mina_base - mina_base.import - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - pickles - pickles_types - pickles.backend - archive_lib - data_hash_lib - signature_lib - logger - mina_caqti - block_time - mina_numbers - mina_transaction - protocol_version - snark_params - currency - unsigned_extended - sgn - with_hash - error_json - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv h_list.ppx))) + ;; opam libraries + async_unix + core + base + base64 + integers + caqti-async + caqti-driver-postgresql + async + core_kernel + caqti + async_kernel + uri + async.async_command + ;; local libraries + consensus.vrf + mina_wire_types + bounded_types + mina_base + mina_base.import + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + archive_lib + data_hash_lib + signature_lib + logger + mina_caqti + block_time + mina_numbers + mina_transaction + protocol_version + snark_params + currency + unsigned_extended + sgn + with_hash + error_json) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_version + ppx_mina + ppx_let + ppx_hash + ppx_compare + ppx_sexp_conv + h_list.ppx))) diff --git a/src/app/generate_keypair/dune b/src/app/generate_keypair/dune index 1a987e63659..6ef13957692 100644 --- a/src/app/generate_keypair/dune +++ b/src/app/generate_keypair/dune @@ -4,6 +4,8 @@ (public_name generate_keypair) (modes native) (libraries cli_lib async core_kernel mina_version async_unix bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_sexp_conv)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) diff --git a/src/app/graphql_schema_dump/dune b/src/app/graphql_schema_dump/dune index 7116105997b..6294dcf3ae4 100644 --- a/src/app/graphql_schema_dump/dune +++ b/src/app/graphql_schema_dump/dune @@ -1,5 +1,16 @@ (executable (name graphql_schema_dump) - (libraries graphql_parser mina_version mina_graphql async async_unix graphql-async yojson bounded_types genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + graphql_parser + mina_version + mina_graphql + async + async_unix + graphql-async + yojson + bounded_types + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/heap_usage/dune b/src/app/heap_usage/dune index a74b285b81f..e9b41204cb6 100644 --- a/src/app/heap_usage/dune +++ b/src/app/heap_usage/dune @@ -3,57 +3,58 @@ (name heap_usage) (public_name heap_usage) (libraries - ;; opam libraries - async_unix - core - result - async_kernel - stdio - async - core_kernel - base - base.caml - base.base_internalhash_types - async.async_command - yojson - sexplib0 - ;; local libraries - mina_block ; temp - mina_base - mina_base.import - mina_numbers - currency - bounded_types - data_hash_lib - signature_lib - merkle_ledger - mina_compile_config - mina_ledger - mina_state - mina_transaction_logic - genesis_constants - staged_ledger_diff - parallel_scan - pickles - pickles.backend - pickles_types - transaction_snark - transaction_snark_scan_state - ledger_proof - blake2 - mina_generators - with_hash - random_oracle - snark_params - crypto_params - kimchi_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - zkapp_command_builder - transaction_snark_tests - snark_profiler_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + async_unix + core + result + async_kernel + stdio + async + core_kernel + base + base.caml + base.base_internalhash_types + async.async_command + yojson + sexplib0 + ;; local libraries + mina_block ; temp + mina_base + mina_base.import + mina_numbers + currency + bounded_types + data_hash_lib + signature_lib + merkle_ledger + mina_compile_config + mina_ledger + mina_state + mina_transaction_logic + genesis_constants + staged_ledger_diff + parallel_scan + pickles + pickles.backend + pickles_types + transaction_snark + transaction_snark_scan_state + ledger_proof + blake2 + mina_generators + with_hash + random_oracle + snark_params + crypto_params + kimchi_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + zkapp_command_builder + transaction_snark_tests + snark_profiler_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) diff --git a/src/app/ledger_export_bench/dune b/src/app/ledger_export_bench/dune index 2352ffd4130..6ee4c2a05c7 100644 --- a/src/app/ledger_export_bench/dune +++ b/src/app/ledger_export_bench/dune @@ -1,15 +1,17 @@ (executable (name ledger_export_benchmark) (libraries - base - core - core_bench - core_kernel - yojson - ; mina libraries - mina_runtime_config) + base + core + core_bench + core_kernel + yojson + ; mina libraries + mina_runtime_config) ; the -w list here should be the same as in src/dune (flags -short-paths -g -w @a-4-29-40-41-42-44-45-48-58-59-60) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane)) (modes native)) diff --git a/src/app/logproc/dune b/src/app/logproc/dune index 4d38e6278e4..77ac3b6d14e 100644 --- a/src/app/logproc/dune +++ b/src/app/logproc/dune @@ -3,20 +3,21 @@ (public_name logproc) (modules logproc) (libraries - ;;opam libraries - result - stdio - core - yojson - cmdliner - core_kernel - ppx_deriving_yojson.runtime - ;;local libraries - logger - logproc_lib - bounded_types - bash_colors - interpolator_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_deriving.std))) + ;;opam libraries + result + stdio + core + yojson + cmdliner + core_kernel + ppx_deriving_yojson.runtime + ;;local libraries + logger + logproc_lib + bounded_types + bash_colors + interpolator_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane ppx_deriving.std))) diff --git a/src/app/logproc/logproc.ml b/src/app/logproc/logproc.ml index 8c519b962f1..44a1f7d3fa9 100644 --- a/src/app/logproc/logproc.ml +++ b/src/app/logproc/logproc.ml @@ -81,8 +81,9 @@ let process_line ~timezone ~interpolation_config ~filter line = type parser_state = | Start_of_line - | Processable_line of {line_start_pos: int; previous_elements: string list} - | Unprocessable_line of {line_start_pos: int} + | Processable_line of + { line_start_pos : int; previous_elements : string list } + | Unprocessable_line of { line_start_pos : int } (* Iterates over lines prefixed by `prefix`, invoking `on_hit` for each * line. If a line does not start with `prefix`, `on_miss` is invoked, @@ -105,11 +106,11 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit match state with | Start_of_line -> failwith "cannot extract start of line state" - | Processable_line {line_start_pos; previous_elements} -> + | Processable_line { line_start_pos; previous_elements } -> let this_element = extract_input_string line_start_pos i in let elements = List.rev (this_element :: previous_elements) in on_hit (String.concat elements ~sep:"") - | Unprocessable_line {line_start_pos} -> + | Unprocessable_line { line_start_pos } -> forward_input line_start_pos i ; Out_channel.output_char output_channel '\n' in @@ -117,14 +118,15 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit match state with | Start_of_line -> Start_of_line - | Processable_line {line_start_pos; previous_elements} -> + | Processable_line { line_start_pos; previous_elements } -> let this_element = extract_input_string line_start_pos buf_len in Processable_line - { line_start_pos= 0 - ; previous_elements= this_element :: previous_elements } - | Unprocessable_line {line_start_pos} -> + { line_start_pos = 0 + ; previous_elements = this_element :: previous_elements + } + | Unprocessable_line { line_start_pos } -> forward_input line_start_pos buf_len ; - Unprocessable_line {line_start_pos= 0} + Unprocessable_line { line_start_pos = 0 } in let parse_char ~buf_len i state = if i >= buf_len then `Stop (carryover_state ~buf_len state) @@ -133,10 +135,10 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit | Start_of_line -> let next_state = if Char.equal (Bytes.unsafe_get input_buffer i) prefix then - Processable_line {line_start_pos= i; previous_elements= []} + Processable_line { line_start_pos = i; previous_elements = [] } else ( on_miss () ; - Unprocessable_line {line_start_pos= i} ) + Unprocessable_line { line_start_pos = i } ) in `Continue next_state | Processable_line _ | Unprocessable_line _ -> @@ -209,9 +211,9 @@ let () = Arg.( value & opt - (enum [("hidden", Hidden); ("inline", Inline); ("after", After)]) + (enum [ ("hidden", Hidden); ("inline", Inline); ("after", After) ]) Inline - & info ["i"; "interpolation-mode"] ~docv:"MODE" ~doc) + & info [ "i"; "interpolation-mode" ] ~docv:"MODE" ~doc) in let max_interpolation_length = let doc = @@ -221,16 +223,17 @@ let () = Arg.( value & opt int 25 & info - ["m"; "max-interpolation-length"] + [ "m"; "max-interpolation-length" ] ~docv:"MAX_INTERPOLATION_LENGTH" ~doc) in let pretty_print = let doc = "Pretty print json values." in - Arg.(value & flag & info ["p"; "pretty-print"] ~docv:"PRETTY_PRINT" ~doc) + Arg.( + value & flag & info [ "p"; "pretty-print" ] ~docv:"PRETTY_PRINT" ~doc) in let lift_interpolation_config mode max_interpolation_length pretty_print = let open Interpolator in - {mode; max_interpolation_length; pretty_print} + { mode; max_interpolation_length; pretty_print } in Term.( const lift_interpolation_config @@ -240,23 +243,23 @@ let () = let doc = "Timezone to display timestamps in. Defaults to the system's timezone." in - Arg.(value & opt string "" & info ["z"; "zone"] ~docv:"TIMEZONE" ~doc) + Arg.(value & opt string "" & info [ "z"; "zone" ] ~docv:"TIMEZONE" ~doc) in let filter = let doc = "Filter displayed log lines. The filter language has similar syntax to \ - javascript, with a few notable differences. Similar to \"jq\", doing \ - an anymous access like \"[\"a\"]\" or \".a\" will refer to that key on \ - the javascript object being processed, which in the context of \ - logproc, is the json log entry itself. Basic literals (such as strings \ - and integers) are supported, as is structural equality (\"==\") and \ - boolean operations (\"!\", \"&&\", \"||\"). There is also support for \ - the \"in\" operator for checking existence in arrays (works like \ + javascript, with a few notable differences. Similar to \"jq\", doing an \ + anymous access like \"[\"a\"]\" or \".a\" will refer to that key on the \ + javascript object being processed, which in the context of logproc, is \ + the json log entry itself. Basic literals (such as strings and \ + integers) are supported, as is structural equality (\"==\") and boolean \ + operations (\"!\", \"&&\", \"||\"). There is also support for the \ + \"in\" operator for checking existence in arrays (works like \ javascript, not like jq). Regexes can also be expressed \ (\"/some_regex/\") and can be matched on using a special \"match\" \ operator. See examples for more information." in - Arg.(value & opt string "" & info ["f"; "filter"] ~docv:"FILTER" ~doc) + Arg.(value & opt string "" & info [ "f"; "filter" ] ~docv:"FILTER" ~doc) in let main_term = Term.(const main $ timezone $ interpolation_config $ filter) @@ -289,7 +292,8 @@ let () = ; `I ( "a complex filter:" , {|logproc -f '.message match /broadcast/ && .metadata.peer.host == "182.9.63.3" || .metadata.peer.discover_port == 8302|} - ) ] + ) + ] in Term.info ~version:"0.1" ~doc ~exits:Term.default_exits ~man "logproc" in diff --git a/src/app/missing_blocks_auditor/dune b/src/app/missing_blocks_auditor/dune index 49e9020bc1c..a08801fafa2 100644 --- a/src/app/missing_blocks_auditor/dune +++ b/src/app/missing_blocks_auditor/dune @@ -3,21 +3,22 @@ (name missing_blocks_auditor) (public_name missing_blocks_auditor) (libraries - ;; opam libraries - core - async_kernel - uri - caqti-driver-postgresql - caqti - async - core_kernel - caqti-async - base - async_unix - async.async_command - ;; local libraries - logger - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let))) + ;; opam libraries + core + async_kernel + uri + caqti-driver-postgresql + caqti + async + core_kernel + caqti-async + base + async_unix + async.async_command + ;; local libraries + logger + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_let))) diff --git a/src/app/print_blockchain_snark_vk/dune b/src/app/print_blockchain_snark_vk/dune index 28872e1df1e..ad5287fc4fd 100644 --- a/src/app/print_blockchain_snark_vk/dune +++ b/src/app/print_blockchain_snark_vk/dune @@ -1,19 +1,23 @@ (executable (name print_blockchain_snark_vk) - (libraries - blockchain_snark genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries blockchain_snark genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (rule (deps print_blockchain_snark_vk.exe) (targets blockchain_snark_vk.json.computed) (action - (with-stdout-to %{targets} + (with-stdout-to + %{targets} (run %{deps})))) (rule - (deps (:orig blockchain_snark_vk.json) (:computed blockchain_snark_vk.json.computed)) + (deps + (:orig blockchain_snark_vk.json) + (:computed blockchain_snark_vk.json.computed)) (alias runtest) (action (diff %{orig} %{computed}))) diff --git a/src/app/reformat/dune b/src/app/reformat/dune index e069e49a5da..c0fbc0fa245 100644 --- a/src/app/reformat/dune +++ b/src/app/reformat/dune @@ -1,7 +1,14 @@ (executable (name reformat) (public_name reformat) - (libraries core async async_kernel async_unix core_kernel sexplib0 async.async_command) + (libraries + core + async + async_kernel + async_unix + core_kernel + sexplib0 + async.async_command) (preprocess (pps ppx_jane)) ; the -w list here should be the same as in src/dune diff --git a/src/app/reformat/reformat.ml b/src/app/reformat/reformat.ml index 0ea294650e1..e71dad30c28 100644 --- a/src/app/reformat/reformat.ml +++ b/src/app/reformat/reformat.ml @@ -19,7 +19,8 @@ let dirs_trustlist = ; "snarky" ; "_opam" ; "opam_switches" - ; ".direnv" ] + ; ".direnv" + ] let rec fold_over_files ~path ~process_path ~init ~f = let%bind all = Sys.ls_dir path in @@ -42,11 +43,11 @@ let main dry_run check path = | `Dir -> not (List.exists dirs_trustlist ~f:(fun s -> - String.is_suffix ~suffix:s path )) + String.is_suffix ~suffix:s path ) ) | `File -> (not (List.exists trustlist ~f:(fun s -> - String.is_suffix ~suffix:s path ))) + String.is_suffix ~suffix:s path ) ) ) && ( String.is_suffix ~suffix:".ml" path || String.is_suffix ~suffix:".mli" path ) ) ~f:(fun () file -> @@ -55,7 +56,7 @@ let main dry_run check path = return () in if check then - let prog, args = ("ocamlformat", ["--doc-comments=before"; file]) in + let prog, args = ("ocamlformat", [ "--doc-comments=before"; file ]) in let%bind formatted = Process.run_exn ~prog ~args () in let%bind raw = Reader.file_contents file in if not (String.equal formatted raw) then ( @@ -64,7 +65,7 @@ let main dry_run check path = else return () else let prog, args = - ("ocamlformat", ["--doc-comments=before"; "-i"; file]) + ("ocamlformat", [ "--doc-comments=before"; "-i"; file ]) in if dry_run then dump prog args else @@ -77,16 +78,16 @@ let _cli = let open Command.Let_syntax in Command.async ~summary:"Format all ml and mli files" (let%map_open path = - flag "--path" ~aliases:["path"] ~doc:"Path to traverse" + flag "--path" ~aliases:[ "path" ] ~doc:"Path to traverse" (required string) - and dry_run = flag "--dry-run" ~aliases:["dry-run"] no_arg ~doc:"Dry run" + and dry_run = flag "--dry-run" ~aliases:[ "dry-run" ] no_arg ~doc:"Dry run" and check = - flag "--check" ~aliases:["check"] no_arg + flag "--check" ~aliases:[ "check" ] no_arg ~doc: "Return with an error code if there exists an ml file that was \ formatted improperly" in - fun () -> main dry_run check path) + fun () -> main dry_run check path ) |> Command.run let () = never_returns (Scheduler.go ()) diff --git a/src/app/replayer/dune b/src/app/replayer/dune index 394d304b180..8cd2e071d22 100644 --- a/src/app/replayer/dune +++ b/src/app/replayer/dune @@ -3,66 +3,74 @@ (name replayer) (public_name replayer) (libraries - ;; opam libraries - stdio - sexplib0 - result - async.async_command - async - base.caml - bin_prot.shape - caqti-async - core_kernel - archive_lib - caqti-driver-postgresql - async_kernel - core - yojson - caqti - base - base.base_internalhash_types - uri - integers - async_unix - ;; local libraries - mina_wire_types - with_hash - snark_params - crypto_params - pickles_types - mina_numbers - coda_genesis_ledger - currency - pickles.backend - signature_lib - genesis_ledger_helper.lib - genesis_ledger_helper - mina_state - mina_ledger - mina_base - bounded_types - mina_base.import - mina_transaction - mina_transaction_logic - mina_caqti - genesis_constants - mina_runtime_config - logger - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - kimchi_backend_common - block_time - data_hash_lib - pickles - sgn - unsigned_extended - random_oracle - codable - ppx_version.runtime - cli_lib - mina_version - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx))) + ;; opam libraries + stdio + sexplib0 + result + async.async_command + async + base.caml + bin_prot.shape + caqti-async + core_kernel + archive_lib + caqti-driver-postgresql + async_kernel + core + yojson + caqti + base + base.base_internalhash_types + uri + integers + async_unix + ;; local libraries + mina_wire_types + with_hash + snark_params + crypto_params + pickles_types + mina_numbers + coda_genesis_ledger + currency + pickles.backend + signature_lib + genesis_ledger_helper.lib + genesis_ledger_helper + mina_state + mina_ledger + mina_base + bounded_types + mina_base.import + mina_transaction + mina_transaction_logic + mina_caqti + genesis_constants + mina_runtime_config + logger + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + kimchi_backend_common + block_time + data_hash_lib + pickles + sgn + unsigned_extended + random_oracle + codable + ppx_version.runtime + cli_lib + mina_version) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/rosetta/dune b/src/app/rosetta/dune index 83be7c2bc27..d1120e13d02 100644 --- a/src/app/rosetta/dune +++ b/src/app/rosetta/dune @@ -4,10 +4,19 @@ (public_name rosetta) (modules rosetta) (modes native) - (libraries lib async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package rosetta) @@ -15,10 +24,20 @@ (public_name rosetta-testnet) (modules rosetta_testnet_signatures) (modes native) - (libraries lib mina_signature_kind.testnet async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + mina_signature_kind.testnet + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package rosetta) @@ -26,7 +45,17 @@ (public_name rosetta-mainnet) (modules rosetta_mainnet_signatures) (modes native) - (libraries lib mina_signature_kind.mainnet async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + mina_signature_kind.mainnet + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/rosetta/lib/dune b/src/app/rosetta/lib/dune index 9c6bf25cd02..7515da8d47c 100644 --- a/src/app/rosetta/lib/dune +++ b/src/app/rosetta/lib/dune @@ -51,8 +51,7 @@ hex mina_numbers mina_transaction - mina_version - ) + mina_version) (preprocessor_deps ../../../graphql-ppx-config.inc ../../../../graphql_schema.json) diff --git a/src/app/rosetta/ocaml-signer/dune b/src/app/rosetta/ocaml-signer/dune index f72365e98d9..f4ac9458523 100644 --- a/src/app/rosetta/ocaml-signer/dune +++ b/src/app/rosetta/ocaml-signer/dune @@ -3,40 +3,39 @@ (public_name signer.cli) (modules signer_cli) (libraries - ;;opam libraries - async_kernel - core - core_kernel - async - base - lib - async_unix - async.async_command - ;;local libraries - kimchi_pasta - kimchi_pasta.basic - pickles - rosetta_coding - rosetta_lib - string_sign - signature_lib - mina_base - snark_params - pickles.backend - secrets - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + ;;opam libraries + async_kernel + core + core_kernel + async + base + lib + async_unix + async.async_command + ;;local libraries + kimchi_pasta + kimchi_pasta.basic + pickles + rosetta_coding + rosetta_lib + string_sign + signature_lib + mina_base + snark_params + pickles.backend + secrets) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -45,18 +44,18 @@ (modes native) (modules signer) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -65,18 +64,18 @@ (modes native) (modules signer_testnet_signatures) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -85,15 +84,15 @@ (modes native) (modules signer_mainnet_signatures) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) diff --git a/src/app/rosetta/scripts/berkeley.env b/src/app/rosetta/scripts/berkeley.env deleted file mode 100644 index d64c3057ffe..00000000000 --- a/src/app/rosetta/scripts/berkeley.env +++ /dev/null @@ -1,5 +0,0 @@ -MINA_NETWORK=berkeley -PEER_LIST_URL=https://bootnodes.minaprotocol.com/networks/berkeley.txt -MINA_ARCHIVE_DUMP_URL=https://storage.googleapis.com/mina-archive-dumps -MINA_GENESIS_LEDGER_URL=https://673156464838-mina-genesis-ledgers.s3.us-west-2.amazonaws.com/berkeley/genesis_ledger.json -BLOCKS_BUCKET=https://storage.googleapis.com/mina_network_block_data diff --git a/src/app/runtime_genesis_ledger/dune b/src/app/runtime_genesis_ledger/dune index 0a1191a086d..a6dba29e46b 100644 --- a/src/app/runtime_genesis_ledger/dune +++ b/src/app/runtime_genesis_ledger/dune @@ -2,28 +2,29 @@ (name runtime_genesis_ledger) (public_name runtime_genesis_ledger) (libraries - ;;opam libraries - base - core_kernel - async_kernel - async - core - result - async.async_command - yojson - async_unix - ;;local libraries - genesis_constants - mina_runtime_config - mina_ledger - genesis_ledger_helper - bounded_types - logger - cache_dir - precomputed_values - coda_genesis_ledger - consensus - mina_base - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let ppx_deriving_yojson))) + ;;opam libraries + base + core_kernel + async_kernel + async + core + result + async.async_command + yojson + async_unix + ;;local libraries + genesis_constants + mina_runtime_config + mina_ledger + genesis_ledger_helper + bounded_types + logger + cache_dir + precomputed_values + coda_genesis_ledger + consensus + mina_base) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_let ppx_deriving_yojson))) diff --git a/src/app/snark_work_debugger/dune b/src/app/snark_work_debugger/dune index 0f8b91aaea9..9eda39a4279 100644 --- a/src/app/snark_work_debugger/dune +++ b/src/app/snark_work_debugger/dune @@ -4,33 +4,40 @@ (public_name snark_work_debugger) (modes native) (libraries - ;; opam libraries - base - async - async_kernel - async_unix - unix - core - core_kernel - sexplib0 - ;; local libraries - mina_base - snark_worker - bounded_types - snark_work_lib - transaction_witness - transaction_snark - genesis_constants - logger - error_json - transaction_protocol_state - mina_state - currency - mina_ledger - signature_lib - mina_base.import - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_here ppx_custom_printf ppx_version ppx_mina ppx_let ppx_deriving_yojson ppx_sexp_conv)) + ;; opam libraries + base + async + async_kernel + async_unix + unix + core + core_kernel + sexplib0 + ;; local libraries + mina_base + snark_worker + bounded_types + snark_work_lib + transaction_witness + transaction_snark + genesis_constants + logger + error_json + transaction_protocol_state + mina_state + currency + mina_ledger + signature_lib + mina_base.import) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_here + ppx_custom_printf + ppx_version + ppx_mina + ppx_let + ppx_deriving_yojson + ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) - diff --git a/src/app/validate_keypair/dune b/src/app/validate_keypair/dune index 78e187b0fa9..4346b5856f5 100644 --- a/src/app/validate_keypair/dune +++ b/src/app/validate_keypair/dune @@ -4,15 +4,16 @@ (public_name validate_keypair) (modes native) (libraries - ;; opam libraries - core_kernel - async - async_unix - ;; local libraries - mina_version - bounded_types - cli_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_sexp_conv)) + ;; opam libraries + core_kernel + async + async_unix + ;; local libraries + mina_version + bounded_types + cli_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) diff --git a/src/app/zkapp_limits/dune b/src/app/zkapp_limits/dune index 6cb781b71b2..2c50a3c87ac 100644 --- a/src/app/zkapp_limits/dune +++ b/src/app/zkapp_limits/dune @@ -3,14 +3,22 @@ (name zkapp_limits) (public_name zkapp_limits) (libraries - ;; opam libraries - base - base.caml - core_kernel - ;; local libraries - mina_base - genesis_constants - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_custom_printf ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + base + base.caml + core_kernel + ;; local libraries + mina_base + genesis_constants + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_version + ppx_mina + ppx_custom_printf + ppx_let + ppx_hash + ppx_compare + ppx_sexp_conv))) diff --git a/src/app/zkapp_test_transaction/dune b/src/app/zkapp_test_transaction/dune index ca1edcbdd3c..c81e6d48a82 100644 --- a/src/app/zkapp_test_transaction/dune +++ b/src/app/zkapp_test_transaction/dune @@ -3,24 +3,34 @@ (name zkapp_test_transaction) (public_name zkapp_test_transaction) (libraries - ;;opam libraries - async.async_command - base - core_kernel - core - async_kernel - zkapp_test_transaction_lib - async - async_unix - ;;local libraries - mina_graphql - mina_numbers - currency - bounded_types - cli_lib - mina_base - signature_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_custom_printf ppx_mina ppx_assert ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx))) + ;;opam libraries + async.async_command + base + core_kernel + core + async_kernel + zkapp_test_transaction_lib + async + async_unix + ;;local libraries + mina_graphql + mina_numbers + currency + bounded_types + cli_lib + mina_base + signature_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_custom_printf + ppx_mina + ppx_assert + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/zkapp_test_transaction/lib/dune b/src/app/zkapp_test_transaction/lib/dune index 3b0f5a18ded..81420ca2622 100644 --- a/src/app/zkapp_test_transaction/lib/dune +++ b/src/app/zkapp_test_transaction/lib/dune @@ -2,58 +2,69 @@ (name zkapp_test_transaction_lib) (public_name zkapp_test_transaction_lib) (libraries - ;;opam libraries - async_unix - graphql_parser - splittable_random - result - async - async_kernel - core - core_kernel - base - sexplib0 - base_quickcheck - ppx_inline_test.config - graphql-async - ;;local libraries - pickles - kimchi_pasta - kimchi_pasta.basic - bounded_types - random_oracle - mina_runtime_config - transaction_witness - transaction_protocol_state - coda_genesis_ledger - pickles_types - mina_numbers - random_oracle_input - genesis_ledger_helper - mina_state - mina_generators - mina_base - currency - mina_ledger - mina_graphql - genesis_constants - secrets - transaction_snark - signature_lib - data_hash_lib - consensus - with_hash - genesis_ledger_helper.lib - snark_params - pickles.backend - kimchi_backend - mina_base.import - mina_transaction_logic - mina_transaction - staged_ledger_diff - mina_wire_types - ) - (inline_tests (flags -verbose -show-counts)) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_custom_printf ppx_mina ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx ppx_inline_test))) + ;;opam libraries + async_unix + graphql_parser + splittable_random + result + async + async_kernel + core + core_kernel + base + sexplib0 + base_quickcheck + ppx_inline_test.config + graphql-async + ;;local libraries + pickles + kimchi_pasta + kimchi_pasta.basic + bounded_types + random_oracle + mina_runtime_config + transaction_witness + transaction_protocol_state + coda_genesis_ledger + pickles_types + mina_numbers + random_oracle_input + genesis_ledger_helper + mina_state + mina_generators + mina_base + currency + mina_ledger + mina_graphql + genesis_constants + secrets + transaction_snark + signature_lib + data_hash_lib + consensus + with_hash + genesis_ledger_helper.lib + snark_params + pickles.backend + kimchi_backend + mina_base.import + mina_transaction_logic + mina_transaction + staged_ledger_diff + mina_wire_types) + (inline_tests + (flags -verbose -show-counts)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_custom_printf + ppx_mina + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx + ppx_inline_test))) diff --git a/src/app/zkapps_examples/add_events/dune b/src/app/zkapps_examples/add_events/dune index 618100a98a1..d8b70325b92 100644 --- a/src/app/zkapps_examples/add_events/dune +++ b/src/app/zkapps_examples/add_events/dune @@ -1,19 +1,20 @@ (library (name zkapps_add_events) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - mina_base - pickles - pickles_types - snarky.backendless - snark_params - signature_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + mina_base + pickles + pickles_types + snarky.backendless + snark_params + signature_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess (pps ppx_version))) diff --git a/src/app/zkapps_examples/calls/dune b/src/app/zkapps_examples/calls/dune index eaaa1d87239..0bf875a0810 100644 --- a/src/app/zkapps_examples/calls/dune +++ b/src/app/zkapps_examples/calls/dune @@ -1,33 +1,32 @@ (library (name zkapps_calls) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - pickles - pickles.backend - pickles_types - random_oracle - random_oracle_input - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - with_hash - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + pickles + pickles.backend + pickles_types + random_oracle + random_oracle_input + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + with_hash + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version - h_list.ppx))) + (pps ppx_version h_list.ppx))) diff --git a/src/app/zkapps_examples/dune b/src/app/zkapps_examples/dune index 1db21807142..393112daabf 100644 --- a/src/app/zkapps_examples/dune +++ b/src/app/zkapps_examples/dune @@ -2,36 +2,35 @@ (name zkapps_examples) (public_name zkapps_examples) (libraries - ;; opam libraries - base - async_kernel - core_kernel - ;; local libraries - mina_numbers - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - mina_ledger - pickles - pickles.backend - pickles_types - random_oracle - random_oracle_input - snarky.backendless - snark_params - bounded_types - sgn - signature_lib - tuple_lib - with_hash) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + async_kernel + core_kernel + ;; local libraries + mina_numbers + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + mina_ledger + pickles + pickles.backend + pickles_types + random_oracle + random_oracle_input + snarky.backendless + snark_params + bounded_types + sgn + signature_lib + tuple_lib + with_hash) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_let - ppx_version))) + (pps ppx_let ppx_version))) diff --git a/src/app/zkapps_examples/empty_update/dune b/src/app/zkapps_examples/empty_update/dune index 10790066528..11e0c98a278 100644 --- a/src/app/zkapps_examples/empty_update/dune +++ b/src/app/zkapps_examples/empty_update/dune @@ -1,28 +1,28 @@ (library (name zkapps_empty_update) (libraries - ;; opam libraries - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - pickles - pickles.backend - pickles_types - random_oracle - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + pickles + pickles.backend + pickles_types + random_oracle + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version))) + (pps ppx_version))) diff --git a/src/app/zkapps_examples/initialize_state/dune b/src/app/zkapps_examples/initialize_state/dune index 5dc9bcf172f..63590bd026c 100644 --- a/src/app/zkapps_examples/initialize_state/dune +++ b/src/app/zkapps_examples/initialize_state/dune @@ -1,29 +1,29 @@ (library (name zkapps_initialize_state) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - pickles - pickles.backend - pickles_types - random_oracle - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + pickles + pickles.backend + pickles_types + random_oracle + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version))) + (pps ppx_version))) diff --git a/src/app/zkapps_examples/test/actions/dune b/src/app/zkapps_examples/test/actions/dune index baa85f43623..3c02027061c 100644 --- a/src/app/zkapps_examples/test/actions/dune +++ b/src/app/zkapps_examples/test/actions/dune @@ -40,7 +40,8 @@ zkapps_examples yojson mina_wire_types) - (inline_tests (flags -verbose -show-counts)) + (inline_tests + (flags -verbose -show-counts)) (preprocess (pps ppx_version ppx_jane)) (instrumentation diff --git a/src/app/zkapps_examples/test/add_events/dune b/src/app/zkapps_examples/test/add_events/dune index 583a5c247c7..e86b6cad48e 100644 --- a/src/app/zkapps_examples/test/add_events/dune +++ b/src/app/zkapps_examples/test/add_events/dune @@ -1,41 +1,42 @@ (library (name add_events) (libraries - ;; opam libraries - async - async_kernel - async_unix - core - core_kernel - ppx_inline_test.config - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - merkle_list_verifier - mina_base - mina_base.import - mina_ledger - mina_numbers - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - signature_lib - snark_params - transaction_snark - transaction_snark_tests - with_hash - zkapps_add_events - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + core + core_kernel + ppx_inline_test.config + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + merkle_list_verifier + mina_base + mina_base.import + mina_ledger + mina_numbers + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + signature_lib + snark_params + transaction_snark + transaction_snark_tests + with_hash + zkapps_add_events + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/calls/dune b/src/app/zkapps_examples/test/calls/dune index 25706b15533..d1ed0f79578 100644 --- a/src/app/zkapps_examples/test/calls/dune +++ b/src/app/zkapps_examples/test/calls/dune @@ -1,48 +1,49 @@ (library (name calls) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_calls - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_calls + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/empty_update/dune b/src/app/zkapps_examples/test/empty_update/dune index 534cf6ac7b1..c36a8be7eff 100644 --- a/src/app/zkapps_examples/test/empty_update/dune +++ b/src/app/zkapps_examples/test/empty_update/dune @@ -1,51 +1,51 @@ (executable (name empty_update) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_base - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_empty_update - zkapps_examples - bounded_types - ) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_base + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_empty_update + zkapps_examples + bounded_types) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) (rule (alias runtest) diff --git a/src/app/zkapps_examples/test/initialize_state/dune b/src/app/zkapps_examples/test/initialize_state/dune index eac8a441070..e5693dee970 100644 --- a/src/app/zkapps_examples/test/initialize_state/dune +++ b/src/app/zkapps_examples/test/initialize_state/dune @@ -1,48 +1,49 @@ (library (name initialize_state) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_initialize_state - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_initialize_state + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/optional_custom_gates/dune b/src/app/zkapps_examples/test/optional_custom_gates/dune index ecf9f3905ac..573edf08c77 100644 --- a/src/app/zkapps_examples/test/optional_custom_gates/dune +++ b/src/app/zkapps_examples/test/optional_custom_gates/dune @@ -1,43 +1,44 @@ (library - (name zkapp_optional_custom_gates_tests) + (name zkapp_optional_custom_gates_tests) (libraries - ;; opam libraries - ppx_inline_test.config - async - async_kernel - async_unix - core - core_kernel - result - base - base.caml - sexplib0 - integers - ;; local libraries - mina_base.import - transaction_snark - mina_ledger - mina_base - mina_generators - currency - pickles - pickles_types - mina_numbers - zkapp_command_builder - signature_lib - genesis_constants - sgn - transaction_snark_tests - test_util - mina_transaction_logic - zkapps_examples - pickles_optional_custom_gates_circuits - snark_keys_header - snark_params - data_hash_lib - ) - (library_flags -linkall) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane ppx_mina)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + ppx_inline_test.config + async + async_kernel + async_unix + core + core_kernel + result + base + base.caml + sexplib0 + integers + ;; local libraries + mina_base.import + transaction_snark + mina_ledger + mina_base + mina_generators + currency + pickles + pickles_types + mina_numbers + zkapp_command_builder + signature_lib + genesis_constants + sgn + transaction_snark_tests + test_util + mina_transaction_logic + zkapps_examples + pickles_optional_custom_gates_circuits + snark_keys_header + snark_params + data_hash_lib) + (library_flags -linkall) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane ppx_mina)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/tokens/dune b/src/app/zkapps_examples/test/tokens/dune index 48ecefc7bbe..05877f35fdf 100644 --- a/src/app/zkapps_examples/test/tokens/dune +++ b/src/app/zkapps_examples/test/tokens/dune @@ -1,49 +1,50 @@ (library (name tokens) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_tokens - zkapps_examples - bounded_types - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_tokens + zkapps_examples + bounded_types) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/tokens/dune b/src/app/zkapps_examples/tokens/dune index 1e82e848990..d9f359830bd 100644 --- a/src/app/zkapps_examples/tokens/dune +++ b/src/app/zkapps_examples/tokens/dune @@ -1,40 +1,38 @@ (library (name zkapps_tokens) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core_kernel - ;; local libraries - cache_dir - crypto_params - currency - genesis_constants - kimchi_backend - kimchi_backend_common - kimchi_pasta - mina_base - mina_base.import - pasta_bindings - pickles - pickles.backend - pickles_base - pickles_types - random_oracle - random_oracle_input - sgn - signature_lib - snark_params - snarky.backendless - tuple_lib - with_hash - yojson - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + async + async_kernel + async_unix + base + core_kernel + ;; local libraries + cache_dir + crypto_params + currency + genesis_constants + kimchi_backend + kimchi_backend_common + kimchi_pasta + mina_base + mina_base.import + pasta_bindings + pickles + pickles.backend + pickles_base + pickles_types + random_oracle + random_oracle_input + sgn + signature_lib + snark_params + snarky.backendless + tuple_lib + with_hash + yojson + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version - h_list.ppx - ppx_let))) + (pps ppx_version h_list.ppx ppx_let))) diff --git a/src/dune-project b/src/dune-project index 5a5d7ed3eb2..c83656b39cd 100644 --- a/src/dune-project +++ b/src/dune-project @@ -135,8 +135,8 @@ (package (name parallel)) (package (name parallel_scan)) (package (name participating_state)) -(package (name pasta_bindings)) (package (name patch_archive_test)) +(package (name pasta_bindings)) (package (name perf_histograms)) (package (name pickles_base)) (package (name pickles)) diff --git a/src/lib/cache_dir/native/cache_dir.ml b/src/lib/cache_dir/native/cache_dir.ml index 8450d40bfd7..d76cd00aa38 100644 --- a/src/lib/cache_dir/native/cache_dir.ml +++ b/src/lib/cache_dir/native/cache_dir.ml @@ -55,7 +55,7 @@ let load_from_s3 s3_bucket_prefix s3_install_path ~logger = @@ Monitor.try_with ~here:[%here] (fun () -> let each_uri (uri_string, file_path) = let open Deferred.Let_syntax in - [%log trace] "Downloading file from S3" + [%log trace] "Downloading file from S3: $url to $local_file_path" ~metadata: [ ("url", `String uri_string) ; ("local_file_path", `String file_path) diff --git a/src/lib/cli_lib/dune b/src/lib/cli_lib/dune index dc0f9c72a9d..05b36e7b400 100644 --- a/src/lib/cli_lib/dune +++ b/src/lib/cli_lib/dune @@ -45,7 +45,7 @@ kimchi_pasta.basic ppx_version.runtime gossip_net - mina_runtime_config) + runtime_config) (preprocess (pps ppx_version ppx_jane ppx_deriving_yojson ppx_deriving.make)) (instrumentation (backend bisect_ppx)) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index fa06f656fc3..18297518678 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -88,14 +88,14 @@ type t = let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full - ; sub_windows_per_window = None - ; ledger_depth = None - ; work_delay = None + ; sub_windows_per_window = Some 11 + ; ledger_depth = Some 35 + ; work_delay = Some 2 ; block_window_duration_ms = Some 120000 - ; transaction_capacity = None - ; coinbase_amount = None - ; supercharged_coinbase_factor = None - ; account_creation_fee = None + ; transaction_capacity = Some (Log_2 7) + ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 720) + ; supercharged_coinbase_factor = Some 1 + ; account_creation_fee = Some (Currency.Fee.of_mina_string_exn "1") ; fork = None } diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 16bba170998..f207055960d 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -105,6 +105,7 @@ module Base_node_config = struct mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" fi done + rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" |} diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 6b23207352e..12ef4242312 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -154,8 +154,17 @@ module Network_config = struct Some { Runtime_config.Daemon.default with txpool_max_size = Some txpool_max_size + ; peer_list_url = None + ; zkapp_proof_update_cost = Some 10.26 + ; zkapp_signed_single_update_cost = Some 9.140000000000001 + ; zkapp_signed_pair_update_cost = Some 10.08 + ; zkapp_transaction_cost_limit = Some 69.45 + ; max_event_elements = Some 100 + ; max_action_elements = Some 100 + ; zkapp_cmd_limit_hardcap = Some 128 ; slot_tx_end ; slot_chain_end + ; minimum_user_command_fee = Some (Currency.Fee.of_string "1000000") ; network_id } ; genesis = diff --git a/src/lib/mina_compile_config/dune b/src/lib/mina_compile_config/dune index fac058923b4..181360fbaf2 100644 --- a/src/lib/mina_compile_config/dune +++ b/src/lib/mina_compile_config/dune @@ -1,6 +1,6 @@ (library (name mina_compile_config) (public_name mina_compile_config) - (libraries mina_node_config mina_node_config.for_unit_tests core_kernel currency) + (libraries mina_node_config node_config_for_unit_tests core_kernel currency) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_version ppx_base ppx_deriving_yojson))) diff --git a/src/test/archive/patch_archive_test/patch_archive_test.ml b/src/test/archive/patch_archive_test/patch_archive_test.ml index 5ba17608041..cdeaa635ddf 100644 --- a/src/test/archive/patch_archive_test/patch_archive_test.ml +++ b/src/test/archive/patch_archive_test/patch_archive_test.ml @@ -77,10 +77,10 @@ let main ~db_uri ~network_data_folder () = let n = List.init missing_blocks_count ~f:(fun _ -> - (* never remove last block as missing-block-guardian can have issues when patching it + (* never remove last and first block as missing-block-guardian can have issues when patching it as it patching only gaps *) - Random.int (List.length extensional_files - 1) ) + Random.int (List.length extensional_files - 2) + 1 ) in let unpatched_extensional_files =