From 7bf4b441ef74008c44311eece88e78f545945b2a Mon Sep 17 00:00:00 2001 From: sai Date: Thu, 6 Jun 2024 12:13:22 -0400 Subject: [PATCH 001/178] normalizing snarky yojson serializers to match kimchi common --- src/lib/crypto/kimchi_backend/common/dune | 1 - src/lib/crypto/kimchi_backend/dune | 1 - src/lib/mina_base/account_update.ml | 14 +------------- src/lib/mina_wire_types/dune | 2 +- .../mina_base/mina_base_account_update.ml | 1 + src/lib/mina_wire_types/snark_params.ml | 6 +++++- src/lib/snark_params/snark_params.ml | 4 ++++ 7 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/lib/crypto/kimchi_backend/common/dune b/src/lib/crypto/kimchi_backend/common/dune index e67f451a091..8e11e3f5201 100644 --- a/src/lib/crypto/kimchi_backend/common/dune +++ b/src/lib/crypto/kimchi_backend/common/dune @@ -29,7 +29,6 @@ base.base_internalhash_types ;; local libraries tuple_lib - snarky.backendless key_cache pickles_types hex diff --git a/src/lib/crypto/kimchi_backend/dune b/src/lib/crypto/kimchi_backend/dune index b35651af400..8292304d4a8 100644 --- a/src/lib/crypto/kimchi_backend/dune +++ b/src/lib/crypto/kimchi_backend/dune @@ -20,7 +20,6 @@ kimchi_types pasta_bindings snarkette - snarky.backendless key_cache hex kimchi_backend.common diff --git a/src/lib/mina_base/account_update.ml b/src/lib/mina_base/account_update.ml index 2569f465ec1..0617448330a 100644 --- a/src/lib/mina_base/account_update.ml +++ b/src/lib/mina_base/account_update.ml @@ -22,19 +22,7 @@ module Authorization_kind = struct type t = Mina_wire_types.Mina_base.Account_update.Authorization_kind.V1.t = | Signature - | Proof of - (Field.t - [@version_asserted] - [@to_yojson fun t -> `String (Snark_params.Tick.Field.to_string t)] - [@of_yojson - function - | `String s -> - let field = Snark_params.Tick.Field.of_string s in - let s' = Snark_params.Tick.Field.to_string field in - if String.equal s s' then Ok field - else Error "Invalid JSON for field" - | _ -> - Error "expected JSON string"] ) + | Proof of (Field.t[@version_asserted]) | None_given [@@deriving sexp, equal, yojson, hash, compare] diff --git a/src/lib/mina_wire_types/dune b/src/lib/mina_wire_types/dune index 8df50d476c7..6384e2229dc 100644 --- a/src/lib/mina_wire_types/dune +++ b/src/lib/mina_wire_types/dune @@ -17,7 +17,7 @@ blake2 ) (preprocessor_deps ../../config.mlh) - (preprocess (pps ppx_version ppx_optcomp)) + (preprocess (pps ppx_version ppx_optcomp ppx_deriving_yojson)) (instrumentation (backend bisect_ppx))) ; Include index.mld in generated doc diff --git a/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml b/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml index 0d6c238efd2..ae2344dbd26 100644 --- a/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml +++ b/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml @@ -2,6 +2,7 @@ module Authorization_kind = struct module V1 = struct (* field for Proof is a verification key hash *) type t = Signature | Proof of Snark_params.Tick.Field.t | None_given + [@@deriving yojson] end end diff --git a/src/lib/mina_wire_types/snark_params.ml b/src/lib/mina_wire_types/snark_params.ml index c1164b677a2..ec8176f1591 100644 --- a/src/lib/mina_wire_types/snark_params.ml +++ b/src/lib/mina_wire_types/snark_params.ml @@ -1,6 +1,10 @@ module Tick = struct module Field = struct - type t = Pasta_bindings.Fp.t + type t = Kimchi_pasta_basic.Fp.t + + let to_yojson = Kimchi_pasta_basic.Fp.to_yojson + + let of_yojson = Kimchi_pasta_basic.Fp.of_yojson end module Inner_curve = struct diff --git a/src/lib/snark_params/snark_params.ml b/src/lib/snark_params/snark_params.ml index 64a572efdce..11e87987079 100644 --- a/src/lib/snark_params/snark_params.ml +++ b/src/lib/snark_params/snark_params.ml @@ -183,6 +183,10 @@ module Tick = struct include Tick0.Field module Bits = Bits.Make_field (Tick0.Field) (Tick0.Bigint) + let to_yojson = Mina_wire_types.Snark_params.Tick.Field.to_yojson + + let of_yojson = Mina_wire_types.Snark_params.Tick.Field.of_yojson + let size_in_triples = Int.((size_in_bits + 2) / 3) end From 8a8396ebbec3addb0172d7810c927256b48ab638 Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 10 Jun 2024 12:03:07 -0400 Subject: [PATCH 002/178] removing unecessary derivation in yojson --- src/lib/mina_wire_types/mina_base/mina_base_account_update.ml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml b/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml index ae2344dbd26..0d6c238efd2 100644 --- a/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml +++ b/src/lib/mina_wire_types/mina_base/mina_base_account_update.ml @@ -2,7 +2,6 @@ module Authorization_kind = struct module V1 = struct (* field for Proof is a verification key hash *) type t = Signature | Proof of Snark_params.Tick.Field.t | None_given - [@@deriving yojson] end end From 0c9a9bafdba233b34c317b5ecbd80f590efda0fd Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 10 Jun 2024 12:19:27 -0400 Subject: [PATCH 003/178] removing dune entry for ppx deriving json in snarky --- src/lib/mina_wire_types/dune | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mina_wire_types/dune b/src/lib/mina_wire_types/dune index 6384e2229dc..8df50d476c7 100644 --- a/src/lib/mina_wire_types/dune +++ b/src/lib/mina_wire_types/dune @@ -17,7 +17,7 @@ blake2 ) (preprocessor_deps ../../config.mlh) - (preprocess (pps ppx_version ppx_optcomp ppx_deriving_yojson)) + (preprocess (pps ppx_version ppx_optcomp)) (instrumentation (backend bisect_ppx))) ; Include index.mld in generated doc From 77e4dcaa2e39a4999716535fb4468f585ff8ddeb Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 10 Jun 2024 12:26:21 -0400 Subject: [PATCH 004/178] removing yojson serializaion from wire types --- src/lib/mina_wire_types/snark_params.ml | 3 --- src/lib/snark_params/snark_params.ml | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/lib/mina_wire_types/snark_params.ml b/src/lib/mina_wire_types/snark_params.ml index ec8176f1591..7a6a07f40be 100644 --- a/src/lib/mina_wire_types/snark_params.ml +++ b/src/lib/mina_wire_types/snark_params.ml @@ -2,9 +2,6 @@ module Tick = struct module Field = struct type t = Kimchi_pasta_basic.Fp.t - let to_yojson = Kimchi_pasta_basic.Fp.to_yojson - - let of_yojson = Kimchi_pasta_basic.Fp.of_yojson end module Inner_curve = struct diff --git a/src/lib/snark_params/snark_params.ml b/src/lib/snark_params/snark_params.ml index 11e87987079..45e240b2f2a 100644 --- a/src/lib/snark_params/snark_params.ml +++ b/src/lib/snark_params/snark_params.ml @@ -183,10 +183,9 @@ module Tick = struct include Tick0.Field module Bits = Bits.Make_field (Tick0.Field) (Tick0.Bigint) - let to_yojson = Mina_wire_types.Snark_params.Tick.Field.to_yojson - - let of_yojson = Mina_wire_types.Snark_params.Tick.Field.of_yojson + let to_yojson = Kimchi_pasta_basic.Fp.to_yojson + let of_yojson = Kimchi_pasta_basic.Fp.of_yojson let size_in_triples = Int.((size_in_bits + 2) / 3) end From 522d55140bcbc956f348752fc068398cf4e28516 Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 10 Jun 2024 12:27:38 -0400 Subject: [PATCH 005/178] re formatting --- src/lib/mina_wire_types/snark_params.ml | 1 - src/lib/snark_params/snark_params.ml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mina_wire_types/snark_params.ml b/src/lib/mina_wire_types/snark_params.ml index 7a6a07f40be..c4248a05fa0 100644 --- a/src/lib/mina_wire_types/snark_params.ml +++ b/src/lib/mina_wire_types/snark_params.ml @@ -1,7 +1,6 @@ module Tick = struct module Field = struct type t = Kimchi_pasta_basic.Fp.t - end module Inner_curve = struct diff --git a/src/lib/snark_params/snark_params.ml b/src/lib/snark_params/snark_params.ml index 45e240b2f2a..93a7227b02d 100644 --- a/src/lib/snark_params/snark_params.ml +++ b/src/lib/snark_params/snark_params.ml @@ -186,6 +186,7 @@ module Tick = struct let to_yojson = Kimchi_pasta_basic.Fp.to_yojson let of_yojson = Kimchi_pasta_basic.Fp.of_yojson + let size_in_triples = Int.((size_in_bits + 2) / 3) end From f75372b3bec2fb8e8f3c1396af4d45ada07ffb13 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 29 Jul 2024 14:44:44 +0200 Subject: [PATCH 006/178] Revert "Auxiliary commit to revert individual files from 5bfd12cc5305ebc9b7662ff40fc408c0fac61445" This reverts commit 25dccfe1bf616e3e2fc70ed5c0f6b3ebd2cbbdc2, reversing changes made to a801a21384e68cf6370859f546c199564c5524a4. --- ...ti-upgrade-plus-archive-init-speedup.patch | 5053 ----------------- buildkite/scripts/caqti-upgrade.patch | 2637 --------- 2 files changed, 7690 deletions(-) delete mode 100644 buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch delete mode 100644 buildkite/scripts/caqti-upgrade.patch diff --git a/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch b/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch deleted file mode 100644 index 92eb042b48d..00000000000 --- a/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch +++ /dev/null @@ -1,5053 +0,0 @@ -diff --git a/opam.export b/opam.export -index 655b623d35..130eb77140 100644 ---- a/opam.export -+++ b/opam.export -@@ -69,9 +69,9 @@ installed: [ - "camlp4.4.14+1" - "camomile.1.0.2" - "capnp.3.4.0" -- "caqti.1.5.0" -- "caqti-async.1.3.0" -- "caqti-driver-postgresql.1.5.1" -+ "caqti.2.0.1" -+ "caqti-async.2.0.1" -+ "caqti-driver-postgresql.2.0.1" - "charInfo_width.1.1.0" - "check_opam_switch.~dev" - "cmdliner.1.0.3" -@@ -138,6 +138,7 @@ installed: [ - "lmdb.1.0" - "logs.0.7.0" - "lwt.5.4.0" -+ "lwt-dllist.1.0.1" - "lwt_log.1.1.1" - "lwt_react.1.1.2" - "macaddr.5.0.1" -@@ -155,6 +156,7 @@ installed: [ - "mirage-crypto-rng.0.11.0" - "mirage-crypto-rng-async.0.11.0" - "mmap.1.1.0" -+ "mtime.2.0.0" - "num.1.1" - "ocaml.4.14.0" - "ocaml-base-compiler.4.14.0" -diff --git a/src/app/archive/cli/archive_cli.ml b/src/app/archive/cli/archive_cli.ml -index 0868f63e37..e4cf825b0e 100644 ---- a/src/app/archive/cli/archive_cli.ml -+++ b/src/app/archive/cli/archive_cli.ml -@@ -85,9 +85,7 @@ let command_prune = - in - let go () = - let open Deferred.Result.Let_syntax in -- let%bind ((module Conn) as conn) = -- Caqti_async.connect postgres.value -- in -+ let%bind ((module Conn) as conn) = Mina_caqti.connect postgres.value in - let%bind () = Conn.start () in - match%bind.Async.Deferred - let%bind () = -diff --git a/src/app/archive/cli/dune b/src/app/archive/cli/dune -index 917ed0b1ba..5f63637d42 100644 ---- a/src/app/archive/cli/dune -+++ b/src/app/archive/cli/dune -@@ -17,6 +17,7 @@ - cli_lib - logger - block_time -+ mina_caqti - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_mina))) -diff --git a/src/app/archive/drop_tables.sql b/src/app/archive/drop_tables.sql -index 571327de81..f9dca92cd7 100644 ---- a/src/app/archive/drop_tables.sql -+++ b/src/app/archive/drop_tables.sql -@@ -105,3 +105,7 @@ DROP TABLE zkapp_events; - DROP TABLE token_symbols; - - DROP TABLE voting_for; -+ -+DROP TABLE zkapp_field; -+ -+DROP TABLE zkapp_verification_key_hashes; -diff --git a/src/app/archive/lib/load_data.ml b/src/app/archive/lib/load_data.ml -index 1ca76d1a0b..4b4faf66dd 100644 ---- a/src/app/archive/lib/load_data.ml -+++ b/src/app/archive/lib/load_data.ml -@@ -91,9 +91,9 @@ let update_of_id pool update_id = - let open Zkapp_basic in - let query_db ~f = Mina_caqti.query ~f pool in - let with_pool ~f arg = -- let open Caqti_async in -+ let open Mina_caqti in - Pool.use -- (fun (module Conn : CONNECTION) -> f (module Conn : CONNECTION) arg) -+ (fun (module Conn : Mina_caqti.CONNECTION) -> f (module Conn : Mina_caqti.CONNECTION) arg) - pool - in - let%bind { app_state_id -@@ -636,9 +636,9 @@ let get_account_accessed ~pool (account : Processor.Accounts_accessed.t) : - (int * Account.t) Deferred.t = - let query_db ~f = Mina_caqti.query ~f pool in - let with_pool ~f arg = -- let open Caqti_async in -+ let open Mina_caqti in - Pool.use -- (fun (module Conn : CONNECTION) -> f (module Conn : CONNECTION) arg) -+ (fun (module Conn : Mina_caqti.CONNECTION) -> f (module Conn : Mina_caqti.CONNECTION) arg) - pool - in - let pk_of_id = pk_of_id pool in -diff --git a/src/app/archive/lib/metrics.ml b/src/app/archive/lib/metrics.ml -index c69e1805d1..bf15430c54 100644 ---- a/src/app/archive/lib/metrics.ml -+++ b/src/app/archive/lib/metrics.ml -@@ -14,10 +14,10 @@ let default_missing_blocks_width = 2000 - - module Max_block_height = struct - let query = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Caqti_request.Infix.(Caqti_type.unit ->! Caqti_type.int) - "SELECT GREATEST(0, MAX(height)) FROM blocks" - -- let update (module Conn : Caqti_async.CONNECTION) metric_server = -+ let update (module Conn : Mina_caqti.CONNECTION) metric_server = - time ~label:"max_block_height" (fun () -> - let open Deferred.Result.Let_syntax in - let%map max_height = Conn.find query () in -@@ -30,7 +30,7 @@ end - module Missing_blocks = struct - (*A block is missing if there is no entry for a specific height. However, if there is an entry then it doesn't necessarily mean that it is part of the main chain. Unparented_blocks will show value > 1 in that case. Look for the last 2000 blocks*) - let query missing_blocks_width = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Caqti_request.Infix.(Caqti_type.unit ->! Caqti_type.int) - (Core_kernel.sprintf - {sql| - SELECT COUNT( * ) -@@ -40,7 +40,7 @@ module Missing_blocks = struct - |sql} - missing_blocks_width ) - -- let update ~missing_blocks_width (module Conn : Caqti_async.CONNECTION) -+ let update ~missing_blocks_width (module Conn : Mina_caqti.CONNECTION) - metric_server = - let open Deferred.Result.Let_syntax in - time ~label:"missing_blocks" (fun () -> -@@ -55,13 +55,13 @@ module Unparented_blocks = struct - (* parent_hashes represent ends of chains leading to an orphan block *) - - let query = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Caqti_request.Infix.(Caqti_type.unit ->! Caqti_type.int) - {sql| - SELECT COUNT( * ) FROM blocks - WHERE parent_id IS NULL - |sql} - -- let update (module Conn : Caqti_async.CONNECTION) metric_server = -+ let update (module Conn : Mina_caqti.CONNECTION) metric_server = - let open Deferred.Result.Let_syntax in - time ~label:"unparented_blocks" (fun () -> - let%map unparented_block_count = Conn.find query () in -@@ -73,13 +73,13 @@ end - - let log_error ~logger pool metric_server - (f : -- (module Caqti_async.CONNECTION) -+ (module Mina_caqti.CONNECTION) - -> Mina_metrics.Archive.t - -> (unit, [> Caqti_error.call_or_retrieve ]) Deferred.Result.t ) = - let open Deferred.Let_syntax in - match%map -- Caqti_async.Pool.use -- (fun (module Conn : Caqti_async.CONNECTION) -> -+ Mina_caqti.Pool.use -+ (fun (module Conn : Mina_caqti.CONNECTION) -> - f (module Conn) metric_server ) - pool - with -diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml -index bec5544e41..fd99cc96e1 100644 ---- a/src/app/archive/lib/processor.ml -+++ b/src/app/archive/lib/processor.ml -@@ -21,7 +21,6 @@ - module Archive_rpc = Rpc - open Async - open Core --open Caqti_async - open Mina_base - open Mina_transaction - open Mina_state -@@ -29,6 +28,7 @@ open Mina_block - open Pipe_lib - open Signature_lib - open Pickles_types -+open Mina_caqti - - let applied_str = "applied" - -@@ -38,39 +38,89 @@ let txn_hash_to_base58_check ?(v1_transaction_hash = false) hash = - if v1_transaction_hash then Transaction_hash.to_base58_check_v1 hash - else Transaction_hash.to_base58_check hash - -+let unwrap t = Result.map_error ~f:Caqti_error.show t |> Result.ok_or_failwith -+ -+let ensure_local_copies (module Conn : Mina_caqti.CONNECTION) ~default tbl = -+ Hashtbl.find_or_add ~default tbl (Uri.to_string Conn.source) -+ -+let load_copy' ~default ~local_copies ~typ ~query ~load_elt -+ (module Conn : Mina_caqti.CONNECTION) = -+ match Hashtbl.find local_copies (Uri.to_string Conn.source) with -+ | Some copy -> -+ Deferred.return copy -+ | None -> -+ let t_to_id = ensure_local_copies (module Conn) ~default local_copies in -+ let%bind all_rows = -+ Conn.collect_list (Mina_caqti.collect_req Caqti_type.unit typ query) () -+ in -+ let%map () = -+ Deferred.List.iter (unwrap all_rows) ~f:(fun row -> -+ load_elt t_to_id row ) -+ in -+ t_to_id -+ -+let add_bidi m1 m2 ~key ~data = -+ Hashtbl.add_exn m1 ~key ~data ; -+ Hashtbl.add_exn m2 ~key:data ~data:key -+ -+let add_bidi_mapped m1 m2 ~key ~data ~value = -+ Hashtbl.add_exn m1 ~key ~data ; -+ Hashtbl.add_exn m2 ~key:value ~data:key -+ - module Public_key = struct -- let find (module Conn : CONNECTION) (t : Public_key.Compressed.t) = -- let public_key = Public_key.Compressed.to_base58_check t in -- Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -- "SELECT id FROM public_keys WHERE value = ?" ) -- public_key -+ type local_copy = -+ { id_to_key : (int, Public_key.Compressed.t) Hashtbl.t -+ ; key_to_id : (Public_key.Compressed.t, int) Hashtbl.t -+ } - -- let find_opt (module Conn : CONNECTION) (t : Public_key.Compressed.t) = -- let public_key = Public_key.Compressed.to_base58_check t in -- Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -- "SELECT id FROM public_keys WHERE value = ?" ) -- public_key -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> -+ { id_to_key = Hashtbl.create (module Int) -+ ; key_to_id = Hashtbl.create (module Public_key.Compressed) -+ } ) -+ ~local_copies -+ ~typ:Caqti_type.(t2 int string) -+ ~query:{sql| SELECT id, value FROM public_keys |sql} -+ ~load_elt:(fun { id_to_key; key_to_id } (id, keytext) -> -+ let key = Public_key.Compressed.of_base58_check_exn keytext in -+ add_bidi id_to_key key_to_id ~key:id ~data:key ; -+ Deferred.unit ) - -- let find_by_id (module Conn : CONNECTION) id = -- Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -- "SELECT value FROM public_keys WHERE id = ?" ) -- id -+ let find (module Conn : Mina_caqti.CONNECTION) (t : Public_key.Compressed.t) = -+ let%map local = load_copy (module Conn) in -+ Ok (Hashtbl.find_exn local.key_to_id t) -+ -+ let find_opt (module Conn : Mina_caqti.CONNECTION) -+ (t : Public_key.Compressed.t) = -+ let%map local = load_copy (module Conn) in -+ Ok (Hashtbl.find local.key_to_id t) - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let find_by_id (module Conn : Mina_caqti.CONNECTION) id = -+ let%map local = load_copy (module Conn) in -+ Ok -+ ( Hashtbl.find_exn local.id_to_key id -+ |> Public_key.Compressed.to_base58_check ) -+ -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (t : Public_key.Compressed.t) = -+ let%bind local = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - match%bind find_opt (module Conn) t with - | Some id -> - return id - | None -> - let public_key = Public_key.Compressed.to_base58_check t in -- Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -- "INSERT INTO public_keys (value) VALUES (?) RETURNING id" ) -- public_key -+ let%map.Deferred.Result new_id = -+ Conn.find -+ (Mina_caqti.find_req Caqti_type.string Caqti_type.int -+ "INSERT INTO public_keys (value) VALUES (?) RETURNING id" ) -+ public_key -+ in -+ add_bidi local.id_to_key local.key_to_id ~key:new_id ~data:t ; -+ new_id - end - - (* Unlike other modules here, `Token_owners` does not correspond with a database table *) -@@ -99,46 +149,72 @@ module Token = struct - include T - include Comparable.Make (T) - -+ type local_copy = -+ { id_to_t : (int, t) Hashtbl.t; value_to_id : (string, int) Hashtbl.t } -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> -+ { id_to_t = Hashtbl.create (module Int) -+ ; value_to_id = Hashtbl.create (module String) -+ } ) -+ ~local_copies -+ ~typ:Caqti_type.(t4 int string (option int) (option int)) -+ ~query: -+ {sql| SELECT id, value, owner_public_key_id, owner_token_id FROM tokens |sql} -+ ~load_elt:(fun { id_to_t; value_to_id } -+ (id, value, owner_public_key_id, owner_token_id) -> -+ let t = { value; owner_public_key_id; owner_token_id } in -+ add_bidi_mapped id_to_t value_to_id ~key:id ~data:t ~value ; -+ Deferred.unit ) -+ - let typ = - Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist - Caqti_type.[ string; option int; option int ] - - let table_name = "tokens" - -- let find_by_id (module Conn : CONNECTION) id = -- Conn.find -- (Caqti_request.find Caqti_type.int typ -- (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -- id -- -- let make_finder conn_finder req_finder token_id = -- conn_finder -- (req_finder Caqti_type.string Caqti_type.int -- (Mina_caqti.select_cols ~table_name ~select:"id" ~cols:[ "value" ] ()) ) -- (Token_id.to_string token_id) -+ let find_by_id (module Conn : Mina_caqti.CONNECTION) id = -+ let%map local = load_copy (module Conn) in -+ Ok (Hashtbl.find_exn local.id_to_t id) - -- let find (module Conn : CONNECTION) = make_finder Conn.find Caqti_request.find -+ let find (module Conn : Mina_caqti.CONNECTION) token_id = -+ let%map local = load_copy (module Conn) in -+ Ok (Hashtbl.find_exn local.value_to_id (Token_id.to_string token_id)) - -- let find_opt (module Conn : CONNECTION) = -- make_finder Conn.find_opt Caqti_request.find_opt -+ let find_opt (module Conn : Mina_caqti.CONNECTION) token_id = -+ let%map local = load_copy (module Conn) in -+ Ok (Hashtbl.find local.value_to_id (Token_id.to_string token_id)) - -- let find_no_owner_opt (module Conn : CONNECTION) token_id = -+ let find_no_owner_opt (module Conn : Mina_caqti.CONNECTION) token_id = -+ let%map local = load_copy (module Conn) in - let value = Token_id.to_string token_id in -- Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -- {sql| SELECT id -- FROM tokens -- WHERE value = $1 -- AND owner_public_key_id IS NULL -- AND owner_token_id IS NULL -- |sql} ) -- value -+ Ok -+ ( match Hashtbl.find local.value_to_id value with -+ | Some id -> ( -+ match Hashtbl.find local.id_to_t id with -+ | Some { owner_public_key_id = None; owner_token_id = None; _ } -> -+ Some id -+ | _ -> -+ None ) -+ | None -> -+ None ) - -- let set_owner (module Conn : CONNECTION) ~id ~owner_public_key_id -+ let set_owner (module Conn : Mina_caqti.CONNECTION) ~id ~owner_public_key_id - ~owner_token_id = -+ let%bind local = load_copy (module Conn) in -+ Hashtbl.set local.id_to_t ~key:id -+ ~data: -+ { (Hashtbl.find_exn local.id_to_t id) with -+ owner_public_key_id = Some owner_public_key_id -+ ; owner_token_id = Some owner_token_id -+ } ; -+ - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - Caqti_type.int - {sql| UPDATE tokens - SET owner_public_key_id = $2, owner_token_id = $3 -@@ -147,16 +223,29 @@ module Token = struct - |sql} ) - (id, owner_public_key_id, owner_token_id) - -- let add_if_doesn't_exist (module Conn : CONNECTION) token_id = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) token_id = -+ let%bind local = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - let value = Token_id.to_string token_id in - match Token_owners.find_owner token_id with -- | None -> -+ | None -> ( - (* not necessarily the default token *) -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name ~cols:(Fields.names, typ) -- (module Conn) -- { value; owner_public_key_id = None; owner_token_id = None } -+ match Hashtbl.find local.value_to_id value with -+ | None -> -+ let t = -+ { value; owner_public_key_id = None; owner_token_id = None } -+ in -+ let%map new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name ~cols:(Fields.names, typ) -+ (module Conn) -+ t -+ in -+ add_bidi_mapped local.id_to_t local.value_to_id ~key:new_id ~data:t -+ ~value ; -+ new_id -+ | Some id -> -+ return id ) - | Some acct_id -> ( - assert (not @@ Token_id.(equal default) token_id) ; - assert ( -@@ -180,10 +269,16 @@ module Token = struct - | None -> - let owner_public_key_id = Some owner_public_key_id in - let owner_token_id = Some owner_token_id in -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name ~cols:(Fields.names, typ) -- (module Conn) -- { value; owner_public_key_id; owner_token_id } ) -+ let t = { value; owner_public_key_id; owner_token_id } in -+ let%map new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name ~cols:(Fields.names, typ) -+ (module Conn) -+ t -+ in -+ add_bidi_mapped local.id_to_t local.value_to_id ~key:new_id ~data:t -+ ~value ; -+ new_id ) - end - - module Voting_for = struct -@@ -193,16 +288,40 @@ module Voting_for = struct - - let table_name = "voting_for" - -- let add_if_doesn't_exist (module Conn : CONNECTION) voting_for = -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name -- ~cols:([ "value" ], typ) -- (module Conn) -- (State_hash.to_base58_check voting_for) -+ type local_copy = (string, int) Hashtbl.t - -- let load (module Conn : CONNECTION) id = -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> Hashtbl.create (module String)) -+ ~local_copies -+ ~typ:Caqti_type.(t2 int string) -+ ~query:{sql| SELECT id, value FROM voting_for |sql} -+ ~load_elt:(fun t_to_id (id, value) -> -+ Hashtbl.add_exn t_to_id ~key:value ~data:id ; -+ Deferred.unit ) -+ -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) voting_for = -+ let%bind t_to_id = load_copy (module Conn) in -+ let voting_for = State_hash.to_base58_check voting_for in -+ match Hashtbl.find t_to_id voting_for with -+ | Some id -> -+ Deferred.Result.return id -+ | None -> -+ let%map.Deferred.Result new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name -+ ~cols:([ "value" ], typ) -+ (module Conn) -+ voting_for -+ in -+ Hashtbl.add_exn t_to_id ~key:voting_for ~data:new_id ; -+ new_id -+ -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -214,16 +333,40 @@ module Token_symbols = struct - - let table_name = "token_symbols" - -- let add_if_doesn't_exist (module Conn : CONNECTION) token_symbol = -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name -- ~cols:([ "value" ], typ) -- (module Conn) -- token_symbol -+ type local_copy = (string, int) Hashtbl.t -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> Hashtbl.create (module String)) -+ ~local_copies -+ ~typ:Caqti_type.(t2 int string) -+ ~query:{sql| SELECT id, value FROM token_symbols |sql} -+ ~load_elt:(fun t_to_id (id, value) -> -+ Hashtbl.add_exn t_to_id ~key:value ~data:id ; -+ Deferred.unit ) - -- let load (module Conn : CONNECTION) id = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) token_symbol = -+ let%bind t_to_id = load_copy (module Conn) in -+ let open Deferred.Result.Let_syntax in -+ match Hashtbl.find t_to_id token_symbol with -+ | Some id -> -+ return id -+ | None -> -+ let%map new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name -+ ~cols:([ "value" ], typ) -+ (module Conn) -+ token_symbol -+ in -+ Hashtbl.add_exn t_to_id ~key:token_symbol ~data:new_id ; -+ new_id -+ -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -242,7 +385,29 @@ module Account_identifiers = struct - - let table_name = "account_identifiers" - -- let add_if_doesn't_exist (module Conn : CONNECTION) account_id = -+ type local_copy = (int * int, int) Hashtbl.t -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let loaded = ref false -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> -+ Hashtbl.create -+ ( module struct -+ type t = int * int [@@deriving compare, sexp, hash] -+ end ) ) -+ ~local_copies -+ ~typ:Caqti_type.(t3 int int int) -+ ~query: -+ {sql| SELECT id,public_key_id,token_id FROM account_identifiers |sql} -+ ~load_elt:(fun t_to_id (id, public_key_id, token_id) -> -+ Hashtbl.add_exn t_to_id ~key:(public_key_id, token_id) ~data:id ; -+ Deferred.unit ) -+ -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) account_id = -+ let%bind t_to_id = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - let pk = Account_id.public_key account_id in - (* this token_id is Token_id.t *) -@@ -251,12 +416,21 @@ module Account_identifiers = struct - (* this token_id is a Postgresql table id *) - let%bind token_id = Token.add_if_doesn't_exist (module Conn) token_id in - let t = { public_key_id; token_id } in -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name ~cols:(Fields.names, typ) -- (module Conn) -- t -+ match Hashtbl.find t_to_id (public_key_id, token_id) with -+ | None -> -+ let%map new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name ~cols:(Fields.names, typ) -+ (module Conn) -+ t -+ in -+ Hashtbl.add_exn t_to_id ~key:(public_key_id, token_id) ~data:new_id ; -+ new_id -+ | Some id -> -+ Deferred.Result.return id - -- let find_opt (module Conn : CONNECTION) account_id = -+ let find_opt (module Conn : Mina_caqti.CONNECTION) account_id = -+ let%bind t_to_id = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - let pk = Account_id.public_key account_id in - match%bind Public_key.find_opt (module Conn) pk with -@@ -268,30 +442,20 @@ module Account_identifiers = struct - | None -> - return None - | Some tok_id -> -- Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 int int) -- Caqti_type.int -- (Mina_caqti.select_cols ~select:"id" ~table_name -- ~cols:Fields.names () ) ) -- (pk_id, tok_id) ) -- -- let find (module Conn : CONNECTION) account_id = -+ return (Hashtbl.find t_to_id (pk_id, tok_id)) ) -+ -+ let find (module Conn : Mina_caqti.CONNECTION) account_id = -+ let%bind t_to_id = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - let pk = Account_id.public_key account_id in - let%bind public_key_id = Public_key.find (module Conn) pk in - let token = Account_id.token_id account_id in -- let%bind token_id = Token.find (module Conn) token in -- Conn.find -- (Caqti_request.find -- Caqti_type.(tup2 int int) -- Caqti_type.int -- (Mina_caqti.select_cols ~select:"id" ~table_name ~cols:Fields.names ()) ) -- (public_key_id, token_id) -+ let%map token_id = Token.find (module Conn) token in -+ Hashtbl.find_exn t_to_id (public_key_id, token_id) - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -299,7 +463,7 @@ end - module Zkapp_field = struct - let table_name = "zkapp_field" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (fp : Pickles.Backend.Tick.Field.t) = - Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) - ~table_name -@@ -307,9 +471,9 @@ module Zkapp_field = struct - (module Conn) - (Pickles.Backend.Tick.Field.to_string fp) - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "field" ]) ) - id - end -@@ -317,7 +481,7 @@ end - module Zkapp_field_array = struct - let table_name = "zkapp_field_array" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (fps : Pickles.Backend.Tick.Field.t array) = - let open Deferred.Result.Let_syntax in - let%bind (element_ids : int array) = -@@ -332,9 +496,9 @@ module Zkapp_field_array = struct - (module Conn) - element_ids - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Mina_caqti.array_int_typ -+ (find_req Caqti_type.int Mina_caqti.array_int_typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "element_ids" ]) ) - id - end -@@ -367,7 +531,7 @@ module Zkapp_states_nullable = struct - - let table_name = "zkapp_states_nullable" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (fps : (Pickles.Backend.Tick.Field.t option, 'n) Vector.vec) = - let open Deferred.Result.Let_syntax in - let%bind (element_ids : int option list) = -@@ -404,9 +568,9 @@ module Zkapp_states_nullable = struct - (module Conn) - t - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -430,7 +594,7 @@ module Zkapp_states = struct - - let table_name = "zkapp_states" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (fps : (Pickles.Backend.Tick.Field.t, 'n) Vector.vec) = - let open Deferred.Result.Let_syntax in - let%bind (element_ids : int list) = -@@ -465,9 +629,9 @@ module Zkapp_states = struct - (module Conn) - t - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -488,7 +652,7 @@ module Zkapp_action_states = struct - - let table_name = "zkapp_action_states" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (fps : (Pickles.Backend.Tick.Field.t, 'n) Vector.vec) = - let open Deferred.Result.Let_syntax in - let%bind (element_ids : int list) = -@@ -507,9 +671,9 @@ module Zkapp_action_states = struct - (module Conn) - t - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -521,7 +685,7 @@ module Zkapp_verification_key_hashes = struct - - let table_name = "zkapp_verification_key_hashes" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (verification_key_hash : Pickles.Backend.Tick.Field.t) = - Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) - ~table_name -@@ -529,9 +693,9 @@ module Zkapp_verification_key_hashes = struct - (module Conn) - (Pickles.Backend.Tick.Field.to_string verification_key_hash) - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -546,7 +710,7 @@ module Zkapp_verification_keys = struct - - let table_name = "zkapp_verification_keys" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (vk : - ( Pickles.Side_loaded.Verification_key.t - , Pickles.Backend.Tick.Field.t ) -@@ -564,9 +728,9 @@ module Zkapp_verification_keys = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -586,32 +750,59 @@ module Protocol_versions = struct - - let table_name = "protocol_versions" - -- let add_if_doesn't_exist (module Conn : CONNECTION) ~transaction ~network -- ~patch = -+ type local_copy = (t, int) Hashtbl.t -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> Hashtbl.create (module T)) -+ ~local_copies -+ ~typ:Caqti_type.(t4 int int int int) -+ ~query: -+ {sql| SELECT id, transaction, network, patch FROM protocol_versions |sql} -+ ~load_elt:(fun t_to_id (id, transaction, network, patch) -> -+ Hashtbl.add_exn t_to_id ~key:{ transaction; network; patch } ~data:id ; -+ Deferred.unit ) -+ -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) ~transaction -+ ~network ~patch = - let t = { transaction; network; patch } in -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name ~cols:(Fields.names, typ) -- (module Conn) -- t -+ let%bind t_to_id = load_copy (module Conn) in -+ match Hashtbl.find t_to_id t with -+ | Some id -> -+ return (Ok id) -+ | None -> -+ let%map.Deferred.Result new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name ~cols:(Fields.names, typ) -+ (module Conn) -+ t -+ in -+ Hashtbl.add_exn t_to_id ~key:t ~data:new_id ; -+ new_id - -- let find (module Conn : CONNECTION) ~transaction ~network ~patch = -+ let find (module Conn : Mina_caqti.CONNECTION) ~transaction ~network ~patch = - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name ~cols:Fields.names ()) ) - (transaction, network, patch) - -- let find_txn_version (module Conn : CONNECTION) ~transaction = -- Conn.collect_list -- (Caqti_request.collect Caqti_type.int Caqti_type.int -- {sql| SELECT id FROM protocol_versions WHERE transaction = ? -- |sql} ) -- transaction -+ let find_by_txn_version (module Conn : Mina_caqti.CONNECTION) ~transaction = -+ let%map t_to_id = load_copy (module Conn) in -+ let matching_ids = -+ Hashtbl.fold ~init:[] -+ ~f:(fun ~key ~data acc -> -+ if Int.(key.transaction = transaction) then data :: acc else acc ) -+ t_to_id -+ in -+ Ok matching_ids - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -646,23 +837,81 @@ module Zkapp_permissions = struct - in - Caqti_type.enum ~encode ~decode "zkapp_auth_required_type" - -- type t = -- { edit_state : Permissions.Auth_required.t -- ; send : Permissions.Auth_required.t -- ; receive : Permissions.Auth_required.t -- ; access : Permissions.Auth_required.t -- ; set_delegate : Permissions.Auth_required.t -- ; set_permissions : Permissions.Auth_required.t -- ; set_verification_key_auth : Permissions.Auth_required.t -- ; set_verification_key_txn_version : int -- ; set_zkapp_uri : Permissions.Auth_required.t -- ; edit_action_state : Permissions.Auth_required.t -- ; set_token_symbol : Permissions.Auth_required.t -- ; increment_nonce : Permissions.Auth_required.t -- ; set_voting_for : Permissions.Auth_required.t -- ; set_timing : Permissions.Auth_required.t -- } -- [@@deriving fields, hlist] -+ module T = struct -+ type t = -+ { edit_state : Permissions.Auth_required.t -+ ; send : Permissions.Auth_required.t -+ ; receive : Permissions.Auth_required.t -+ ; access : Permissions.Auth_required.t -+ ; set_delegate : Permissions.Auth_required.t -+ ; set_permissions : Permissions.Auth_required.t -+ ; set_verification_key_auth : Permissions.Auth_required.t -+ ; set_verification_key_txn_version : int -+ ; set_zkapp_uri : Permissions.Auth_required.t -+ ; edit_action_state : Permissions.Auth_required.t -+ ; set_token_symbol : Permissions.Auth_required.t -+ ; increment_nonce : Permissions.Auth_required.t -+ ; set_voting_for : Permissions.Auth_required.t -+ ; set_timing : Permissions.Auth_required.t -+ } -+ [@@deriving fields, hlist, compare, sexp, hash] -+ end -+ -+ module With_id = struct -+ type t = -+ { id : int -+ ; edit_state : Permissions.Auth_required.t -+ ; send : Permissions.Auth_required.t -+ ; receive : Permissions.Auth_required.t -+ ; access : Permissions.Auth_required.t -+ ; set_delegate : Permissions.Auth_required.t -+ ; set_permissions : Permissions.Auth_required.t -+ ; set_verification_key_auth : Permissions.Auth_required.t -+ ; set_verification_key_txn_version : int -+ ; set_zkapp_uri : Permissions.Auth_required.t -+ ; edit_action_state : Permissions.Auth_required.t -+ ; set_token_symbol : Permissions.Auth_required.t -+ ; increment_nonce : Permissions.Auth_required.t -+ ; set_voting_for : Permissions.Auth_required.t -+ ; set_timing : Permissions.Auth_required.t -+ } -+ [@@deriving fields, hlist, compare, sexp, hash] -+ -+ let forget_id -+ { id = _ -+ ; edit_state -+ ; send -+ ; receive -+ ; access -+ ; set_delegate -+ ; set_permissions -+ ; set_verification_key_auth -+ ; set_verification_key_txn_version -+ ; set_zkapp_uri -+ ; edit_action_state -+ ; set_token_symbol -+ ; increment_nonce -+ ; set_voting_for -+ ; set_timing -+ } = -+ { T.edit_state -+ ; send -+ ; receive -+ ; access -+ ; set_delegate -+ ; set_permissions -+ ; set_verification_key_auth -+ ; set_verification_key_txn_version -+ ; set_zkapp_uri -+ ; edit_action_state -+ ; set_token_symbol -+ ; increment_nonce -+ ; set_voting_for -+ ; set_timing -+ } -+ end -+ -+ include T - - let typ = - Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist -@@ -682,9 +931,51 @@ module Zkapp_permissions = struct - ; auth_required_typ - ] - -+ let typ_with_id : With_id.t Caqti_type.t = -+ let open With_id in -+ Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist -+ [ Caqti_type.int -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; Caqti_type.int -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ; auth_required_typ -+ ] -+ - let table_name = "zkapp_permissions" - -- let add_if_doesn't_exist (module Conn : CONNECTION) (perms : Permissions.t) = -+ let load (module Conn : Mina_caqti.CONNECTION) id = -+ Conn.find -+ (Mina_caqti.find_req Caqti_type.int typ -+ (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -+ id -+ -+ type local_copy = (t, int) Hashtbl.t -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> Hashtbl.create (module T)) -+ ~local_copies ~typ:typ_with_id -+ ~query: -+ (sprintf {sql| SELECT %s FROM zkapp_permissions |sql} -+ (String.concat ~sep:"," With_id.Fields.names) ) -+ ~load_elt:(fun t_to_id t -> -+ Hashtbl.add_exn t_to_id ~key:(With_id.forget_id t) ~data:t.id ; -+ Deferred.unit ) -+ -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) -+ (perms : Permissions.t) = - let value = - { edit_state = perms.edit_state - ; send = perms.send -@@ -703,16 +994,21 @@ module Zkapp_permissions = struct - ; set_timing = perms.set_timing - } - in -- Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) -- ~table_name ~cols:(Fields.names, typ) -- (module Conn) -- value -+ let%bind t_to_id = load_copy (module Conn) in - -- let load (module Conn : CONNECTION) id = -- Conn.find -- (Caqti_request.find Caqti_type.int typ -- (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -- id -+ let open Deferred.Result.Let_syntax in -+ match Hashtbl.find t_to_id value with -+ | Some id -> -+ return id -+ | None -> -+ let%map new_id = -+ Mina_caqti.insert_assuming_new ~select:("id", Caqti_type.int) -+ ~table_name ~cols:(Fields.names, typ) -+ (module Conn) -+ value -+ in -+ Hashtbl.add_exn t_to_id ~key:value ~data:new_id ; -+ new_id - end - - module Zkapp_timing_info = struct -@@ -731,7 +1027,7 @@ module Zkapp_timing_info = struct - - let table_name = "zkapp_timing_info" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (timing_info : Account_update.Update.Timing_info.t) = - let initial_minimum_balance = - Currency.Balance.to_string timing_info.initial_minimum_balance -@@ -761,9 +1057,9 @@ module Zkapp_timing_info = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -775,16 +1071,16 @@ module Zkapp_uri = struct - - let table_name = "zkapp_uris" - -- let add_if_doesn't_exist (module Conn : CONNECTION) zkapp_uri = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) zkapp_uri = - Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int) - ~table_name - ~cols:([ "value" ], typ) - (module Conn) - zkapp_uri - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -817,7 +1113,7 @@ module Zkapp_updates = struct - - let table_name = "zkapp_updates" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (update : Account_update.Update.t) = - let open Deferred.Result.Let_syntax in - let%bind app_state_id = -@@ -875,9 +1171,9 @@ module Zkapp_updates = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -892,7 +1188,7 @@ module Zkapp_balance_bounds = struct - - let table_name = "zkapp_balance_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (balance_bounds : - Currency.Balance.t Mina_base.Zkapp_precondition.Closed_interval.t ) = - let balance_lower_bound = Currency.Balance.to_string balance_bounds.lower in -@@ -903,9 +1199,9 @@ module Zkapp_balance_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -920,7 +1216,7 @@ module Zkapp_nonce_bounds = struct - - let table_name = "zkapp_nonce_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (nonce_bounds : - Mina_numbers.Account_nonce.t - Mina_base.Zkapp_precondition.Closed_interval.t ) = -@@ -932,9 +1228,9 @@ module Zkapp_nonce_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -967,7 +1263,7 @@ module Zkapp_account_precondition = struct - - let table_name = "zkapp_account_precondition" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (acct : Zkapp_precondition.Account.t) = - let open Deferred.Result.Let_syntax in - let%bind balance_id = -@@ -1016,9 +1312,9 @@ module Zkapp_account_precondition = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1033,7 +1329,7 @@ module Zkapp_token_id_bounds = struct - - let table_name = "zkapp_token_id_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (token_id_bounds : - Token_id.t Mina_base.Zkapp_precondition.Closed_interval.t ) = - let token_id_lower_bound = token_id_bounds.lower |> Token_id.to_string in -@@ -1044,9 +1340,9 @@ module Zkapp_token_id_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1061,7 +1357,7 @@ module Zkapp_timestamp_bounds = struct - - let table_name = "zkapp_timestamp_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (timestamp_bounds : - Block_time.t Mina_base.Zkapp_precondition.Closed_interval.t ) = - let timestamp_lower_bound = -@@ -1076,9 +1372,9 @@ module Zkapp_timestamp_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1093,7 +1389,7 @@ module Zkapp_length_bounds = struct - - let table_name = "zkapp_length_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (length_bounds : - Unsigned.uint32 Mina_base.Zkapp_precondition.Closed_interval.t ) = - let length_lower_bound = Unsigned.UInt32.to_int64 length_bounds.lower in -@@ -1104,9 +1400,9 @@ module Zkapp_length_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1121,7 +1417,7 @@ module Zkapp_amount_bounds = struct - - let table_name = "zkapp_amount_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (amount_bounds : - Currency.Amount.t Mina_base.Zkapp_precondition.Closed_interval.t ) = - let amount_lower_bound = Currency.Amount.to_string amount_bounds.lower in -@@ -1132,9 +1428,9 @@ module Zkapp_amount_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1149,7 +1445,7 @@ module Zkapp_global_slot_bounds = struct - - let table_name = "zkapp_global_slot_bounds" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (global_slot_bounds : - Mina_numbers.Global_slot_since_genesis.t - Mina_base.Zkapp_precondition.Closed_interval.t ) = -@@ -1167,38 +1463,104 @@ module Zkapp_global_slot_bounds = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end - - module Timing_info = struct -- type t = -- { account_identifier_id : int -- ; initial_minimum_balance : string -- ; cliff_time : int64 -- ; cliff_amount : string -- ; vesting_period : int64 -- ; vesting_increment : string -- } -- [@@deriving hlist, fields] -+ module T = struct -+ type t = -+ { account_identifier_id : int -+ ; initial_minimum_balance : string -+ ; cliff_time : int64 -+ ; cliff_amount : string -+ ; vesting_period : int64 -+ ; vesting_increment : string -+ } -+ [@@deriving hlist, fields, compare, sexp, hash] -+ end -+ -+ module With_id = struct -+ type t = -+ { id : int -+ ; account_identifier_id : int -+ ; initial_minimum_balance : string -+ ; cliff_time : int64 -+ ; cliff_amount : string -+ ; vesting_period : int64 -+ ; vesting_increment : string -+ } -+ [@@deriving hlist, fields, compare, sexp, hash] -+ -+ let forget_id -+ { id = _ -+ ; account_identifier_id -+ ; initial_minimum_balance -+ ; cliff_time -+ ; cliff_amount -+ ; vesting_period -+ ; vesting_increment -+ } = -+ { T.account_identifier_id -+ ; initial_minimum_balance -+ ; cliff_time -+ ; cliff_amount -+ ; vesting_period -+ ; vesting_increment -+ } -+ end -+ -+ include T - - let typ = - Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist - Caqti_type.[ int; string; int64; string; int64; string ] - -+ let typ_with_id = -+ let open With_id in -+ Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist -+ Caqti_type.[ int; int; string; int64; string; int64; string ] -+ - let table_name = "timing_info" - -- let find (module Conn : CONNECTION) (acc : Account.t) = -+ type local_copy = (t, int) Hashtbl.t -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let load (module Conn : Mina_caqti.CONNECTION) id = -+ Conn.find -+ (Mina_caqti.find_req Caqti_type.int typ -+ (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -+ id -+ -+ let load_opt (module Conn : Mina_caqti.CONNECTION) id = -+ Conn.find_opt -+ (Mina_caqti.find_opt_req Caqti_type.int typ -+ (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -+ id -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> Hashtbl.create (module T)) -+ ~local_copies ~typ:typ_with_id -+ ~query: -+ (sprintf {sql| SELECT %s FROM timing_info |sql} -+ (String.concat ~sep:"," With_id.Fields.names) ) -+ ~load_elt:(fun t_to_id t -> -+ Hashtbl.add_exn t_to_id ~key:(With_id.forget_id t) ~data:t.id ; -+ Deferred.unit ) -+ -+ let find (module Conn : Mina_caqti.CONNECTION) (acc : Account.t) = - let open Deferred.Result.Let_syntax in - let%bind account_identifier_id = - let account_id = Account_id.create acc.public_key acc.token_id in - Account_identifiers.find (module Conn) account_id - in - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - {sql| SELECT account_identifier_id, initial_minimum_balance, - cliff_time, cliff_amount, - vesting_period, vesting_increment -@@ -1207,10 +1569,10 @@ module Timing_info = struct - |sql} ) - account_identifier_id - -- let find_by_account_identifier_id_opt (module Conn : CONNECTION) -+ let find_by_account_identifier_id_opt (module Conn : Mina_caqti.CONNECTION) - account_identifier_id = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.int typ -+ (find_opt_req Caqti_type.int typ - {sql| SELECT account_identifier_id, initial_minimum_balance, - cliff_time, cliff_amount, - vesting_period, vesting_increment -@@ -1219,8 +1581,9 @@ module Timing_info = struct - |sql} ) - account_identifier_id - -- let add_if_doesn't_exist (module Conn : CONNECTION) account_identifier_id -- (timing : Account_timing.t) = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) -+ account_identifier_id (timing : Account_timing.t) = -+ let%bind t_to_id = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - let slot_to_int64 x = - Mina_numbers.Global_slot_since_genesis.to_uint32 x -@@ -1251,65 +1614,46 @@ module Timing_info = struct - ; vesting_increment = zero - } - in -- match%bind -- Conn.find_opt -- (Caqti_request.find_opt typ Caqti_type.int -- {sql| SELECT id FROM timing_info -- WHERE account_identifier_id = ? -- AND initial_minimum_balance = ? -- AND cliff_time = ? -- AND cliff_amount = ? -- AND vesting_period = ? -- AND vesting_increment = ? |sql} ) -- values -- with -+ match Hashtbl.find t_to_id values with - | Some id -> - return id - | None -> -- Conn.find -- (Caqti_request.find typ Caqti_type.int -- {sql| INSERT INTO timing_info -+ let%map new_id = -+ Conn.find -+ (Mina_caqti.find_req typ Caqti_type.int -+ {sql| INSERT INTO timing_info - (account_identifier_id,initial_minimum_balance, - cliff_time, cliff_amount, vesting_period, vesting_increment) - VALUES (?, ?, ?, ?, ?, ?) - RETURNING id - |sql} ) -- values -- -- let load (module Conn : CONNECTION) id = -- Conn.find -- (Caqti_request.find Caqti_type.int typ -- (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -- id -- -- let load_opt (module Conn : CONNECTION) id = -- Conn.find_opt -- (Caqti_request.find_opt Caqti_type.int typ -- (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) -- id -+ values -+ in -+ Hashtbl.add_exn t_to_id ~key:values ~data:new_id ; -+ new_id - end - - module Snarked_ledger_hash = struct -- let find (module Conn : CONNECTION) (t : Frozen_ledger_hash.t) = -+ let find (module Conn : Mina_caqti.CONNECTION) (t : Frozen_ledger_hash.t) = - let hash = Frozen_ledger_hash.to_base58_check t in - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (find_req Caqti_type.string Caqti_type.int - "SELECT id FROM snarked_ledger_hashes WHERE value = ?" ) - hash - -- let find_by_id (module Conn : CONNECTION) id = -+ let find_by_id (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) - id - -- let add_if_doesn't_exist (module Conn : CONNECTION) (t : Frozen_ledger_hash.t) -- = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) -+ (t : Frozen_ledger_hash.t) = - let open Deferred.Result.Let_syntax in - let hash = Frozen_ledger_hash.to_base58_check t in - match%bind - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ (find_opt_req Caqti_type.string Caqti_type.int - "SELECT id FROM snarked_ledger_hashes WHERE value = ?" ) - hash - with -@@ -1317,13 +1661,13 @@ module Snarked_ledger_hash = struct - return id - | None -> - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (find_req Caqti_type.string Caqti_type.int - "INSERT INTO snarked_ledger_hashes (value) VALUES (?) RETURNING id" ) - hash - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) - id - end -@@ -1338,7 +1682,7 @@ module Zkapp_epoch_ledger = struct - - let table_name = "zkapp_epoch_ledger" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (epoch_ledger : _ Epoch_ledger.Poly.t) = - let open Deferred.Result.Let_syntax in - let%bind hash_id = -@@ -1357,9 +1701,9 @@ module Zkapp_epoch_ledger = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1381,7 +1725,7 @@ module Zkapp_epoch_data = struct - - let table_name = "zkapp_epoch_data" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (epoch_data : Mina_base.Zkapp_precondition.Protocol_state.Epoch_data.t) = - let open Deferred.Result.Let_syntax in - let%bind epoch_ledger_id = -@@ -1417,9 +1761,9 @@ module Zkapp_epoch_data = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1443,7 +1787,7 @@ module Zkapp_network_precondition = struct - - let table_name = "zkapp_network_precondition" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (ps : Mina_base.Zkapp_precondition.Protocol_state.t) = - let open Deferred.Result.Let_syntax in - let%bind snarked_ledger_hash_id = -@@ -1492,9 +1836,9 @@ module Zkapp_network_precondition = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1528,7 +1872,7 @@ module Zkapp_events = struct - 7. use "M'" and the list of list of field_ids to compute the list of field_array_ids - 8. insert the list of field_arrays - *) -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (events : Account_update.Body.Events'.t) = - let open Deferred.Result.Let_syntax in - let%bind field_array_id_list = -@@ -1584,9 +1928,9 @@ module Zkapp_events = struct - (module Conn) - field_array_id_list - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Mina_caqti.array_int_typ -+ (find_req Caqti_type.int Mina_caqti.array_int_typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "element_ids" ]) ) - id - end -@@ -1635,7 +1979,7 @@ module Zkapp_account_update_body = struct - - let table_name = "zkapp_account_update_body" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (body : Account_update.Body.Simple.t) = - let open Deferred.Result.Let_syntax in - let account_identifier = Account_id.create body.public_key body.token_id in -@@ -1736,9 +2080,9 @@ module Zkapp_account_update_body = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1751,7 +2095,7 @@ module Zkapp_account_update = struct - - let table_name = "zkapp_account_update" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (account_update : Account_update.Simple.t) = - let open Deferred.Result.Let_syntax in - let%bind body_id = -@@ -1767,9 +2111,9 @@ module Zkapp_account_update = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1789,7 +2133,7 @@ module Zkapp_fee_payer_body = struct - - let table_name = "zkapp_fee_payer_body" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (body : Account_update.Body.Fee_payer.t) = - let open Deferred.Result.Let_syntax in - let%bind public_key_id = -@@ -1811,9 +2155,9 @@ module Zkapp_fee_payer_body = struct - (module Conn) - value - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1840,7 +2184,7 @@ module Epoch_data = struct - - let table_name = "epoch_data" - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - (t : Mina_base.Epoch_data.Value.t) = - let open Deferred.Result.Let_syntax in - let Mina_base.Epoch_ledger.Poly.{ hash; total_currency } = -@@ -1868,9 +2212,9 @@ module Epoch_data = struct - ; epoch_length - } - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1908,22 +2252,22 @@ module User_command = struct - - let table_name = "user_commands" - -- let find (module Conn : CONNECTION) ~(transaction_hash : Transaction_hash.t) -- ~v1_transaction_hash = -+ let find (module Conn : Mina_caqti.CONNECTION) -+ ~(transaction_hash : Transaction_hash.t) ~v1_transaction_hash = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ (find_opt_req Caqti_type.string Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name ~cols:[ "hash" ] ()) ) - (txn_hash_to_base58_check transaction_hash ~v1_transaction_hash) - -- let load (module Conn : CONNECTION) ~(id : int) = -+ let load (module Conn : Mina_caqti.CONNECTION) ~(id : int) = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - - type balance_public_key_ids = { fee_payer_id : int; receiver_id : int } - -- let add_account_ids_if_don't_exist (module Conn : CONNECTION) -+ let add_account_ids_if_don't_exist (module Conn : Mina_caqti.CONNECTION) - (t : Signed_command.t) = - let open Deferred.Result.Let_syntax in - let%bind fee_payer_id = -@@ -1936,8 +2280,9 @@ module User_command = struct - in - { fee_payer_id; receiver_id } - -- let add_if_doesn't_exist ?(via = `Ident) (module Conn : CONNECTION) -- (t : Signed_command.t) ~v1_transaction_hash = -+ let add_if_doesn't_exist ?(via = `Ident) -+ (module Conn : Mina_caqti.CONNECTION) (t : Signed_command.t) -+ ~v1_transaction_hash = - let open Deferred.Result.Let_syntax in - let transaction_hash = Transaction_hash.hash_command (Signed_command t) in - match%bind find (module Conn) ~transaction_hash ~v1_transaction_hash with -@@ -1961,7 +2306,7 @@ module User_command = struct - in - (* TODO: Converting these uint64s to int64 can overflow; see #5419 *) - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -1988,7 +2333,7 @@ module User_command = struct - |> txn_hash_to_base58_check ~v1_transaction_hash - } - -- let add_extensional_if_doesn't_exist (module Conn : CONNECTION) -+ let add_extensional_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - ?(v1_transaction_hash = false) (user_cmd : Extensional.User_command.t) = - let open Deferred.Result.Let_syntax in - match%bind -@@ -2008,7 +2353,7 @@ module User_command = struct - Public_key.add_if_doesn't_exist (module Conn) user_cmd.receiver - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2046,21 +2391,22 @@ module User_command = struct - - let table_name = "zkapp_commands" - -- let find_opt (module Conn : CONNECTION) -+ let find_opt (module Conn : Mina_caqti.CONNECTION) - ~(transaction_hash : Transaction_hash.t) = - Conn.find_opt -- ( Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ ( find_opt_req Caqti_type.string Caqti_type.int - @@ Mina_caqti.select_cols ~select:"id" ~table_name ~cols:[ "hash" ] () - ) - (Transaction_hash.to_base58_check transaction_hash) - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- ( Caqti_request.find Caqti_type.int typ -+ ( find_req Caqti_type.int typ - @@ Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names ) - id - -- let add_if_doesn't_exist (module Conn : CONNECTION) (ps : Zkapp_command.t) = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) -+ (ps : Zkapp_command.t) = - let open Deferred.Result.Let_syntax in - let zkapp_command = Zkapp_command.to_simple ps in - let%bind zkapp_fee_payer_body_id = -@@ -2129,11 +2475,12 @@ module Internal_command = struct - - let table_name = "internal_commands" - -- let find_opt (module Conn : CONNECTION) ~(v1_transaction_hash : bool) -- ~(transaction_hash : Transaction_hash.t) ~(command_type : string) = -+ let find_opt (module Conn : Mina_caqti.CONNECTION) -+ ~(v1_transaction_hash : bool) ~(transaction_hash : Transaction_hash.t) -+ ~(command_type : string) = - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 string string) -+ (find_opt_req -+ Caqti_type.(t2 string string) - Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name - ~tannot:(function -@@ -2142,13 +2489,13 @@ module Internal_command = struct - ( txn_hash_to_base58_check ~v1_transaction_hash transaction_hash - , command_type ) - -- let load (module Conn : CONNECTION) ~(id : int) = -+ let load (module Conn : Mina_caqti.CONNECTION) ~(id : int) = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - -- let add_extensional_if_doesn't_exist (module Conn : CONNECTION) -+ let add_extensional_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - ?(v1_transaction_hash = false) - (internal_cmd : Extensional.Internal_command.t) = - let open Deferred.Result.Let_syntax in -@@ -2165,7 +2512,7 @@ module Internal_command = struct - Public_key.add_if_doesn't_exist (module Conn) internal_cmd.receiver - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "command_type" -> Some "internal_command_type" | _ -> None -@@ -2210,10 +2557,10 @@ module Fee_transfer = struct - in - Ok { kind; receiver_id; fee; hash } - in -- let rep = Caqti_type.(tup4 string int int64 string) in -+ let rep = Caqti_type.(t4 string int int64 string) in - Caqti_type.custom ~encode ~decode rep - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - ?(v1_transaction_hash = false) (t : Fee_transfer.Single.t) - (kind : [ `Normal | `Via_coinbase ]) = - let open Deferred.Result.Let_syntax in -@@ -2232,7 +2579,7 @@ module Fee_transfer = struct - Public_key.add_if_doesn't_exist (module Conn) pk - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - {sql| INSERT INTO internal_commands - (command_type, receiver_id, fee, hash) - VALUES (?::internal_command_type, ?, ?, ?) -@@ -2260,10 +2607,10 @@ module Coinbase = struct - let decode (_, receiver_id, amount, hash) = - Ok { receiver_id; amount; hash } - in -- let rep = Caqti_type.(tup4 string int int64 string) in -+ let rep = Caqti_type.(t4 string int int64 string) in - Caqti_type.custom ~encode ~decode rep - -- let add_if_doesn't_exist (module Conn : CONNECTION) -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - ?(v1_transaction_hash = false) (t : Coinbase.t) = - let open Deferred.Result.Let_syntax in - let transaction_hash = Transaction_hash.hash_coinbase t in -@@ -2281,7 +2628,7 @@ module Coinbase = struct - Public_key.add_if_doesn't_exist (module Conn) pk - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - {sql| INSERT INTO internal_commands - (command_type, receiver_id, fee, hash) - VALUES (?::internal_command_type, ?, ?, ?) -@@ -2313,14 +2660,14 @@ module Block_and_internal_command = struct - Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist - Caqti_type.[ int; int; int; int; string; option string ] - -- let add (module Conn : CONNECTION) ~block_id ~internal_command_id ~sequence_no -- ~secondary_sequence_no ~status -+ let add (module Conn : Mina_caqti.CONNECTION) ~block_id ~internal_command_id -+ ~sequence_no ~secondary_sequence_no ~status - ~(failure_reason : Transaction_status.Failure.t option) = - let failure_reason = - Option.map ~f:Transaction_status.Failure.to_string failure_reason - in - Conn.exec -- (Caqti_request.exec typ -+ (exec_req typ - {sql| INSERT INTO blocks_internal_commands - (block_id, - internal_command_id, -@@ -2338,11 +2685,11 @@ module Block_and_internal_command = struct - ; failure_reason - } - -- let find (module Conn : CONNECTION) ~block_id ~internal_command_id -+ let find (module Conn : Mina_caqti.CONNECTION) ~block_id ~internal_command_id - ~sequence_no ~secondary_sequence_no = - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup4 int int int int) -+ (find_opt_req -+ Caqti_type.(t4 int int int int) - Caqti_type.string - {sql| SELECT 'exists' FROM blocks_internal_commands - WHERE block_id = $1 -@@ -2352,7 +2699,7 @@ module Block_and_internal_command = struct - |sql} ) - (block_id, internal_command_id, sequence_no, secondary_sequence_no) - -- let add_if_doesn't_exist (module Conn : CONNECTION) ~block_id -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) ~block_id - ~internal_command_id ~sequence_no ~secondary_sequence_no ~status - ~failure_reason = - let open Deferred.Result.Let_syntax in -@@ -2369,12 +2716,12 @@ module Block_and_internal_command = struct - ~block_id ~internal_command_id ~sequence_no ~secondary_sequence_no - ~status ~failure_reason - -- let load (module Conn : CONNECTION) ~block_id ~internal_command_id -+ let load (module Conn : Mina_caqti.CONNECTION) ~block_id ~internal_command_id - ~sequence_no ~secondary_sequence_no = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup4 int int int int) -+ (find_req -+ Caqti_type.(t4 int int int int) - typ - (sprintf - {sql| SELECT %s FROM blocks_internal_commands -@@ -2403,13 +2750,13 @@ module Block_and_signed_command = struct - - let table_name = "blocks_user_commands" - -- let add (module Conn : CONNECTION) ~block_id ~user_command_id ~sequence_no -- ~status ~failure_reason = -+ let add (module Conn : Mina_caqti.CONNECTION) ~block_id ~user_command_id -+ ~sequence_no ~status ~failure_reason = - let failure_reason = - Option.map ~f:Transaction_status.Failure.to_string failure_reason - in - Conn.exec -- (Caqti_request.exec typ -+ (exec_req typ - {sql| INSERT INTO blocks_user_commands - (block_id, - user_command_id, -@@ -2420,8 +2767,8 @@ module Block_and_signed_command = struct - |sql} ) - { block_id; user_command_id; sequence_no; status; failure_reason } - -- let add_with_status (module Conn : CONNECTION) ~block_id ~user_command_id -- ~sequence_no ~(status : Transaction_status.t) = -+ let add_with_status (module Conn : Mina_caqti.CONNECTION) ~block_id -+ ~user_command_id ~sequence_no ~(status : Transaction_status.t) = - let status_str, failure_reason = - match status with - | Applied -> -@@ -2434,13 +2781,13 @@ module Block_and_signed_command = struct - (module Conn) - ~block_id ~user_command_id ~sequence_no ~status:status_str ~failure_reason - -- let add_if_doesn't_exist (module Conn : CONNECTION) ~block_id ~user_command_id -- ~sequence_no ~(status : string) ~failure_reason = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) ~block_id -+ ~user_command_id ~sequence_no ~(status : string) ~failure_reason = - let open Deferred.Result.Let_syntax in - match%bind - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup3 int int int) -+ (find_opt_req -+ Caqti_type.(t3 int int int) - Caqti_type.string - {sql| SELECT 'exists' FROM blocks_user_commands - WHERE block_id = $1 -@@ -2456,11 +2803,12 @@ module Block_and_signed_command = struct - (module Conn) - ~block_id ~user_command_id ~sequence_no ~status ~failure_reason - -- let load (module Conn : CONNECTION) ~block_id ~user_command_id ~sequence_no = -+ let load (module Conn : Mina_caqti.CONNECTION) ~block_id ~user_command_id -+ ~sequence_no = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - typ - (sprintf - {sql| SELECT %s FROM blocks_user_commands -@@ -2481,7 +2829,8 @@ module Zkapp_account_update_failures = struct - - let table_name = "zkapp_account_update_failures" - -- let add_if_doesn't_exist (module Conn : CONNECTION) index failures = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) index failures -+ = - let failures = - List.map failures ~f:Transaction_status.Failure.to_string |> Array.of_list - in -@@ -2492,9 +2841,9 @@ module Zkapp_account_update_failures = struct - (module Conn) - { index; failures } - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name - ~cols:[ "index"; "failures" ] ) ) - id -@@ -2516,7 +2865,7 @@ module Block_and_zkapp_command = struct - Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist - Caqti_type.[ int; int; int; string; option Mina_caqti.array_int_typ ] - -- let add_if_doesn't_exist (module Conn : CONNECTION) ~block_id -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) ~block_id - ~zkapp_command_id ~sequence_no ~status - ~(failure_reasons : Transaction_status.Failure.Collection.Display.t option) - = -@@ -2537,8 +2886,7 @@ module Block_and_zkapp_command = struct - in - Mina_caqti.select_insert_into_cols - ~select: -- ( "block_id, zkapp_command_id, sequence_no" -- , Caqti_type.(tup3 int int int) ) -+ ("block_id, zkapp_command_id, sequence_no", Caqti_type.(t3 int int int)) - ~table_name - ~cols: - ( [ "block_id" -@@ -2558,21 +2906,22 @@ module Block_and_zkapp_command = struct - (module Conn) - { block_id; zkapp_command_id; sequence_no; status; failure_reasons_ids } - -- let load (module Conn : CONNECTION) ~block_id ~zkapp_command_id ~sequence_no = -+ let load (module Conn : Mina_caqti.CONNECTION) ~block_id ~zkapp_command_id -+ ~sequence_no = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - typ - (Mina_caqti.select_cols ~table_name ~select:comma_cols - ~cols:[ "block_id"; "zkapp_command_id"; "sequence_no" ] - () ) ) - (block_id, zkapp_command_id, sequence_no) - -- let all_from_block (module Conn : CONNECTION) ~block_id = -+ let all_from_block (module Conn : Mina_caqti.CONNECTION) ~block_id = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.collect_list -- (Caqti_request.collect Caqti_type.int typ -+ (collect_req Caqti_type.int typ - (Mina_caqti.select_cols ~table_name ~select:comma_cols - ~cols:[ "block_id" ] () ) ) - block_id -@@ -2596,7 +2945,7 @@ module Zkapp_account = struct - - let table_name = "zkapp_accounts" - -- let add_if_doesn't_exist (module Conn : CONNECTION) zkapp_account = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) zkapp_account = - let open Deferred.Result.Let_syntax in - let ({ app_state - ; verification_key -@@ -2642,9 +2991,9 @@ module Zkapp_account = struct - ; zkapp_uri_id - } - -- let load (module Conn : CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -2685,11 +3034,31 @@ module Accounts_accessed = struct - - let table_name = "accounts_accessed" - -- let find_opt (module Conn : CONNECTION) ~block_id ~account_identifier_id = -+ type local_copy = (t, int) Hashtbl.t -+ -+ let local_copies = Hashtbl.create (module String) -+ -+ let load_copy = -+ load_copy' -+ ~default:(fun () -> -+ Hash_set.create -+ ( module struct -+ type t = int * int [@@deriving compare, sexp, hash] -+ end ) ) -+ ~local_copies -+ ~typ:Caqti_type.(t2 int int) -+ ~query: -+ {sql| SELECT block_id,account_identifier_id FROM accounts_accessed |sql} -+ ~load_elt:(fun exists_index key -> -+ Hash_set.add exists_index key ; -+ Deferred.unit ) -+ -+ let find_opt (module Conn : Mina_caqti.CONNECTION) ~block_id -+ ~account_identifier_id = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 int int) -+ (find_opt_req -+ Caqti_type.(t2 int int) - typ - (sprintf - {sql| SELECT %s -@@ -2700,72 +3069,71 @@ module Accounts_accessed = struct - comma_cols table_name ) ) - (block_id, account_identifier_id) - -- let add_if_doesn't_exist (module Conn : CONNECTION) block_id -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) block_id - (ledger_index, (account : Account.t)) = -+ let%bind exists_index = load_copy (module Conn) in - let open Deferred.Result.Let_syntax in - let account_id = Account_id.create account.public_key account.token_id in - let%bind account_identifier_id = - Account_identifiers.add_if_doesn't_exist (module Conn) account_id - in -- match%bind find_opt (module Conn) ~block_id ~account_identifier_id with -- | Some result -> -- return (result.block_id, result.account_identifier_id) -- | None -> -- let%bind token_symbol_id = -- Token_symbols.add_if_doesn't_exist (module Conn) account.token_symbol -- in -- let balance = Currency.Balance.to_string account.balance in -- let nonce = -- account.nonce |> Account.Nonce.to_uint32 |> Unsigned.UInt32.to_int64 -- in -- let receipt_chain_hash = -- account.receipt_chain_hash |> Receipt.Chain_hash.to_base58_check -- in -- let%bind delegate_id = -- Mina_caqti.add_if_some -- (Public_key.add_if_doesn't_exist (module Conn)) -- account.delegate -- in -- let%bind voting_for_id = -- Voting_for.add_if_doesn't_exist (module Conn) account.voting_for -- in -- let%bind timing_id = -- Timing_info.add_if_doesn't_exist -- (module Conn) -- account_identifier_id account.timing -- in -- let%bind permissions_id = -- Zkapp_permissions.add_if_doesn't_exist -- (module Conn) -- account.permissions -- in -- let%bind zkapp_id = -- Mina_caqti.add_if_some -- (Zkapp_account.add_if_doesn't_exist (module Conn)) -- account.zkapp -- in -- let account_accessed : t = -- { ledger_index -- ; block_id -- ; account_identifier_id -- ; token_symbol_id -- ; balance -- ; nonce -- ; receipt_chain_hash -- ; delegate_id -- ; voting_for_id -- ; timing_id -- ; permissions_id -- ; zkapp_id -- } -- in -- Mina_caqti.select_insert_into_cols -- ~select:("block_id,account_identifier_id", Caqti_type.(tup2 int int)) -- ~table_name ~cols:(Fields.names, typ) -+ if Hash_set.mem exists_index (block_id, account_identifier_id) then -+ return (block_id, account_identifier_id) -+ else -+ let%bind token_symbol_id = -+ Token_symbols.add_if_doesn't_exist (module Conn) account.token_symbol -+ in -+ let balance = Currency.Balance.to_string account.balance in -+ let nonce = -+ account.nonce |> Account.Nonce.to_uint32 |> Unsigned.UInt32.to_int64 -+ in -+ let receipt_chain_hash = -+ account.receipt_chain_hash |> Receipt.Chain_hash.to_base58_check -+ in -+ let%bind delegate_id = -+ Mina_caqti.add_if_some -+ (Public_key.add_if_doesn't_exist (module Conn)) -+ account.delegate -+ in -+ let%bind voting_for_id = -+ Voting_for.add_if_doesn't_exist (module Conn) account.voting_for -+ in -+ let%bind timing_id = -+ Timing_info.add_if_doesn't_exist - (module Conn) -- account_accessed -+ account_identifier_id account.timing -+ in -+ let%bind permissions_id = -+ Zkapp_permissions.add_if_doesn't_exist (module Conn) account.permissions -+ in -+ let%bind zkapp_id = -+ Mina_caqti.add_if_some -+ (Zkapp_account.add_if_doesn't_exist (module Conn)) -+ account.zkapp -+ in -+ let account_accessed : t = -+ { ledger_index -+ ; block_id -+ ; account_identifier_id -+ ; token_symbol_id -+ ; balance -+ ; nonce -+ ; receipt_chain_hash -+ ; delegate_id -+ ; voting_for_id -+ ; timing_id -+ ; permissions_id -+ ; zkapp_id -+ } -+ in -+ Hash_set.add exists_index (block_id, account_identifier_id) ; -+ Mina_caqti.insert_assuming_new -+ ~select:("block_id,account_identifier_id", Caqti_type.(t2 int int)) -+ ~table_name ~cols:(Fields.names, typ) -+ (module Conn) -+ account_accessed - -- let add_accounts_if_don't_exist (module Conn : CONNECTION) block_id -+ let add_accounts_if_don't_exist (module Conn : Mina_caqti.CONNECTION) block_id - (accounts : (int * Account.t) list) = - let%map results = - Deferred.List.map accounts ~f:(fun account -> -@@ -2773,10 +3141,10 @@ module Accounts_accessed = struct - in - Result.all results - -- let all_from_block (module Conn : CONNECTION) block_id = -+ let all_from_block (module Conn : Mina_caqti.CONNECTION) block_id = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.collect_list -- (Caqti_request.collect Caqti_type.int typ -+ (collect_req Caqti_type.int typ - (Mina_caqti.select_cols ~select:comma_cols ~table_name - ~cols:[ "block_id" ] () ) ) - block_id -@@ -2793,30 +3161,30 @@ module Accounts_created = struct - - let table_name = "accounts_created" - -- let add_if_doesn't_exist (module Conn : CONNECTION) block_id account_id -- creation_fee = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) block_id -+ account_id creation_fee = - let open Deferred.Result.Let_syntax in - let%bind account_identifier_id = - Account_identifiers.add_if_doesn't_exist (module Conn) account_id - in - let creation_fee = Currency.Fee.to_string creation_fee in - Mina_caqti.select_insert_into_cols -- ~select:("block_id,account_identifier_id", Caqti_type.(tup2 int int)) -+ ~select:("block_id,account_identifier_id", Caqti_type.(t2 int int)) - ~table_name ~cols:(Fields.names, typ) - (module Conn) - { block_id; account_identifier_id; creation_fee } - -- let add_accounts_created_if_don't_exist (module Conn : CONNECTION) block_id -- accounts_created = -+ let add_accounts_created_if_don't_exist (module Conn : Mina_caqti.CONNECTION) -+ block_id accounts_created = - let%map results = - Deferred.List.map accounts_created ~f:(fun (pk, creation_fee) -> - add_if_doesn't_exist (module Conn) block_id pk creation_fee ) - in - Result.all results - -- let all_from_block (module Conn : CONNECTION) block_id = -+ let all_from_block (module Conn : Mina_caqti.CONNECTION) block_id = - Conn.collect_list -- (Caqti_request.collect Caqti_type.int typ -+ (collect_req Caqti_type.int typ - {sql| SELECT block_id, account_identifier_id, creation_fee - FROM accounts_created - WHERE block_id = ? -@@ -2882,18 +3250,19 @@ module Block = struct - "SELECT id FROM blocks WHERE state_hash = ?" ) - (State_hash.to_base58_check state_hash) - -- let find (module Conn : CONNECTION) = make_finder Conn.find Caqti_request.find -+ let find (module Conn : Mina_caqti.CONNECTION) = -+ make_finder Conn.find Mina_caqti.find_req - -- let find_opt (module Conn : CONNECTION) = -- make_finder Conn.find_opt Caqti_request.find_opt -+ let find_opt (module Conn : Mina_caqti.CONNECTION) = -+ make_finder Conn.find_opt Mina_caqti.find_opt_req - -- let load (module Conn : CONNECTION) ~id = -+ let load (module Conn : Mina_caqti.CONNECTION) ~id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name:"blocks" ~cols:Fields.names) ) - id - -- let add_parts_if_doesn't_exist (module Conn : CONNECTION) -+ let add_parts_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) - ~constraint_constants ~protocol_state ~staged_ledger_diff - ~protocol_version ~proposed_protocol_version ~hash ~v1_transaction_hash = - let open Deferred.Result.Let_syntax in -@@ -2994,7 +3363,7 @@ module Block = struct - let blockchain_state = Protocol_state.blockchain_state protocol_state in - let%bind block_id = - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "chain_status" -> -@@ -3314,7 +3683,7 @@ module Block = struct - - (* NB: this batching logic an lead to partial writes; it is acceptable to be used with the - migration tool, but not acceptable to be used with the archive node in its current form *) -- let add_from_extensional_batch (module Conn : CONNECTION) -+ let add_from_extensional_batch (module Conn : Mina_caqti.CONNECTION) - ?(v1_transaction_hash = false) (blocks : Extensional.Block.t list) = - let open Deferred.Result.Let_syntax in - (* zkapps are currently unsupported in the batch implementation of this function *) -@@ -3376,9 +3745,7 @@ module Block = struct - in - - (* we don't need to specify all types here, just the ones that sql may infer incorrectly *) -- let field_name : type a. a Caqti_type.Field.t -> string option = -- let open Caqti_type in -- function -+ let field_name : type a. a Caqti_type.Field.t -> string option = function - | Bool -> - Some "BOOL" - | Int -> -@@ -3391,39 +3758,36 @@ module Block = struct - Some "BIGINT" - | Float -> - Some "FLOAT" -+ | String -> -+ Some "TEXT" - | Enum name -> - Some name -- | _ -> -- None -+ | Octets | Pdate | Ptime | Ptime_span -> -+ failwith "unsupported field type" - in - - let rec type_field_names : type a. a Caqti_type.t -> string option list = -+ let open Caqti_type in - function -- | Unit -> -- [] - | Field f -> - [ field_name f ] - | Option t -> - type_field_names t -- | Tup2 (at, bt) -> -- List.concat [ type_field_names at; type_field_names bt ] -- | Tup3 (at, bt, ct) -> -- List.concat -- [ type_field_names at; type_field_names bt; type_field_names ct ] -- | Tup4 (at, bt, ct, dt) -> -- List.concat -- [ type_field_names at -- ; type_field_names bt -- ; type_field_names ct -- ; type_field_names dt -- ] -- | Custom custom -> -- type_field_names custom.rep -- in -- -- let rec render_field : type a. a Caqti_type.Field.t -> a -> string = -+ | Product (_, prod) -> -+ let rec loop : type a i. _ -> (a, i) product -> _ = -+ fun acc -> function -+ | Proj_end -> -+ List.concat (List.rev acc) -+ | Proj (t, _, prod) -> -+ loop (type_field_names t :: acc) prod -+ in -+ loop [] prod -+ | Annot (_, t) -> -+ type_field_names t -+ in -+ -+ let render_field : type a. a Caqti_type.Field.t -> a -> string = - fun typ value -> -- let open Caqti_type in - match typ with - | Bool -> - Bool.to_string value -@@ -3451,45 +3815,30 @@ module Block = struct - (* we are ignoring the enum annotation in this context because it's not always valid to apply *) - (* NOTE: we assume enum values do not contain special characters (eg "'") *) - "'" ^ value ^ "'" -- | _ -> ( -- match Caqti_type.Field.coding Conn.driver_info typ with -- | None -> -- failwithf "unable to render caqti field: %s" -- (Caqti_type.Field.to_string typ) -- () -- | Some (Coding coding) -> -- render_field coding.rep -- (Result.ok_or_failwith @@ coding.encode value) ) - in - let rec render_type : type a. a Caqti_type.t -> a -> string list = -- fun typ value -> -- match typ with -- | Unit -> -- [] -- | Field f -> -- [ render_field f value ] -- | Option t -> ( -- match value with -- | None -> -- List.init (Caqti_type.length typ) ~f:(Fn.const "NULL") -- | Some x -> -- render_type t x ) -- | Tup2 (at, bt) -> -- let a, b = value in -- List.concat [ render_type at a; render_type bt b ] -- | Tup3 (at, bt, ct) -> -- let a, b, c = value in -- List.concat [ render_type at a; render_type bt b; render_type ct c ] -- | Tup4 (at, bt, ct, dt) -> -- let a, b, c, d = value in -- List.concat -- [ render_type at a -- ; render_type bt b -- ; render_type ct c -- ; render_type dt d -- ] -- | Custom custom -> -- render_type custom.rep (Result.ok_or_failwith @@ custom.encode value) -+ let open Caqti_type in -+ fun typ value -> -+ match typ with -+ | Field f -> -+ [ render_field f value ] -+ | Option t -> ( -+ match value with -+ | None -> -+ List.init (Caqti_type.length typ) ~f:(Fn.const "NULL") -+ | Some x -> -+ render_type t x ) -+ | Product (_, prod) -> -+ let rec loop : type i. _ -> (a, i) product -> _ = -+ fun acc -> function -+ | Proj_end -> -+ List.concat (List.rev acc) -+ | Proj (t, prj, prod) -> -+ loop (render_type t (prj value) :: acc) prod -+ in -+ loop [] prod -+ | Annot (_, t) -> -+ render_type t value - in - let render_row (type a) (typ : a Caqti_type.t) (value : a) : string = - "(" ^ String.concat ~sep:"," (render_type typ value) ^ ")" -@@ -3540,8 +3889,8 @@ module Block = struct - in - let%map entries = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 typ int) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 typ int) - query ) - () - in -@@ -3559,7 +3908,7 @@ module Block = struct - String.concat ~sep:"," @@ List.map ~f:(render_row typ) values - in - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit Caqti_type.int -+ (Mina_caqti.collect_req Caqti_type.unit Caqti_type.int - (sprintf "INSERT INTO %s (%s) VALUES %s RETURNING id" table - fields_sql values_sql ) ) - () ) -@@ -4050,7 +4399,7 @@ module Block = struct - let ids_sql = String.concat ~sep:"," ids in - let parent_ids_sql = String.concat ~sep:"," parent_ids in - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - "UPDATE %s AS b SET parent_id = data.parent_id FROM (SELECT \ - unnest(array[%s]) as id, unnest(array[%s]) as parent_id) AS \ -@@ -4060,7 +4409,7 @@ module Block = struct - in - return () - -- let add_from_extensional (module Conn : CONNECTION) -+ let add_from_extensional (module Conn : Mina_caqti.CONNECTION) - ?(v1_transaction_hash = false) (block : Extensional.Block.t) = - let open Deferred.Result.Let_syntax in - let%bind block_id = -@@ -4119,7 +4468,7 @@ module Block = struct - Some id ) - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "sub_window_densities" -> -@@ -4277,18 +4626,19 @@ module Block = struct - in - return block_id - -- let set_parent_id_if_null (module Conn : CONNECTION) ~parent_hash -+ let set_parent_id_if_null (module Conn : Mina_caqti.CONNECTION) ~parent_hash - ~(parent_id : int) = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int string) -+ (exec_req -+ Caqti_type.(t2 int string) - {sql| UPDATE blocks SET parent_id = ? - WHERE parent_hash = ? - AND parent_id IS NULL - |sql} ) - (parent_id, State_hash.to_base58_check parent_hash) - -- let get_subchain (module Conn : CONNECTION) ~start_block_id ~end_block_id = -+ let get_subchain (module Conn : Mina_caqti.CONNECTION) ~start_block_id -+ ~end_block_id = - (* derive query from type `t` *) - let concat = String.concat ~sep:"," in - let columns_with_id = concat ("id" :: Fields.names) in -@@ -4297,8 +4647,8 @@ module Block = struct - in - let columns = concat Fields.names in - Conn.collect_list -- (Caqti_request.collect -- Caqti_type.(tup2 int int) -+ (collect_req -+ Caqti_type.(t2 int int) - typ - (sprintf - {sql| WITH RECURSIVE chain AS ( -@@ -4322,39 +4672,42 @@ module Block = struct - columns_with_id b_columns_with_id columns ) ) - (end_block_id, start_block_id) - -- let get_highest_canonical_block_opt (module Conn : CONNECTION) = -+ let get_highest_canonical_block_opt (module Conn : Mina_caqti.CONNECTION) = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.unit -- Caqti_type.(tup2 int int64) -+ (find_opt_req Caqti_type.unit -+ Caqti_type.(t2 int int64) - "SELECT id,height FROM blocks WHERE chain_status='canonical' ORDER BY \ - height DESC LIMIT 1" ) - -- let get_nearest_canonical_block_above (module Conn : CONNECTION) height = -+ let get_nearest_canonical_block_above (module Conn : Mina_caqti.CONNECTION) -+ height = - Conn.find -- (Caqti_request.find Caqti_type.int64 -- Caqti_type.(tup2 int int64) -+ (find_req Caqti_type.int64 -+ Caqti_type.(t2 int int64) - "SELECT id,height FROM blocks WHERE chain_status='canonical' AND \ - height > ? ORDER BY height ASC LIMIT 1" ) - height - -- let get_nearest_canonical_block_below (module Conn : CONNECTION) height = -+ let get_nearest_canonical_block_below (module Conn : Mina_caqti.CONNECTION) -+ height = - Conn.find -- (Caqti_request.find Caqti_type.int64 -- Caqti_type.(tup2 int int64) -+ (find_req Caqti_type.int64 -+ Caqti_type.(t2 int int64) - "SELECT id,height FROM blocks WHERE chain_status='canonical' AND \ - height < ? ORDER BY height DESC LIMIT 1" ) - height - -- let mark_as_canonical (module Conn : CONNECTION) ~state_hash = -+ let mark_as_canonical (module Conn : Mina_caqti.CONNECTION) ~state_hash = - Conn.exec -- (Caqti_request.exec Caqti_type.string -+ (exec_req Caqti_type.string - "UPDATE blocks SET chain_status='canonical' WHERE state_hash = ?" ) - state_hash - -- let mark_as_orphaned (module Conn : CONNECTION) ~state_hash ~height = -+ let mark_as_orphaned (module Conn : Mina_caqti.CONNECTION) ~state_hash ~height -+ = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 string int64) -+ (exec_req -+ Caqti_type.(t2 string int64) - {sql| UPDATE blocks SET chain_status='orphaned' - WHERE height = $2 - AND state_hash <> $1 -@@ -4362,7 +4715,7 @@ module Block = struct - (state_hash, height) - - (* update chain_status for blocks now known to be canonical or orphaned *) -- let update_chain_status (module Conn : CONNECTION) ~block_id = -+ let update_chain_status (module Conn : Mina_caqti.CONNECTION) ~block_id = - let open Deferred.Result.Let_syntax in - match%bind get_highest_canonical_block_opt (module Conn) () with - | None -> -@@ -4434,7 +4787,7 @@ module Block = struct - Deferred.Result.return () - - let delete_if_older_than ?height ?num_blocks ?timestamp -- (module Conn : CONNECTION) = -+ (module Conn : Mina_caqti.CONNECTION) = - let open Deferred.Result.Let_syntax in - let%bind height = - match (height, num_blocks) with -@@ -4443,7 +4796,7 @@ module Block = struct - | None, Some num_blocks -> ( - match%map - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.unit Caqti_type.int -+ (find_opt_req Caqti_type.unit Caqti_type.int - "SELECT MAX(height) FROM blocks" ) - () - with -@@ -4459,8 +4812,8 @@ module Block = struct - let%bind () = - (* Delete user commands from old blocks. *) - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int int64) -+ (exec_req -+ Caqti_type.(t2 int int64) - "DELETE FROM user_commands\n\ - WHERE id IN\n\ - (SELECT user_command_id FROM blocks_user_commands\n\ -@@ -4471,8 +4824,8 @@ module Block = struct - let%bind () = - (* Delete old blocks. *) - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int int64) -+ (exec_req -+ Caqti_type.(t2 int int64) - "DELETE FROM blocks WHERE blocks.height < ? OR blocks.timestamp < \ - ?" ) - (height, timestamp) -@@ -4480,7 +4833,7 @@ module Block = struct - let%bind () = - (* Delete orphaned internal commands. *) - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (exec_req Caqti_type.unit - "DELETE FROM internal_commands\n\ - WHERE id NOT IN\n\ - (SELECT internal_commands.id FROM internal_commands\n\ -@@ -4491,7 +4844,7 @@ module Block = struct - let%bind () = - (* Delete orphaned snarked ledger hashes. *) - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (exec_req Caqti_type.unit - "DELETE FROM snarked_ledger_hashes\n\ - WHERE id NOT IN\n\ - (SELECT snarked_ledger_hash_id FROM blocks)" ) -@@ -4500,7 +4853,7 @@ module Block = struct - let%bind () = - (* Delete orphaned public keys. *) - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (exec_req Caqti_type.unit - "DELETE FROM public_keys\n\ - WHERE id NOT IN (SELECT fee_payer_id FROM user_commands)\n\ - AND id NOT IN (SELECT source_id FROM user_commands)\n\ -@@ -4550,8 +4903,8 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash - () - | Some acct_id -> - Token_owners.add_if_doesn't_exist token_id acct_id ) ; -- Caqti_async.Pool.use -- (fun (module Conn : CONNECTION) -> -+ Mina_caqti.Pool.use -+ (fun (module Conn : Mina_caqti.CONNECTION) -> - let%bind res = - let open Deferred.Result.Let_syntax in - let%bind () = Conn.start () in -@@ -4562,7 +4915,7 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash - O1trace.thread "archive_processor.add_block" - @@ fun () -> - Metrics.time ~label:"add_block" -- @@ fun () -> add_block (module Conn : CONNECTION) block -+ @@ fun () -> add_block (module Conn : Mina_caqti.CONNECTION) block - in - (* if an existing block has a parent hash that's for the block just added, - set its parent id -@@ -4617,8 +4970,8 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash - ] ; - let%bind.Deferred.Result () = Conn.start () in - match%bind -- Caqti_async.Pool.use -- (fun (module Conn : CONNECTION) -> -+ Mina_caqti.Pool.use -+ (fun (module Conn : Mina_caqti.CONNECTION) -> - Accounts_accessed.add_accounts_if_don't_exist - (module Conn) - block_id accounts_accessed ) -@@ -4643,8 +4996,8 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash - , `Int (List.length accounts_accessed) ) - ] ; - match%bind -- Caqti_async.Pool.use -- (fun (module Conn : CONNECTION) -> -+ Mina_caqti.Pool.use -+ (fun (module Conn : Mina_caqti.CONNECTION) -> - Accounts_created.add_accounts_created_if_don't_exist - (module Conn) - block_id accounts_created ) -@@ -4758,8 +5111,8 @@ let add_genesis_accounts ~logger ~(runtime_config_opt : Runtime_config.t option) - With_hash.{ data = block; hash = the_hash } - in - let add_accounts () = -- Caqti_async.Pool.use -- (fun (module Conn : CONNECTION) -> -+ Mina_caqti.Pool.use -+ (fun (module Conn : Mina_caqti.CONNECTION) -> - let%bind.Deferred.Result genesis_block_id = - Block.add_if_doesn't_exist - (module Conn) -@@ -4893,7 +5246,7 @@ let setup_server ~metrics_server_port ~constraint_constants ~logger - Strict_pipe.Writer.write extensional_block_writer extensional_block ) - ] - in -- match Caqti_async.connect_pool ~max_size:30 postgres_address with -+ match Mina_caqti.connect_pool ~max_size:128 postgres_address with - | Error e -> - [%log error] - "Failed to create a Caqti pool for Postgresql, see error: $error" -diff --git a/src/app/archive/lib/test.ml b/src/app/archive/lib/test.ml -index 7795a5817f..6f31ad3523 100644 ---- a/src/app/archive/lib/test.ml -+++ b/src/app/archive/lib/test.ml -@@ -37,13 +37,13 @@ let%test_module "Archive node unit tests" = - @@ fun () -> - match%map Caqti_async.connect archive_uri with - | Ok conn -> -- conn -+ Mina_caqti.wrap_conn ~source:archive_uri conn - | Error e -> - failwith @@ Caqti_error.show e ) - - let conn_pool_lazy = - lazy -- ( match Caqti_async.connect_pool archive_uri with -+ ( match Mina_caqti.connect_pool archive_uri with - | Ok pool -> - pool - | Error e -> -@@ -312,7 +312,7 @@ let%test_module "Archive node unit tests" = - match%map - Mina_caqti.deferred_result_list_fold breadcrumbs ~init:() - ~f:(fun () breadcrumb -> -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun conn -> - let open Deferred.Result.Let_syntax in - match%bind -diff --git a/src/app/archive_blocks/archive_blocks.ml b/src/app/archive_blocks/archive_blocks.ml -index caa4203939..a2934a6184 100644 ---- a/src/app/archive_blocks/archive_blocks.ml -+++ b/src/app/archive_blocks/archive_blocks.ml -@@ -20,7 +20,7 @@ let main ~archive_uri ~precomputed ~extensional ~success_file ~failure_file - if Bool.equal precomputed extensional then - failwith "Must provide exactly one of -precomputed and -extensional" ; - let logger = Logger.create () in -- match Caqti_async.connect_pool archive_uri with -+ match Mina_caqti.connect_pool archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] - -diff --git a/src/app/best_tip_merger/best_tip_merger.ml b/src/app/best_tip_merger/best_tip_merger.ml -index 5682f29ec5..90f418c153 100644 ---- a/src/app/best_tip_merger/best_tip_merger.ml -+++ b/src/app/best_tip_merger/best_tip_merger.ml -@@ -205,7 +205,7 @@ module Compact_display = struct - { current : State_hash.t - ; parent : State_hash.t - ; blockchain_length : Mina_numbers.Length.t -- ; global_slot : Mina_numbers.Global_slot_since_genesis.t -+ ; global_slot : Mina_numbers.Global_slot_since_hard_fork.t - } - [@@deriving yojson] - -@@ -243,7 +243,7 @@ module Graph_node = struct - | Node of - { current : State_hash.t - ; length : Mina_numbers.Length.t -- ; slot : Mina_numbers.Global_slot_since_genesis.t -+ ; slot : Mina_numbers.Global_slot_since_hard_fork.t - } - [@@deriving yojson, equal, hash] - -@@ -267,7 +267,7 @@ module Graph_node = struct - ("NA", "NA") - | Node s -> - ( Mina_numbers.Length.to_string s.length -- , Mina_numbers.Global_slot_since_genesis.to_string s.slot ) -+ , Mina_numbers.Global_slot_since_hard_fork.to_string s.slot ) - in - { state; slot; length; peers = t.peers } - -diff --git a/src/app/delegation_compliance/delegation_compliance.ml b/src/app/delegation_compliance/delegation_compliance.ml -index 8a0e92a6c5..d91071a33c 100644 ---- a/src/app/delegation_compliance/delegation_compliance.ml -+++ b/src/app/delegation_compliance/delegation_compliance.ml -@@ -113,7 +113,7 @@ let global_slot_hashes_tbl : (Int64.t, State_hash.t * Ledger_hash.t) Hashtbl.t = - let pk_tbl : (int, Account.key) Hashtbl.t = Int.Table.create () - - let query_db pool ~f ~item = -- match%bind Caqti_async.Pool.use f pool with -+ match%bind Mina_caqti.Pool.use f pool with - | Ok v -> - return v - | Error msg -> -@@ -141,7 +141,7 @@ let pk_of_pk_id pool pk_id : Account.key Deferred.t = - | None -> ( - (* not in cache, consult database *) - match%map -- Caqti_async.Pool.use (fun db -> Sql.Public_key.run db pk_id) pool -+ Mina_caqti.Pool.use (fun db -> Sql.Public_key.run db pk_id) pool - with - | Ok (Some pk) -> ( - match Signature_lib.Public_key.Compressed.of_base58_check pk with -@@ -162,7 +162,7 @@ let pk_of_pk_id pool pk_id : Account.key Deferred.t = - let pk_id_of_pk pool pk : int Deferred.t = - let open Deferred.Let_syntax in - match%map -- Caqti_async.Pool.use (fun db -> Sql.Public_key.run_for_id db pk) pool -+ Mina_caqti.Pool.use (fun db -> Sql.Public_key.run_for_id db pk) pool - with - | Ok (Some id) -> - id -@@ -343,7 +343,7 @@ let main ~input_file ~csv_file ~preliminary_csv_file_opt ~archive_uri - csv_datas - in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match Mina_caqti.connect_pool ~max_size:128 archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -@@ -603,7 +603,7 @@ let main ~input_file ~csv_file ~preliminary_csv_file_opt ~archive_uri - ] ; - let%bind coinbase_receiver_ids = - match%map -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> - Sql.Coinbase_receivers_for_block_creator.run db - ~block_creator_id:delegatee_id ) -diff --git a/src/app/delegation_compliance/sql.ml b/src/app/delegation_compliance/sql.ml -index 3d214eb9c9..713ea23916 100644 ---- a/src/app/delegation_compliance/sql.ml -+++ b/src/app/delegation_compliance/sql.ml -@@ -16,7 +16,7 @@ module Block_info = struct - - (* find all blocks, working back from block with given state hash *) - let query = -- Caqti_request.collect Caqti_type.string typ -+ Mina_caqti.collect_req Caqti_type.string typ - {sql| WITH RECURSIVE chain AS ( - - SELECT id,parent_id,global_slot,state_hash,ledger_hash FROM blocks b WHERE b.state_hash = ? -@@ -34,7 +34,7 @@ module Block_info = struct - - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) state_hash = -+ let run (module Conn : Mina_caqti.CONNECTION) state_hash = - Conn.collect_list query state_hash - end - -@@ -122,7 +122,7 @@ module User_command = struct - Caqti_type.custom ~encode ~decode (to_rep spec) - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT type,fee_payer_id, source_id,receiver_id,fee,fee_token,token,amount,valid_until,memo,nonce, - blocks.id,blocks.global_slot,parent.global_slot_since_genesis, - sequence_no,status,created_token, -@@ -144,12 +144,12 @@ module User_command = struct - - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) user_cmd_id = -+ let run (module Conn : Mina_caqti.CONNECTION) user_cmd_id = - Conn.collect_list query user_cmd_id - - let query_payments_by_source_and_receiver = -- Caqti_request.collect -- Caqti_type.(tup2 int int) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 int int) - typ - {sql| SELECT type,fee_payer_id, source_id, receiver_id, fee,fee_token, - token, amount, valid_until, memo, nonce, blocks.id, blocks.global_slot, -@@ -176,13 +176,13 @@ module User_command = struct - - |sql} - -- let run_payments_by_source_and_receiver (module Conn : Caqti_async.CONNECTION) -+ let run_payments_by_source_and_receiver (module Conn : Mina_caqti.CONNECTION) - ~source_id ~receiver_id = - Conn.collect_list query_payments_by_source_and_receiver - (source_id, receiver_id) - - let query_payments_by_receiver = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT type,fee_payer_id, source_id, receiver_id, fee,fee_token, - token, amount, valid_until, memo, nonce, blocks.id, blocks.global_slot, - parent.global_slot_since_genesis, sequence_no, status, created_token, -@@ -207,51 +207,51 @@ module User_command = struct - - |sql} - -- let run_payments_by_receiver (module Conn : Caqti_async.CONNECTION) -+ let run_payments_by_receiver (module Conn : Mina_caqti.CONNECTION) - ~receiver_id = - Conn.collect_list query_payments_by_receiver receiver_id - end - - module Public_key = struct - let query = -- Caqti_request.find_opt Caqti_type.int Caqti_type.string -+ Mina_caqti.find_opt_req Caqti_type.int Caqti_type.string - {sql| SELECT value FROM public_keys - WHERE id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) pk_id = -+ let run (module Conn : Mina_caqti.CONNECTION) pk_id = - Conn.find_opt query pk_id - - let query_for_id = -- Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ Mina_caqti.find_opt_req Caqti_type.string Caqti_type.int - {sql| SELECT id FROM public_keys - WHERE value = ? - |sql} - -- let run_for_id (module Conn : Caqti_async.CONNECTION) pk = -+ let run_for_id (module Conn : Mina_caqti.CONNECTION) pk = - Conn.find_opt query_for_id pk - end - - module Block = struct - let max_slot_query = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int - {sql| SELECT MAX(global_slot) FROM blocks - |sql} - -- let get_max_slot (module Conn : Caqti_async.CONNECTION) () = -+ let get_max_slot (module Conn : Mina_caqti.CONNECTION) () = - Conn.find max_slot_query () - - let state_hashes_by_slot_query = -- Caqti_request.collect Caqti_type.int Caqti_type.string -+ Mina_caqti.collect_req Caqti_type.int Caqti_type.string - {sql| SELECT state_hash FROM blocks WHERE global_slot = $1 - |sql} - -- let get_state_hashes_by_slot (module Conn : Caqti_async.CONNECTION) slot = -+ let get_state_hashes_by_slot (module Conn : Mina_caqti.CONNECTION) slot = - Conn.collect_list state_hashes_by_slot_query slot - - let creator_slot_bounds_query = -- Caqti_request.collect -- Caqti_type.(tup3 int int64 int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t3 int int64 int64) - Caqti_type.int - {sql| SELECT id FROM blocks - WHERE creator_id = $1 -@@ -259,7 +259,7 @@ module Block = struct - |sql} - - let get_block_ids_for_creator_in_slot_bounds -- (module Conn : Caqti_async.CONNECTION) ~creator ~low_slot ~high_slot = -+ (module Conn : Mina_caqti.CONNECTION) ~creator ~low_slot ~high_slot = - Conn.collect_list creator_slot_bounds_query (creator, low_slot, high_slot) - end - -@@ -268,7 +268,7 @@ module Coinbase_receivers_for_block_creator = struct - with given creator_id, where the receiver distinct from the creator_id - *) - let query = -- Caqti_request.collect Caqti_type.int Caqti_type.int -+ Mina_caqti.collect_req Caqti_type.int Caqti_type.int - {sql| SELECT DISTINCT ic.receiver_id - - FROM blocks b -@@ -290,6 +290,6 @@ module Coinbase_receivers_for_block_creator = struct - AND ic.receiver_id <> b.creator_id - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_creator_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_creator_id = - Conn.collect_list query block_creator_id - end -diff --git a/src/app/extract_blocks/extract_blocks.ml b/src/app/extract_blocks/extract_blocks.ml -index 3605538ae4..a104ffaa68 100644 ---- a/src/app/extract_blocks/extract_blocks.ml -+++ b/src/app/extract_blocks/extract_blocks.ml -@@ -479,7 +479,7 @@ let main ~archive_uri ~start_state_hash_opt ~end_state_hash_opt ~all_blocks () = - (* sanity-check input state hashes *) - check_state_hash ~logger start_state_hash_opt ; - check_state_hash ~logger end_state_hash_opt ; -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match Mina_caqti.connect_pool ~max_size:128 archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/extract_blocks/sql.ml b/src/app/extract_blocks/sql.ml -index 2c4b659973..5233a16a89 100644 ---- a/src/app/extract_blocks/sql.ml -+++ b/src/app/extract_blocks/sql.ml -@@ -31,23 +31,23 @@ module Subchain = struct - fields b_fields join_condition fields - - let query_unparented = -- Caqti_request.collect Caqti_type.string Archive_lib.Processor.Block.typ -+ Mina_caqti.collect_req Caqti_type.string Archive_lib.Processor.Block.typ - (make_sql ~join_condition:"b.id = chain.parent_id") - - let query_from_start = -- Caqti_request.collect -- Caqti_type.(tup2 string string) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string string) - Archive_lib.Processor.Block.typ - (make_sql - ~join_condition: - "b.id = chain.parent_id AND (chain.state_hash <> $2 OR b.state_hash \ - = $2)" ) - -- let start_from_unparented (module Conn : Caqti_async.CONNECTION) -+ let start_from_unparented (module Conn : Mina_caqti.CONNECTION) - ~end_state_hash = - Conn.collect_list query_unparented end_state_hash - -- let start_from_specified (module Conn : Caqti_async.CONNECTION) -+ let start_from_specified (module Conn : Mina_caqti.CONNECTION) - ~start_state_hash ~end_state_hash = - Conn.collect_list query_from_start (end_state_hash, start_state_hash) - -@@ -56,10 +56,10 @@ module Subchain = struct - let comma_fields = - String.concat Archive_lib.Processor.Block.Fields.names ~sep:"," - in -- Caqti_request.collect Caqti_type.unit Archive_lib.Processor.Block.typ -+ Mina_caqti.collect_req Caqti_type.unit Archive_lib.Processor.Block.typ - (sprintf "SELECT %s FROM blocks" comma_fields) - -- let all_blocks (module Conn : Caqti_async.CONNECTION) = -+ let all_blocks (module Conn : Mina_caqti.CONNECTION) = - Conn.collect_list query_all () - end - -@@ -67,14 +67,14 @@ end - - module Blocks_and_user_commands = struct - let query = -- Caqti_request.collect Caqti_type.int -- Caqti_type.(tup2 int int) -+ Mina_caqti.collect_req Caqti_type.int -+ Caqti_type.(t2 int int) - {sql| SELECT user_command_id, sequence_no - FROM blocks_user_commands - WHERE block_id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.collect_list query block_id - end - -@@ -91,7 +91,7 @@ module Block_user_command_tokens = struct - Caqti_type.[ string; option int; option int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT tokens.value, owner_public_key_id, owner_token_id - FROM (blocks_user_commands buc - INNER JOIN blocks -@@ -105,7 +105,7 @@ module Block_user_command_tokens = struct - WHERE block_id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.collect_list query block_id - end - -@@ -122,7 +122,7 @@ module Blocks_and_internal_commands = struct - Caqti_type.[ int; int; int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT internal_command_id, sequence_no, secondary_sequence_no - FROM (blocks_internal_commands - INNER JOIN blocks -@@ -130,7 +130,7 @@ module Blocks_and_internal_commands = struct - WHERE block_id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.collect_list query block_id - end - -@@ -147,7 +147,7 @@ module Block_internal_command_tokens = struct - Caqti_type.[ string; option int; option int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT tokens.value, owner_public_key_id, owner_token_id - FROM (blocks_internal_commands bic - INNER JOIN blocks -@@ -161,20 +161,20 @@ module Block_internal_command_tokens = struct - WHERE block_id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.collect_list query block_id - end - - module Blocks_and_zkapp_commands = struct - let query = -- Caqti_request.collect Caqti_type.int -- Caqti_type.(tup2 int int) -+ Mina_caqti.collect_req Caqti_type.int -+ Caqti_type.(t2 int int) - {sql| SELECT zkapp_command_id, sequence_no - FROM blocks_zkapp_commands - WHERE block_id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.collect_list query block_id - end - -@@ -191,7 +191,7 @@ module Block_zkapp_command_tokens = struct - Caqti_type.[ string; option int; option int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT tokens.value, owner_public_key_id, owner_token_id - FROM (blocks_zkapp_commands bzkc - INNER JOIN blocks -@@ -208,6 +208,6 @@ module Block_zkapp_command_tokens = struct - WHERE block_id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.collect_list query block_id - end -diff --git a/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml b/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml -index 65b34d6b55..efe0668cc2 100644 ---- a/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml -+++ b/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml -@@ -67,15 +67,11 @@ let amount_geq_min_balance ~amount ~initial_min_balance = - let generate_delegate_account ~logger delegatee_pk = - [%log info] "Generating account for delegatee $delegatee" - ~metadata:[ ("delegatee", `String delegatee_pk) ] ; -- let pk = Some delegatee_pk in -- let balance = Currency.Balance.zero in -- let timing = None in -- let delegate = None in - { Runtime_config.Json_layout.Accounts.Single.default with -- pk -- ; balance -- ; timing -- ; delegate -+ pk = delegatee_pk -+ ; balance = Currency.Balance.zero -+ ; timing = None -+ ; delegate = None - } - - let generate_missing_delegate_accounts ~logger = -@@ -97,7 +93,6 @@ let runtime_config_account ~logger ~wallet_pk ~amount ~initial_min_balance - ~delegatee_pk = - [%log info] "Processing record for $wallet_pk" - ~metadata:[ ("wallet_pk", `String wallet_pk) ] ; -- let pk = Some wallet_pk in - let balance = Currency.Balance.of_mina_string_exn amount in - let initial_minimum_balance = - (* if omitted in the TSV, use balance *) -@@ -135,7 +130,9 @@ let runtime_config_account ~logger ~wallet_pk ~amount ~initial_min_balance - .initial_minimum_balance - ; cliff_time - ; cliff_amount -- ; vesting_period -+ ; vesting_period = -+ Global_slot_since_genesis.( -+ diff vesting_period zero |> Option.value_exn) - ; vesting_increment - } - in -@@ -144,7 +141,7 @@ let runtime_config_account ~logger ~wallet_pk ~amount ~initial_min_balance - if no_delegatee delegatee_pk then None else Some delegatee_pk - in - { Runtime_config.Json_layout.Accounts.Single.default with -- pk -+ pk = wallet_pk - ; balance - ; timing - ; delegate -diff --git a/src/app/ledger_export_bench/ledger_export_benchmark.ml b/src/app/ledger_export_bench/ledger_export_benchmark.ml -index cd5846345a..5125f8ba90 100644 ---- a/src/app/ledger_export_bench/ledger_export_benchmark.ml -+++ b/src/app/ledger_export_bench/ledger_export_benchmark.ml -@@ -26,7 +26,7 @@ let () = - let cfg = load_daemon_cfg runtime_config () in - let accounts = - match cfg.ledger with -- | None | Some { base = Named _; _ } | Some { base = Hash _; _ } -> -+ | None | Some { base = Named _; _ } | Some { base = Hash; _ } -> - [] - | Some { base = Accounts accs; _ } -> - List.map ~f:Runtime_config.Accounts.Single.to_account accs -diff --git a/src/app/migrate-balances-table/dune b/src/app/migrate-balances-table/dune -index dbb463c78c..dbe6d09aad 100644 ---- a/src/app/migrate-balances-table/dune -+++ b/src/app/migrate-balances-table/dune -@@ -18,6 +18,7 @@ - ;; local libraries - logger - bounded_types -+ mina_caqti - ) - (preprocessor_deps ../../config.mlh) - (instrumentation (backend bisect_ppx)) -diff --git a/src/app/migrate-balances-table/migrate_balances_table.ml b/src/app/migrate-balances-table/migrate_balances_table.ml -index 76d0549c2a..ea2efbce41 100644 ---- a/src/app/migrate-balances-table/migrate_balances_table.ml -+++ b/src/app/migrate-balances-table/migrate_balances_table.ml -@@ -4,7 +4,7 @@ open Core_kernel - open Async - - let query_db pool ~f ~item = -- match%bind Caqti_async.Pool.use f pool with -+ match%bind Mina_caqti.Pool.use f pool with - | Ok v -> - return v - | Error msg -> -@@ -14,7 +14,7 @@ let query_db pool ~f ~item = - let main ~archive_uri () = - let logger = Logger.create () in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match Mina_caqti.connect_pool ~max_size:128 archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/migrate-balances-table/sql.ml b/src/app/migrate-balances-table/sql.ml -index 4c0a8c86dd..be4d000a4d 100644 ---- a/src/app/migrate-balances-table/sql.ml -+++ b/src/app/migrate-balances-table/sql.ml -@@ -2,9 +2,9 @@ - - open Core_kernel - --let create_temp_balances_table (module Conn : Caqti_async.CONNECTION) = -+let create_temp_balances_table (module Conn : Mina_caqti.CONNECTION) = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - {sql| CREATE TABLE IF NOT EXISTS balances_temp - ( id serial PRIMARY KEY - , public_key_id int NOT NULL REFERENCES public_keys(id) -@@ -17,80 +17,80 @@ let create_temp_balances_table (module Conn : Caqti_async.CONNECTION) = - ) - |sql} ) - --let copy_table_to_temp_table (module Conn : Caqti_async.CONNECTION) table = -+let copy_table_to_temp_table (module Conn : Mina_caqti.CONNECTION) table = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE TABLE IF NOT EXISTS %s_temp AS (SELECT * FROM %s) - |sql} - table table ) ) - --let create_table_index (module Conn : Caqti_async.CONNECTION) table col = -+let create_table_index (module Conn : Mina_caqti.CONNECTION) table col = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE INDEX IF NOT EXISTS idx_%s_%s ON %s(%s) - |sql} - table col table col ) ) - --let create_temp_table_index (module Conn : Caqti_async.CONNECTION) table col = -+let create_temp_table_index (module Conn : Mina_caqti.CONNECTION) table col = - create_table_index (module Conn) (sprintf "%s_temp" table) col - --let create_table_named_index (module Conn : Caqti_async.CONNECTION) table col -+let create_table_named_index (module Conn : Mina_caqti.CONNECTION) table col - name = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE INDEX IF NOT EXISTS idx_%s_%s ON %s(%s) - |sql} - table name table col ) ) - --let create_temp_table_named_index (module Conn : Caqti_async.CONNECTION) table -+let create_temp_table_named_index (module Conn : Mina_caqti.CONNECTION) table - col name = - create_table_named_index (module Conn) (sprintf "%s_temp" table) col name - --let drop_table_index (module Conn : Caqti_async.CONNECTION) table col = -+let drop_table_index (module Conn : Mina_caqti.CONNECTION) table col = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf {sql| DROP INDEX IF EXISTS idx_%s_%s - |sql} table col ) ) - --let drop_temp_table_index (module Conn : Caqti_async.CONNECTION) table col = -+let drop_temp_table_index (module Conn : Mina_caqti.CONNECTION) table col = - drop_table_index (module Conn) (sprintf "%s_temp" table) col - --let create_cursor (module Conn : Caqti_async.CONNECTION) name = -+let create_cursor (module Conn : Mina_caqti.CONNECTION) name = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE TABLE IF NOT EXISTS %s_cursor - ( value int NOT NULL) - |sql} - name ) ) - --let initialize_cursor (module Conn : Caqti_async.CONNECTION) name = -+let initialize_cursor (module Conn : Mina_caqti.CONNECTION) name = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| INSERT INTO %s_cursor (value) VALUES (0) - |sql} - name ) ) - --let current_cursor (module Conn : Caqti_async.CONNECTION) name = -+let current_cursor (module Conn : Mina_caqti.CONNECTION) name = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.unit Caqti_type.int -+ (Mina_caqti.find_opt_req Caqti_type.unit Caqti_type.int - (sprintf {sql| SELECT value FROM %s_cursor - |sql} name ) ) - --let update_cursor (module Conn : Caqti_async.CONNECTION) name ndx = -+let update_cursor (module Conn : Mina_caqti.CONNECTION) name ndx = - Conn.exec -- (Caqti_request.exec Caqti_type.int -+ (Mina_caqti.exec_req Caqti_type.int - (sprintf - {sql| UPDATE %s_cursor SET value = $1 - |sql} - name ) ) - ndx - --let drop_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) table -+let drop_foreign_key_constraint (module Conn : Mina_caqti.CONNECTION) table - foreign_key = - let sql = - sprintf -@@ -99,9 +99,9 @@ let drop_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) table - |sql} - table foreign_key - in -- Conn.exec (Caqti_request.exec Caqti_type.unit sql) -+ Conn.exec (Mina_caqti.exec_req Caqti_type.unit sql) - --let add_balances_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) -+let add_balances_foreign_key_constraint (module Conn : Mina_caqti.CONNECTION) - table col foreign_key = - let sql = - sprintf -@@ -112,9 +112,9 @@ let add_balances_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) - |sql} - table foreign_key col - in -- Conn.exec (Caqti_request.exec Caqti_type.unit sql) -+ Conn.exec (Mina_caqti.exec_req Caqti_type.unit sql) - --let add_blocks_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) -+let add_blocks_foreign_key_constraint (module Conn : Mina_caqti.CONNECTION) - table col foreign_key = - let sql = - sprintf -@@ -125,14 +125,14 @@ let add_blocks_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) - |sql} - table foreign_key col - in -- Conn.exec (Caqti_request.exec Caqti_type.unit sql) -+ Conn.exec (Mina_caqti.exec_req Caqti_type.unit sql) - --let find_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id -+let find_balance_entry (module Conn : Mina_caqti.CONNECTION) ~public_key_id - ~balance ~block_id ~block_height ~block_sequence_no - ~block_secondary_sequence_no = - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup3 int int64 (tup4 int int int int)) -+ (Mina_caqti.find_opt_req -+ Caqti_type.(t3 int int64 (t4 int int int int)) - Caqti_type.int - {sql| SELECT id - FROM balances_temp -@@ -148,12 +148,12 @@ let find_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id - , (block_id, block_height, block_sequence_no, block_secondary_sequence_no) - ) - --let insert_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id -+let insert_balance_entry (module Conn : Mina_caqti.CONNECTION) ~public_key_id - ~balance ~block_id ~block_height ~block_sequence_no - ~block_secondary_sequence_no = - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int64 (tup4 int int int int)) -+ (Mina_caqti.find_req -+ Caqti_type.(t3 int int64 (t4 int int int int)) - Caqti_type.int - {sql| INSERT INTO balances_temp - ( public_key_id -@@ -176,10 +176,10 @@ let insert_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id - , (block_id, block_height, block_sequence_no, block_secondary_sequence_no) - ) - --let get_internal_commands (module Conn : Caqti_async.CONNECTION) = -+let get_internal_commands (module Conn : Mina_caqti.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup4 int int64 (tup4 int int int int) int) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t4 int int64 (t4 int int int int) int) - {sql| SELECT bal.public_key_id,bal.balance,bic.block_id,blocks.height,bic.sequence_no,bic.secondary_sequence_no, - internal_command_id - FROM blocks_internal_commands bic -@@ -190,11 +190,11 @@ let get_internal_commands (module Conn : Caqti_async.CONNECTION) = - |sql} ) - - let update_internal_command_receiver_balance -- (module Conn : Caqti_async.CONNECTION) ~new_balance_id ~block_id -+ (module Conn : Mina_caqti.CONNECTION) ~new_balance_id ~block_id - ~internal_command_id ~block_sequence_no ~block_secondary_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup4 int int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t4 int int int int)) - {sql| UPDATE blocks_internal_commands_temp SET receiver_balance = $1 - WHERE block_id = $2 - AND internal_command_id = $3 -@@ -207,10 +207,10 @@ let update_internal_command_receiver_balance - , block_sequence_no - , block_secondary_sequence_no ) ) - --let get_user_command_fee_payers (module Conn : Caqti_async.CONNECTION) = -+let get_user_command_fee_payers (module Conn : Mina_caqti.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (tup4 int int int int) (tup2 int int64)) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 (t4 int int int int) (t2 int int64)) - {sql| SELECT buc.block_id,blocks.height,buc.sequence_no,user_command_id, - bal_fee_payer.public_key_id,bal_fee_payer.balance - FROM blocks_user_commands buc -@@ -220,10 +220,10 @@ let get_user_command_fee_payers (module Conn : Caqti_async.CONNECTION) = - bal_fee_payer.public_key_id,bal_fee_payer.balance) - |sql} ) - --let get_user_command_sources (module Conn : Caqti_async.CONNECTION) = -+let get_user_command_sources (module Conn : Mina_caqti.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (tup4 int int int int) (tup2 int int64)) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 (t4 int int int int) (t2 int int64)) - {sql| SELECT buc.block_id,blocks.height,buc.sequence_no,user_command_id, - bal_source.public_key_id,bal_source.balance - FROM blocks_user_commands buc -@@ -234,10 +234,10 @@ let get_user_command_sources (module Conn : Caqti_async.CONNECTION) = - bal_source.public_key_id,bal_source.balance) - |sql} ) - --let get_user_command_receivers (module Conn : Caqti_async.CONNECTION) = -+let get_user_command_receivers (module Conn : Mina_caqti.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (tup4 int int int int) (tup2 int int64)) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 (t4 int int int int) (t2 int int64)) - {sql| SELECT buc.block_id,blocks.height,buc.sequence_no,user_command_id, - bal_receiver.public_key_id,bal_receiver.balance - FROM blocks_user_commands buc -@@ -248,11 +248,11 @@ let get_user_command_receivers (module Conn : Caqti_async.CONNECTION) = - bal_receiver.public_key_id,bal_receiver.balance) - |sql} ) - --let update_user_command_fee_payer_balance (module Conn : Caqti_async.CONNECTION) -+let update_user_command_fee_payer_balance (module Conn : Mina_caqti.CONNECTION) - ~new_balance_id ~block_id ~user_command_id ~block_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup3 int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t3 int int int)) - {sql| UPDATE blocks_user_commands_temp SET fee_payer_balance = $1 - WHERE block_id = $2 - AND user_command_id = $3 -@@ -260,11 +260,11 @@ let update_user_command_fee_payer_balance (module Conn : Caqti_async.CONNECTION) - |sql} ) - (new_balance_id, (block_id, user_command_id, block_sequence_no)) - --let update_user_command_source_balance (module Conn : Caqti_async.CONNECTION) -+let update_user_command_source_balance (module Conn : Mina_caqti.CONNECTION) - ~new_balance_id ~block_id ~user_command_id ~block_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup3 int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t3 int int int)) - {sql| UPDATE blocks_user_commands_temp SET source_balance = $1 - WHERE block_id = $2 - AND user_command_id = $3 -@@ -273,11 +273,11 @@ let update_user_command_source_balance (module Conn : Caqti_async.CONNECTION) - |sql} ) - (new_balance_id, (block_id, user_command_id, block_sequence_no)) - --let update_user_command_receiver_balance (module Conn : Caqti_async.CONNECTION) -+let update_user_command_receiver_balance (module Conn : Mina_caqti.CONNECTION) - ~new_balance_id ~block_id ~user_command_id ~block_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup3 int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t3 int int int)) - {sql| UPDATE blocks_user_commands_temp SET receiver_balance = $1 - WHERE block_id = $2 - AND user_command_id = $3 -@@ -286,24 +286,24 @@ let update_user_command_receiver_balance (module Conn : Caqti_async.CONNECTION) - |sql} ) - (new_balance_id, (block_id, user_command_id, block_sequence_no)) - --let drop_table (module Conn : Caqti_async.CONNECTION) table = -+let drop_table (module Conn : Mina_caqti.CONNECTION) table = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf {sql| DROP TABLE %s - |sql} table ) ) - --let rename_temp_table (module Conn : Caqti_async.CONNECTION) table = -+let rename_temp_table (module Conn : Mina_caqti.CONNECTION) table = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| ALTER TABLE %s_temp - RENAME TO %s - |sql} - table table ) ) - --let get_column_count (module Conn : Caqti_async.CONNECTION) table = -+let get_column_count (module Conn : Mina_caqti.CONNECTION) table = - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (Mina_caqti.find_req Caqti_type.string Caqti_type.int - {sql| SELECT COUNT(*) FROM information_schema.columns - WHERE table_name=$1 - |sql} ) -diff --git a/src/app/missing_blocks_auditor/dune b/src/app/missing_blocks_auditor/dune -index 623a5079ba..7ec1eae5af 100644 ---- a/src/app/missing_blocks_auditor/dune -+++ b/src/app/missing_blocks_auditor/dune -@@ -18,6 +18,7 @@ - ;; local libraries - logger - bounded_types -+ mina_caqti - ) - (preprocessor_deps ../../config.mlh) - (instrumentation (backend bisect_ppx)) -diff --git a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml -index 5bce995ea2..7fbadcf59b 100644 ---- a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml -+++ b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml -@@ -22,7 +22,7 @@ let add_error, get_exit_code = - let main ~archive_uri () = - let logger = Logger.create () in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match Mina_caqti.connect_pool ~max_size:128 archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -@@ -33,7 +33,7 @@ let main ~archive_uri () = - [%log info] "Querying missing blocks" ; - let%bind missing_blocks_raw = - match%bind -- Caqti_async.Pool.use (fun db -> Sql.Unparented_blocks.run db ()) pool -+ Mina_caqti.Pool.use (fun db -> Sql.Unparented_blocks.run db ()) pool - with - | Ok blocks -> - return blocks -@@ -54,7 +54,7 @@ let main ~archive_uri () = - Deferred.List.iter missing_blocks - ~f:(fun (block_id, state_hash, height, parent_hash) -> - match%map -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> Sql.Missing_blocks_gap.run db height) - pool - with -@@ -76,7 +76,7 @@ let main ~archive_uri () = - [%log info] "Querying for gaps in chain statuses" ; - let%bind highest_canonical = - match%bind -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> Sql.Chain_status.run_highest_canonical db ()) - pool - with -@@ -89,7 +89,7 @@ let main ~archive_uri () = - in - let%bind pending_below = - match%bind -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> - Sql.Chain_status.run_count_pending_below db highest_canonical ) - pool -@@ -118,7 +118,7 @@ let main ~archive_uri () = - ] ) ; - let%bind canonical_chain = - match%bind -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> Sql.Chain_status.run_canonical_chain db highest_canonical) - pool - with -diff --git a/src/app/missing_blocks_auditor/sql.ml b/src/app/missing_blocks_auditor/sql.ml -index 3e2bf01e93..7aa4108f02 100644 ---- a/src/app/missing_blocks_auditor/sql.ml -+++ b/src/app/missing_blocks_auditor/sql.ml -@@ -4,49 +4,49 @@ module Unparented_blocks = struct - (* parent_hashes represent ends of chains leading to an orphan block *) - - let query = -- Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup4 int string int string) -+ Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t4 int string int string) - {sql| - SELECT id, state_hash, height, parent_hash FROM blocks - WHERE parent_id IS NULL - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) () = Conn.collect_list query () -+ let run (module Conn : Mina_caqti.CONNECTION) () = Conn.collect_list query () - end - - module Missing_blocks_gap = struct - let query = -- Caqti_request.find Caqti_type.int Caqti_type.int -+ Mina_caqti.find_req Caqti_type.int Caqti_type.int - {sql| SELECT $1 - MAX(height) - 1 FROM blocks - WHERE height < $1 - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) height = Conn.find query height -+ let run (module Conn : Mina_caqti.CONNECTION) height = Conn.find query height - end - - module Chain_status = struct - let query_highest_canonical = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT max(height) FROM blocks - WHERE chain_status = 'canonical' - |sql} - -- let run_highest_canonical (module Conn : Caqti_async.CONNECTION) () = -+ let run_highest_canonical (module Conn : Mina_caqti.CONNECTION) () = - Conn.find query_highest_canonical () - - let query_count_pending_below = -- Caqti_request.find Caqti_type.int64 Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.int64 Caqti_type.int64 - {sql| SELECT count(*) FROM blocks - WHERE chain_status = 'pending' - AND height <= ? - |sql} - -- let run_count_pending_below (module Conn : Caqti_async.CONNECTION) height = -+ let run_count_pending_below (module Conn : Mina_caqti.CONNECTION) height = - Conn.find query_count_pending_below height - - let query_canonical_chain = -- Caqti_request.collect Caqti_type.int64 -- Caqti_type.(tup3 int string string) -+ Mina_caqti.collect_req Caqti_type.int64 -+ Caqti_type.(t3 int string string) - {sql| WITH RECURSIVE chain AS ( - - (SELECT id, state_hash, parent_id, chain_status -@@ -69,6 +69,6 @@ module Chain_status = struct - ORDER BY id ASC - |sql} - -- let run_canonical_chain (module Conn : Caqti_async.CONNECTION) height = -+ let run_canonical_chain (module Conn : Mina_caqti.CONNECTION) height = - Conn.collect_list query_canonical_chain height - end -diff --git a/src/app/patch_archive_test/patch_archive_test.ml b/src/app/patch_archive_test/patch_archive_test.ml -index 1fbc9e3104..b01932ec95 100644 ---- a/src/app/patch_archive_test/patch_archive_test.ml -+++ b/src/app/patch_archive_test/patch_archive_test.ml -@@ -21,7 +21,7 @@ let make_archive_copy_uri archive_uri = - Uri.with_path archive_uri ("/copy_of_" ^ db) - - let query_db pool ~f ~item = -- match%bind Caqti_async.Pool.use f pool with -+ match%bind Mina_caqti.Pool.use f pool with - | Ok v -> - return v - | Error msg -> -@@ -124,7 +124,7 @@ let main ~archive_uri ~num_blocks_to_patch ~archive_blocks_path - let copy_uri = make_archive_copy_uri archive_uri in - [%log info] "Connecting to original database" ; - let%bind () = -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match Mina_caqti.connect_pool ~max_size:128 archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -@@ -137,7 +137,7 @@ let main ~archive_uri ~num_blocks_to_patch ~archive_blocks_path - [%log info] "Dropping copied database, in case it already exists" ; - let%bind () = - match%bind -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> Sql.Copy_database.run_drop_db db ~copy_db) - pool - with -@@ -159,7 +159,7 @@ let main ~archive_uri ~num_blocks_to_patch ~archive_blocks_path - () - in - [%log info] "Connecting to copied database" ; -- match Caqti_async.connect_pool ~max_size:128 copy_uri with -+ match Mina_caqti.connect_pool ~max_size:128 copy_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/replayer/replayer.ml b/src/app/replayer/replayer.ml -index f9f22df42c..5023ea7217 100644 ---- a/src/app/replayer/replayer.ml -+++ b/src/app/replayer/replayer.ml -@@ -46,7 +46,7 @@ type output = - - module type Get_command_ids = sig - val run : -- Caqti_async.connection -+ (module Mina_caqti.CONNECTION) - -> state_hash:string - -> start_slot:int64 - -> (int list, [> Caqti_error.call_or_retrieve ]) Deferred.Result.t -@@ -173,7 +173,7 @@ let get_slot_hashes slot = Hashtbl.find global_slot_hashes_tbl slot - - let process_block_infos_of_state_hash ~logger pool ~state_hash ~start_slot ~f = - match%bind -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> Sql.Block_info.run db ~state_hash ~start_slot) - pool - with -@@ -661,7 +661,7 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri - msg ) - in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match Mina_caqti.connect_pool ~max_size:128 archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -@@ -799,7 +799,7 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri - (* end mutable state *) - let get_command_ids (module Command_ids : Get_command_ids) name = - match%bind -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> - Command_ids.run db ~state_hash:target_state_hash - ~start_slot:input.start_slot_since_genesis ) -@@ -835,7 +835,7 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri - Deferred.List.map internal_cmd_ids ~f:(fun id -> - let open Deferred.Let_syntax in - match%map -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> - Sql.Internal_command.run db - ~start_slot:input.start_slot_since_genesis -@@ -892,7 +892,7 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri - Deferred.List.map user_cmd_ids ~f:(fun id -> - let open Deferred.Let_syntax in - match%map -- Caqti_async.Pool.use (fun db -> Sql.User_command.run db id) pool -+ Mina_caqti.Pool.use (fun db -> Sql.User_command.run db id) pool - with - | Ok [] -> - failwithf "Expected at least one user command with id %d" id () -@@ -923,7 +923,7 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri - Deferred.List.map zkapp_cmd_ids ~f:(fun id -> - let open Deferred.Let_syntax in - match%map -- Caqti_async.Pool.use (fun db -> Sql.Zkapp_command.run db id) pool -+ Mina_caqti.Pool.use (fun db -> Sql.Zkapp_command.run db id) pool - with - | Ok [] -> - failwithf "Expected at least one zkApp command with id %d" id () -diff --git a/src/app/replayer/sql.ml b/src/app/replayer/sql.ml -index 40caaf5af1..160e213187 100644 ---- a/src/app/replayer/sql.ml -+++ b/src/app/replayer/sql.ml -@@ -18,8 +18,8 @@ module Block_info = struct - - (* find all blocks, working back from block with given state hash *) - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - typ - {sql| WITH RECURSIVE chain AS ( - SELECT id,parent_id,global_slot_since_genesis,state_hash,ledger_hash, snarked_ledger_hash_id FROM blocks b WHERE b.state_hash = $1 -@@ -34,7 +34,7 @@ module Block_info = struct - - SELECT id,global_slot_since_genesis,state_hash,ledger_hash, snarked_ledger_hash_id FROM chain c WHERE c.global_slot_since_genesis >= $2 |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~state_hash ~start_slot = -+ let run (module Conn : Mina_caqti.CONNECTION) ~state_hash ~start_slot = - Conn.collect_list query (state_hash, start_slot) - end - -@@ -71,57 +71,57 @@ let find_command_ids_query s = - - module Block = struct - let state_hash_query = -- Caqti_request.find Caqti_type.int Caqti_type.string -+ Mina_caqti.find_req Caqti_type.int Caqti_type.string - {sql| SELECT state_hash FROM blocks - WHERE id = ? - |sql} - -- let get_state_hash (module Conn : Caqti_async.CONNECTION) id = -+ let get_state_hash (module Conn : Mina_caqti.CONNECTION) id = - Conn.find state_hash_query id - - let parent_id_query = -- Caqti_request.find Caqti_type.int Caqti_type.int -+ Mina_caqti.find_req Caqti_type.int Caqti_type.int - {sql| SELECT parent_id FROM blocks - WHERE id = ? - |sql} - -- let get_parent_id (module Conn : Caqti_async.CONNECTION) id = -+ let get_parent_id (module Conn : Mina_caqti.CONNECTION) id = - Conn.find parent_id_query id - - let unparented_query = -- Caqti_request.collect Caqti_type.unit Caqti_type.int -+ Mina_caqti.collect_req Caqti_type.unit Caqti_type.int - {sql| SELECT id FROM blocks - WHERE parent_id IS NULL - |sql} - -- let get_unparented (module Conn : Caqti_async.CONNECTION) () = -+ let get_unparented (module Conn : Mina_caqti.CONNECTION) () = - Conn.collect_list unparented_query () - - let get_height_query = -- Caqti_request.find Caqti_type.int Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.int Caqti_type.int64 - {sql| SELECT height FROM blocks WHERE id = $1 |sql} - -- let get_height (module Conn : Caqti_async.CONNECTION) ~block_id = -+ let get_height (module Conn : Mina_caqti.CONNECTION) ~block_id = - Conn.find get_height_query block_id - - let max_slot_query = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT MAX(global_slot_since_genesis) FROM blocks |sql} - -- let get_max_slot (module Conn : Caqti_async.CONNECTION) () = -+ let get_max_slot (module Conn : Mina_caqti.CONNECTION) () = - Conn.find max_slot_query () - - let max_canonical_slot_query = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT MAX(global_slot_since_genesis) FROM blocks - WHERE chain_status = 'canonical' - |sql} - -- let get_max_canonical_slot (module Conn : Caqti_async.CONNECTION) () = -+ let get_max_canonical_slot (module Conn : Mina_caqti.CONNECTION) () = - Conn.find max_canonical_slot_query () - - let next_slot_query = -- Caqti_request.find_opt Caqti_type.int64 Caqti_type.int64 -+ Mina_caqti.find_opt_req Caqti_type.int64 Caqti_type.int64 - {sql| SELECT global_slot_since_genesis FROM blocks - WHERE global_slot_since_genesis >= $1 - AND chain_status <> 'orphaned' -@@ -129,19 +129,19 @@ module Block = struct - LIMIT 1 - |sql} - -- let get_next_slot (module Conn : Caqti_async.CONNECTION) slot = -+ let get_next_slot (module Conn : Mina_caqti.CONNECTION) slot = - Conn.find_opt next_slot_query slot - - let state_hashes_by_slot_query = -- Caqti_request.collect Caqti_type.int64 Caqti_type.string -+ Mina_caqti.collect_req Caqti_type.int64 Caqti_type.string - {sql| SELECT state_hash FROM blocks WHERE global_slot_since_genesis = $1 |sql} - -- let get_state_hashes_by_slot (module Conn : Caqti_async.CONNECTION) slot = -+ let get_state_hashes_by_slot (module Conn : Mina_caqti.CONNECTION) slot = - Conn.collect_list state_hashes_by_slot_query slot - - (* find all blocks, working back from block with given state hash *) - let chain_query = -- Caqti_request.collect Caqti_type.string Caqti_type.string -+ Mina_caqti.collect_req Caqti_type.string Caqti_type.string - {sql| WITH RECURSIVE chain AS ( - - SELECT id,parent_id FROM blocks b WHERE b.state_hash = ? -@@ -159,7 +159,7 @@ module Block = struct - - |sql} - -- let get_chain (module Conn : Caqti_async.CONNECTION) state_hash = -+ let get_chain (module Conn : Mina_caqti.CONNECTION) state_hash = - Conn.collect_list chain_query state_hash - - (* either the bonafide genesis block, or the most recent "linking" block -@@ -171,7 +171,7 @@ module Block = struct - let pk, _ = Mina_state.Consensus_state_hooks.genesis_winner in - Signature_lib.Public_key.Compressed.to_base58_check pk - in -- Caqti_request.find Caqti_type.int64 Caqti_type.string -+ Mina_caqti.find_req Caqti_type.int64 Caqti_type.string - (sprintf - {sql| SELECT snarked_ledger_hashes.value - FROM blocks -@@ -186,18 +186,18 @@ module Block = struct - |sql} - genesis_winner ) - -- let genesis_snarked_ledger (module Conn : Caqti_async.CONNECTION) start_slot = -+ let genesis_snarked_ledger (module Conn : Mina_caqti.CONNECTION) start_slot = - Conn.find genesis_snarked_ledger_query start_slot - end - - module User_command_ids = struct - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - Caqti_type.int - (find_command_ids_query "user") - -- let run (module Conn : Caqti_async.CONNECTION) ~state_hash ~start_slot = -+ let run (module Conn : Mina_caqti.CONNECTION) ~state_hash ~start_slot = - Conn.collect_list query (state_hash, start_slot) - end - -@@ -240,7 +240,7 @@ module User_command = struct - ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT command_type,fee_payer_id, source_id,receiver_id,fee,amount,valid_until,memo,nonce, - blocks.id,blocks.height,blocks.global_slot_since_genesis, - sequence_no,status -@@ -259,18 +259,18 @@ module User_command = struct - - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) user_cmd_id = -+ let run (module Conn : Mina_caqti.CONNECTION) user_cmd_id = - Conn.collect_list query user_cmd_id - end - - module Zkapp_command_ids = struct - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - Caqti_type.int - (find_command_ids_query "zkapp") - -- let run (module Conn : Caqti_async.CONNECTION) ~state_hash ~start_slot = -+ let run (module Conn : Mina_caqti.CONNECTION) ~state_hash ~start_slot = - Conn.collect_list query (state_hash, start_slot) - end - -@@ -292,7 +292,7 @@ module Zkapp_command = struct - [ int; Mina_caqti.array_int_typ; string; int; int64; int; string ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT zkapp_fee_payer_body_id,zkapp_account_updates_ids,memo, - blocks.id,blocks.global_slot_since_genesis, - sequence_no,hash -@@ -311,18 +311,18 @@ module Zkapp_command = struct - - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) zkapp_cmd_id = -+ let run (module Conn : Mina_caqti.CONNECTION) zkapp_cmd_id = - Conn.collect_list query zkapp_cmd_id - end - - module Internal_command_ids = struct - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - Caqti_type.int - (find_command_ids_query "internal") - -- let run (module Conn : Caqti_async.CONNECTION) ~state_hash ~start_slot = -+ let run (module Conn : Mina_caqti.CONNECTION) ~state_hash ~start_slot = - Conn.collect_list query (state_hash, start_slot) - end - -@@ -347,8 +347,8 @@ module Internal_command = struct - the call to Staged_ledger.apply in Block_producer - *) - let query = -- Caqti_request.collect -- Caqti_type.(tup2 int64 int) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 int64 int) - typ - {sql| SELECT command_type,receiver_id,fee, - b.id,b.height,b.global_slot_since_genesis, -@@ -371,29 +371,29 @@ module Internal_command = struct - WHERE b.global_slot_since_genesis >= $1 - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~start_slot ~internal_cmd_id = -+ let run (module Conn : Mina_caqti.CONNECTION) ~start_slot ~internal_cmd_id = - Conn.collect_list query (start_slot, internal_cmd_id) - end - - module Public_key = struct - let query = -- Caqti_request.find_opt Caqti_type.int Caqti_type.string -+ Mina_caqti.find_opt_req Caqti_type.int Caqti_type.string - {sql| SELECT value FROM public_keys - WHERE id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) pk_id = -+ let run (module Conn : Mina_caqti.CONNECTION) pk_id = - Conn.find_opt query pk_id - end - - module Snarked_ledger_hashes = struct - let query = -- Caqti_request.find Caqti_type.int Caqti_type.string -+ Mina_caqti.find_req Caqti_type.int Caqti_type.string - {sql| SELECT value FROM snarked_ledger_hashes - WHERE id = ? - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) id = Conn.find query id -+ let run (module Conn : Mina_caqti.CONNECTION) id = Conn.find query id - end - - module Epoch_data = struct -@@ -404,11 +404,11 @@ module Epoch_data = struct - let decode (epoch_ledger_hash, epoch_data_seed) = - Ok { epoch_ledger_hash; epoch_data_seed } - in -- let rep = Caqti_type.(tup2 string string) in -+ let rep = Caqti_type.(t2 string string) in - Caqti_type.custom ~encode ~decode rep - - let query_epoch_data = -- Caqti_request.find Caqti_type.int epoch_data_typ -+ Mina_caqti.find_req Caqti_type.int epoch_data_typ - {sql| SELECT slh.value, ed.seed FROM snarked_ledger_hashes AS slh - - INNER JOIN -@@ -421,36 +421,36 @@ module Epoch_data = struct - - |sql} - -- let get_epoch_data (module Conn : Caqti_async.CONNECTION) epoch_ledger_id = -+ let get_epoch_data (module Conn : Mina_caqti.CONNECTION) epoch_ledger_id = - Conn.find query_epoch_data epoch_ledger_id - - let query_staking_epoch_data_id = -- Caqti_request.find Caqti_type.string Caqti_type.int -+ Mina_caqti.find_req Caqti_type.string Caqti_type.int - {sql| SELECT staking_epoch_data_id FROM blocks - - WHERE state_hash = ? - - |sql} - -- let get_staking_epoch_data_id (module Conn : Caqti_async.CONNECTION) -- state_hash = -+ let get_staking_epoch_data_id (module Conn : Mina_caqti.CONNECTION) state_hash -+ = - Conn.find query_staking_epoch_data_id state_hash - - let query_next_epoch_data_id = -- Caqti_request.find Caqti_type.string Caqti_type.int -+ Mina_caqti.find_req Caqti_type.string Caqti_type.int - {sql| SELECT next_epoch_data_id FROM blocks - - WHERE state_hash = ? - |sql} - -- let get_next_epoch_data_id (module Conn : Caqti_async.CONNECTION) state_hash = -+ let get_next_epoch_data_id (module Conn : Mina_caqti.CONNECTION) state_hash = - Conn.find query_next_epoch_data_id state_hash - end - - module Parent_block = struct - (* fork block is parent of block with the given state hash *) - let query_parent_state_hash = -- Caqti_request.find Caqti_type.string Caqti_type.string -+ Mina_caqti.find_req Caqti_type.string Caqti_type.string - {sql| SELECT parent.state_hash FROM blocks AS parent - - INNER JOIN -@@ -460,7 +460,7 @@ module Parent_block = struct - ON epoch_ledgers_block.parent_id = parent.id - |sql} - -- let get_parent_state_hash (module Conn : Caqti_async.CONNECTION) -+ let get_parent_state_hash (module Conn : Mina_caqti.CONNECTION) - epoch_ledgers_state_hash = - Conn.find query_parent_state_hash epoch_ledgers_state_hash - end -diff --git a/src/app/rosetta/lib/account.ml b/src/app/rosetta/lib/account.ml -index 0b2265c206..7a999e2bd6 100644 ---- a/src/app/rosetta/lib/account.ml -+++ b/src/app/rosetta/lib/account.ml -@@ -36,9 +36,9 @@ module Sql = struct - .transaction_capacity_log_2 - - let query_pending = -- Caqti_request.find_opt -- Caqti_type.(tup3 string int64 string) -- Caqti_type.(tup2 (tup4 int64 int64 int64 int64) int) -+ Mina_caqti.find_opt_req -+ Caqti_type.(t3 string int64 string) -+ Caqti_type.(t2 (t4 int64 int64 int64 int64) int) - {sql| - WITH RECURSIVE pending_chain AS ( - -@@ -87,9 +87,9 @@ module Sql = struct - |sql} - - let query_canonical = -- Caqti_request.find_opt -- Caqti_type.(tup3 string int64 string) -- Caqti_type.(tup2 (tup4 int64 int64 int64 int64) int) -+ Mina_caqti.find_opt_req -+ Caqti_type.(t3 string int64 string) -+ Caqti_type.(t2 (t4 int64 int64 int64 int64) int) - {sql| - SELECT b.height,b.global_slot_since_genesis AS block_global_slot_since_genesis,balance,nonce,timing_id - -@@ -108,7 +108,7 @@ module Sql = struct - LIMIT 1 - |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~requested_block_height -+ let run (module Conn : Mina_caqti.CONNECTION) ~requested_block_height - ~address ~token_id = - let open Deferred.Result.Let_syntax in - let%bind has_canonical_height = -@@ -143,7 +143,7 @@ module Sql = struct - ~cliff_time ~cliff_amount ~vesting_period ~vesting_increment - ~initial_minimum_balance - -- let find_current_balance (module Conn : Caqti_async.CONNECTION) -+ let find_current_balance (module Conn : Mina_caqti.CONNECTION) - ~requested_block_global_slot_since_genesis ~last_relevant_command_info - ?timing_id () = - let open Deferred.Result.Let_syntax in -@@ -198,7 +198,7 @@ module Sql = struct - let balance_info : Balance_info.t = { liquid_balance; total_balance } in - Deferred.Result.return (balance_info, nonce) - -- let run (module Conn : Caqti_async.CONNECTION) ~block_query ~address ~token_id -+ let run (module Conn : Mina_caqti.CONNECTION) ~block_query ~address ~token_id - = - let open Deferred.Result.Let_syntax in - (* First find the block referenced by the block identifier. Then -@@ -298,7 +298,7 @@ module Balance = struct - ; db_block_identifier_and_balance_info = - (fun ~block_query ~address ~token_id -> - with_db (fun ~db -> -- let (module Conn : Caqti_async.CONNECTION) = db in -+ let (module Conn : Mina_caqti.CONNECTION) = db in - Sql.run (module Conn) ~block_query ~address ~token_id - |> Errors.Lift.wrap ) - |> Deferred.Result.map_error ~f:(function `App e -> e) ) -diff --git a/src/app/rosetta/lib/block.ml b/src/app/rosetta/lib/block.ml -index b6cf0ea4d0..b56bd1eb40 100644 ---- a/src/app/rosetta/lib/block.ml -+++ b/src/app/rosetta/lib/block.ml -@@ -104,7 +104,7 @@ module Sql = struct - String.concat ~sep:"," fields - - let query_count_canonical_at_height = -- Caqti_request.find Caqti_type.int64 Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.int64 Caqti_type.int64 - {sql| SELECT COUNT(*) FROM blocks - WHERE height = ? - AND chain_status = 'canonical' -@@ -112,7 +112,7 @@ module Sql = struct - - let query_height_canonical = - let c_fields = block_fields ~prefix:"c." () in -- Caqti_request.find_opt Caqti_type.int64 typ -+ Mina_caqti.find_opt_req Caqti_type.int64 typ - (* The archive database will only reconcile the canonical columns for - * blocks older than k + epsilon - *) -@@ -135,7 +135,7 @@ module Sql = struct - let fields = block_fields () in - let b_fields = block_fields ~prefix:"b." () in - let c_fields = block_fields ~prefix:"c." () in -- Caqti_request.find_opt Caqti_type.int64 typ -+ Mina_caqti.find_opt_req Caqti_type.int64 typ - (* According to the clarification of the Rosetta spec here - * https://community.rosetta-api.org/t/querying-block-by-just-its-index/84/3 , - * it is important to select only the block on the canonical chain for a -@@ -180,7 +180,7 @@ module Sql = struct - - let query_hash = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt Caqti_type.string typ -+ Mina_caqti.find_opt_req Caqti_type.string typ - [%string - {| - SELECT b.id, -@@ -197,8 +197,8 @@ module Sql = struct - - let query_both = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt -- Caqti_type.(tup2 string int64) -+ Mina_caqti.find_opt_req -+ Caqti_type.(t2 string int64) - typ - [%string - {| -@@ -217,7 +217,7 @@ module Sql = struct - - let query_by_id = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt Caqti_type.int typ -+ Mina_caqti.find_opt_req Caqti_type.int typ - [%string - {| - SELECT b.id, -@@ -234,7 +234,7 @@ module Sql = struct - - let query_best = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt Caqti_type.unit typ -+ Mina_caqti.find_opt_req Caqti_type.unit typ - [%string - {| - SELECT b.id, -@@ -251,10 +251,10 @@ module Sql = struct - LIMIT 1 - |}] - -- let run_by_id (module Conn : Caqti_async.CONNECTION) id = -+ let run_by_id (module Conn : Mina_caqti.CONNECTION) id = - Conn.find_opt query_by_id id - -- let run_has_canonical_height (module Conn : Caqti_async.CONNECTION) ~height -+ let run_has_canonical_height (module Conn : Mina_caqti.CONNECTION) ~height - = - let open Deferred.Result.Let_syntax in - let%map num_canonical_at_height = -@@ -262,7 +262,7 @@ module Sql = struct - in - Int64.( > ) num_canonical_at_height Int64.zero - -- let run (module Conn : Caqti_async.CONNECTION) = function -+ let run (module Conn : Mina_caqti.CONNECTION) = function - | Some (`This (`Height h)) -> - let open Deferred.Result.Let_syntax in - let%bind has_canonical_height = -@@ -272,7 +272,7 @@ module Sql = struct - else - let%bind max_height = - Conn.find -- (Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ (Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT MAX(height) FROM blocks |sql} ) - () - in -@@ -323,8 +323,7 @@ module Sql = struct - - let typ = - Caqti_type.( -- tup3 int Archive_lib.Processor.User_command.Signed_command.typ -- Extras.typ) -+ t3 int Archive_lib.Processor.User_command.Signed_command.typ Extras.typ) - - let fields = - String.concat ~sep:"," -@@ -333,8 +332,8 @@ module Sql = struct - Archive_lib.Processor.User_command.Signed_command.Fields.names - - let query = -- Caqti_request.collect -- Caqti_type.(tup2 int string) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 int string) - typ - [%string - {| -@@ -385,7 +384,7 @@ module Sql = struct - AND (t.value = ? OR t.id IS NULL) - |}] - -- let run (module Conn : Caqti_async.CONNECTION) id = -+ let run (module Conn : Mina_caqti.CONNECTION) id = - Conn.collect_list query (id, Mina_base.Token_id.(to_string default)) - end - -@@ -539,9 +538,9 @@ module Sql = struct - ON ic_coinbase_receiver.receiver_id = coinbase_receiver_pk.id - |sql}] - -- let run (module Conn : Caqti_async.CONNECTION) id = -+ let run (module Conn : Mina_caqti.CONNECTION) id = - Conn.collect_list -- (Caqti_request.collect Caqti_type.(tup2 int string) typ query) -+ (Mina_caqti.collect_req Caqti_type.(t2 int string) typ query) - (id, Mina_base.Token_id.(to_string default)) - - let to_info t = -@@ -682,9 +681,9 @@ module Sql = struct - |}] - - let query = -- Caqti_request.collect Caqti_type.(tup2 int string) typ query_string -+ Mina_caqti.collect_req Caqti_type.(t2 int string) typ query_string - -- let run (module Conn : Caqti_async.CONNECTION) id = -+ let run (module Conn : Mina_caqti.CONNECTION) id = - Conn.collect_list query (id, Mina_base.Token_id.(to_string default)) - - module Make_common (M : sig -@@ -797,7 +796,7 @@ module Sql = struct - end) - end - -- let run (module Conn : Caqti_async.CONNECTION) input = -+ let run (module Conn : Mina_caqti.CONNECTION) input = - let module Result = struct - include Result - -@@ -955,12 +954,12 @@ module Specific = struct - module Mock = T (Result) - - let real : -- logger:Logger.t -> db:(module Caqti_async.CONNECTION) -> 'gql Real.t = -+ logger:Logger.t -> db:(module Mina_caqti.CONNECTION) -> 'gql Real.t = - fun ~logger ~db -> - { logger - ; db_block = - (fun query -> -- let (module Conn : Caqti_async.CONNECTION) = db in -+ let (module Conn : Mina_caqti.CONNECTION) = db in - Sql.run (module Conn) query ) - ; validate_network_choice = Network.Validate_choice.Real.validate - } -diff --git a/src/app/rosetta/lib/construction.ml b/src/app/rosetta/lib/construction.ml -index 90297f8b17..91ff6dc674 100644 ---- a/src/app/rosetta/lib/construction.ml -+++ b/src/app/rosetta/lib/construction.ml -@@ -866,7 +866,7 @@ module Submit = struct - Caqti_type.custom ~encode ~decode (to_rep spec) - - let query = -- Caqti_request.find_opt params_typ Caqti_type.string -+ Mina_caqti.find_opt_req params_typ Caqti_type.string - {sql| SELECT uc.id FROM user_commands uc - INNER JOIN public_keys AS pks ON pks.id = uc.source_id - INNER JOIN public_keys AS pkr ON pkr.id = uc.receiver_id -@@ -876,7 +876,7 @@ module Submit = struct - AND uc.amount = $4 - AND uc.fee = $5 |sql} - -- let run (module Conn : Caqti_async.CONNECTION) ~nonce ~source ~receiver -+ let run (module Conn : Mina_caqti.CONNECTION) ~nonce ~source ~receiver - ~amount ~fee = - let open Unsigned_extended in - Conn.find_opt query -@@ -924,7 +924,7 @@ module Submit = struct - module Mock = T (Result) - - let real : -- db:(module Caqti_async.CONNECTION) -+ db:(module Mina_caqti.CONNECTION) - -> graphql_uri:Uri.t - -> ( 'gql_payment - , 'gql_delegation -diff --git a/src/app/rosetta/lib/network.ml b/src/app/rosetta/lib/network.ml -index 420d3569ac..157fc50002 100644 ---- a/src/app/rosetta/lib/network.ml -+++ b/src/app/rosetta/lib/network.ml -@@ -75,8 +75,8 @@ end - - module Sql = struct - let oldest_block_query = -- Caqti_request.find Caqti_type.unit -- Caqti_type.(tup2 int64 string) -+ Mina_caqti.find_req Caqti_type.unit -+ Caqti_type.(t2 int64 string) - "SELECT height, state_hash FROM blocks ORDER BY timestamp ASC, \ - state_hash ASC LIMIT 1" - -@@ -88,8 +88,8 @@ module Sql = struct - 0L - - let latest_block_query = -- Caqti_request.find Caqti_type.unit -- Caqti_type.(tup3 int64 string int64) -+ Mina_caqti.find_req Caqti_type.unit -+ Caqti_type.(t3 int64 string int64) - (sprintf - {sql| SELECT height, state_hash, timestamp FROM blocks b - WHERE height = (select MAX(height) - %Ld FROM blocks) -@@ -218,9 +218,9 @@ module Status = struct - let oldest_block_ref = ref None - - let real : -- db:(module Caqti_async.CONNECTION) -> graphql_uri:Uri.t -> 'gql Real.t = -+ db:(module Mina_caqti.CONNECTION) -> graphql_uri:Uri.t -> 'gql Real.t = - fun ~db ~graphql_uri -> -- let (module Db : Caqti_async.CONNECTION) = db in -+ let (module Db : Mina_caqti.CONNECTION) = db in - { gql = Get_status_t.query ~graphql_uri - ; db_oldest_block = - (fun () -> -diff --git a/src/app/rosetta/lib/pg_data.ml b/src/app/rosetta/lib/pg_data.ml -index 15065b1fa3..1dda81f8e9 100644 ---- a/src/app/rosetta/lib/pg_data.ml -+++ b/src/app/rosetta/lib/pg_data.ml -@@ -1,19 +1,19 @@ - (* pg_data.ml -- Postgres data *) - - let query_connection_count = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT count(*) FROM pg_stat_activity - WHERE state = 'active' - |sql} - --let run_connection_count (module Conn : Caqti_async.CONNECTION) = -+let run_connection_count (module Conn : Mina_caqti.CONNECTION) = - Conn.find query_connection_count - - let query_lock_count = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT count(*) FROM pg_locks - WHERE mode = 'SIReadLock' - |sql} - --let run_lock_count (module Conn : Caqti_async.CONNECTION) = -+let run_lock_count (module Conn : Mina_caqti.CONNECTION) = - Conn.find query_lock_count -diff --git a/src/app/rosetta/lib/rosetta.ml b/src/app/rosetta/lib/rosetta.ml -index 135235db65..459befd8ec 100644 ---- a/src/app/rosetta/lib/rosetta.ml -+++ b/src/app/rosetta/lib/rosetta.ml -@@ -4,7 +4,7 @@ open Rosetta_lib - - let router ~graphql_uri - ~(pool : -- ( (Caqti_async.connection, [> Caqti_error.connect ]) Caqti_async.Pool.t -+ ( (Caqti_async.connection, [> Caqti_error.connect ]) Mina_caqti.Pool.t - , [ `App of Errors.t ] ) - Deferred.Result.t - lazy_t ) ~logger route body = -@@ -18,7 +18,7 @@ let router ~graphql_uri - in - let with_db f = - let%bind pool = Lazy.force pool in -- Caqti_async.Pool.use (fun db -> f ~db) pool -+ Mina_caqti.Pool.use (fun db -> f ~db) pool - |> Deferred.Result.map_error ~f:(function - | `App e -> - `App e -@@ -65,7 +65,7 @@ let pg_log_data ~logger ~pool : unit Deferred.t = - match%bind Lazy.force pool with - | Ok pool -> - let get_logs () : (unit, _) Deferred.Result.t = -- Caqti_async.Pool.use -+ Mina_caqti.Pool.use - (fun db -> - let open Deferred.Result.Let_syntax in - let%bind num_conns = Pg_data.run_connection_count db () in -@@ -190,7 +190,7 @@ let command = - "MINA_ROSETTA_MAX_DB_POOL_SIZE not set or invalid. Please set \ - this to a number (try 64 or 128)" - in -- match Caqti_async.connect_pool ~max_size:max_pool_size archive_uri with -+ match Mina_caqti.connect_pool ~max_size:max_pool_size archive_uri with - | Error e -> - [%log error] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/rosetta/lib/search.ml b/src/app/rosetta/lib/search.ml -index a4d16f4921..97f6eea093 100644 ---- a/src/app/rosetta/lib/search.ml -+++ b/src/app/rosetta/lib/search.ml -@@ -418,8 +418,8 @@ module Sql = struct - in - let offset = offset_sql offset in - let limit = limit_sql limit in -- Caqti_request.collect Params.typ -- Caqti_type.(tup2 int64 typ) -+ Mina_caqti.collect_req Params.typ -+ Caqti_type.(t2 int64 typ) - [%string - {sql| - WITH filtered_ids AS ( -@@ -523,7 +523,7 @@ module Sql = struct - ORDER BY u.id, CASE WHEN b.chain_status = 'canonical' THEN 1 WHEN b.chain_status = 'orphaned' THEN 2 END - |sql}] - -- let run (module Conn : Caqti_async.CONNECTION) ~offset ~limit input = -+ let run (module Conn : Mina_caqti.CONNECTION) ~offset ~limit input = - let open Deferred.Result.Let_syntax in - let params = Params.of_query input in - match%map -@@ -775,7 +775,7 @@ module Sql = struct - ORDER BY i.id, i.hash, i.command_type, bic.block_id, bic.sequence_no, bic.secondary_sequence_no - |sql}] - -- let run (module Conn : Caqti_async.CONNECTION) ~offset ~limit input = -+ let run (module Conn : Mina_caqti.CONNECTION) ~offset ~limit input = - let open Deferred.Result.Let_syntax in - let params = Params.of_query input in - let query = -@@ -783,7 +783,7 @@ module Sql = struct - in - match%map - Conn.collect_list -- (Caqti_request.collect Params.typ Caqti_type.(tup2 int64 typ) query) -+ (Mina_caqti.collect_req Params.typ Caqti_type.(t2 int64 typ) query) - params - with - | [] -> -@@ -939,10 +939,10 @@ module Sql = struct - ~address_fields:[ "pk_fee_payer.value"; "pk_update_body.value" ] - ~op_type_filters operator - in -- Caqti_request.collect Params.typ Caqti_type.(tup2 int64 typ) -+ Mina_caqti.collect_req Params.typ Caqti_type.(t2 int64 typ) - @@ query_string ~offset ~limit ~filters - -- let run (module Conn : Caqti_async.CONNECTION) ~offset ~limit input = -+ let run (module Conn : Mina_caqti.CONNECTION) ~offset ~limit input = - let open Deferred.Result.Let_syntax in - let params = Params.of_query input in - match%map -@@ -981,7 +981,7 @@ module Sql = struct - end) - end - -- let run (module Conn : Caqti_async.CONNECTION) query = -+ let run (module Conn : Mina_caqti.CONNECTION) query = - let module Result = struct - include Result - -@@ -1062,7 +1062,7 @@ module Specific = struct - (* But for tests, we want things to go fast *) - module Mock = T (Result) - -- let real : db:(module Caqti_async.CONNECTION) -> 'gql Real.t = -+ let real : db:(module Mina_caqti.CONNECTION) -> 'gql Real.t = - fun ~db -> - { db_transactions = Sql.run db - ; validate_network_choice = Network.Validate_choice.Real.validate -diff --git a/src/app/swap_bad_balances/sql.ml b/src/app/swap_bad_balances/sql.ml -index 82ec136a10..bd85672b7e 100644 ---- a/src/app/swap_bad_balances/sql.ml -+++ b/src/app/swap_bad_balances/sql.ml -@@ -5,8 +5,8 @@ open Async - module Receiver_balances = struct - (* find receiver balances for combined fee transfer *) - let query_ids_from_fee_transfer = -- Caqti_request.collect -- Caqti_type.(tup2 string int) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int) - Caqti_type.(int) - {sql| SELECT bic.receiver_balance - FROM blocks_internal_commands bic -@@ -15,18 +15,17 @@ module Receiver_balances = struct - WHERE b.state_hash = $1 AND bic.sequence_no = $2 - |sql} - -- let run_ids_from_fee_transfer (module Conn : Caqti_async.CONNECTION) -- state_hash seq_no = -+ let run_ids_from_fee_transfer (module Conn : Mina_caqti.CONNECTION) state_hash -+ seq_no = - Conn.collect_list query_ids_from_fee_transfer (state_hash, seq_no) - -- let add_if_doesn't_exist (module Conn : Caqti_async.CONNECTION) (pk, balance) -- = -+ let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) (pk, balance) = - let open Deferred.Result.Let_syntax in - (* if duplicates, any is acceptable *) - match%bind - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 int int64) -+ (Mina_caqti.find_opt_req -+ Caqti_type.(t2 int int64) - Caqti_type.int - {sql| SELECT id - FROM balances -@@ -40,18 +39,18 @@ module Receiver_balances = struct - return id - | None -> - Conn.find -- (Caqti_request.find -- Caqti_type.(tup2 int int64) -+ (Mina_caqti.find_req -+ Caqti_type.(t2 int int64) - Caqti_type.int - "INSERT INTO balances (public_key_id,balance) VALUES ($1,$2) \ - RETURNING id" ) - (pk, balance) - -- let load (module Conn : Caqti_async.CONNECTION) id = -+ let load (module Conn : Mina_caqti.CONNECTION) id = - Conn.find -- (Caqti_request.find -+ (Mina_caqti.find_req - Caqti_type.(int) -- Caqti_type.(tup2 int int64) -+ Caqti_type.(t2 int int64) - {sql| SELECT public_key_id,balance - FROM balances - WHERE id = $1 -@@ -59,8 +58,8 @@ module Receiver_balances = struct - id - - let query_swap_in_new_balance = -- Caqti_request.exec -- Caqti_type.(tup4 string int int int) -+ Mina_caqti.exec_req -+ Caqti_type.(t4 string int int int) - {sql| UPDATE blocks_internal_commands bic SET receiver_balance = $4 - FROM blocks b - WHERE b.id = bic.block_id -@@ -69,7 +68,7 @@ module Receiver_balances = struct - AND bic.receiver_balance = $3 - |sql} - -- let swap_in_new_balance (module Conn : Caqti_async.CONNECTION) state_hash -+ let swap_in_new_balance (module Conn : Mina_caqti.CONNECTION) state_hash - seq_no old_balance_id new_balance_id = - Conn.exec query_swap_in_new_balance - (state_hash, seq_no, old_balance_id, new_balance_id) -diff --git a/src/app/swap_bad_balances/swap_bad_balances.ml b/src/app/swap_bad_balances/swap_bad_balances.ml -index a01bb86938..5ff0d88e9f 100644 ---- a/src/app/swap_bad_balances/swap_bad_balances.ml -+++ b/src/app/swap_bad_balances/swap_bad_balances.ml -@@ -4,7 +4,7 @@ open Core_kernel - open Async - - let query_db pool ~f ~item = -- match%bind Caqti_async.Pool.use f pool with -+ match%bind Mina_caqti.Pool.use f pool with - | Ok v -> - return v - | Error msg -> -@@ -14,7 +14,7 @@ let query_db pool ~f ~item = - let main ~archive_uri ~state_hash ~sequence_no () = - let archive_uri = Uri.of_string archive_uri in - let logger = Logger.create () in -- match Caqti_async.connect_pool archive_uri with -+ match Mina_caqti.connect_pool archive_uri with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml -index 20b64dcbf1..667689fc29 100644 ---- a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml -+++ b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml -@@ -75,10 +75,10 @@ let assert_filehash_equal ~file ~hash ~logger = - else - let%map () = Unix.rename ~src:file ~dst:(file ^ ".incorrect-hash") in - [%log error] -- "Verification failure: downloaded $file and expected SHA3-256 = $hash \ -- but it had $computed_hash" -+ "Verification failure: downloaded $file and expected SHA3-256 = \ -+ $expected_hash but it had $computed_hash" - ~metadata: -- [ ("path", `String file) -+ [ ("file", `String file) - ; ("expected_hash", `String hash) - ; ("computed_hash", `String computed_hash) - ] ; -diff --git a/src/lib/mina_caqti/mina_caqti.ml b/src/lib/mina_caqti/mina_caqti.ml -index b1e49d9bcb..d70a04fcba 100644 ---- a/src/lib/mina_caqti/mina_caqti.ml -+++ b/src/lib/mina_caqti/mina_caqti.ml -@@ -2,17 +2,67 @@ - - open Async - open Core_kernel --open Caqti_async - open Mina_base - --type _ Caqti_type.field += -- | Array_nullable_int : int option array Caqti_type.field -+let find_req t u s = Caqti_request.Infix.(t ->! u) s - --type _ Caqti_type.field += -- | Array_nullable_int64 : int64 option array Caqti_type.field -+let find_opt_req t u s = Caqti_request.Infix.(t ->? u) s - --type _ Caqti_type.field += -- | Array_nullable_string : string option array Caqti_type.field -+let collect_req t u s = Caqti_request.Infix.(t ->* u) s -+ -+let exec_req t s = Caqti_request.Infix.(t ->. Caqti_type.unit) s -+ -+module type CONNECTION = sig -+ include Caqti_async.CONNECTION -+ -+ (** Code expects any queries to differing sources to never interfere. *) -+ val source : Uri.t -+end -+ -+module Wrap -+ (Conn : Caqti_async.CONNECTION) (Arg : sig -+ val source : Uri.t -+ end) : CONNECTION = struct -+ include Conn -+ include Arg -+end -+ -+let wrap_conn (module Conn : Caqti_async.CONNECTION) ~source = -+ let module Conn = -+ Wrap -+ (Conn) -+ (struct -+ let source = source -+ end) -+ in -+ (module Conn : CONNECTION) -+ -+module Pool = struct -+ type ('a, 'e) t = { source : Uri.t; pool : ('a, 'e) Caqti_async.Pool.t } -+ -+ let wrap ~source pool = { source; pool } -+ -+ let use (f : (module CONNECTION) -> 'a) pool = -+ Caqti_async.Pool.use -+ (fun (module Conn : Caqti_async.CONNECTION) -> -+ f (wrap_conn (module Conn) ~source:pool.source) ) -+ pool.pool -+end -+ -+let connect_pool ?max_size uri = -+ let size = max_size in -+ let%map.Result pool = -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ?max_size:size ())) -+ uri -+ in -+ Pool.wrap ~source:uri pool -+ -+let connect uri = -+ let%map.Deferred.Result conn = Caqti_async.connect uri in -+ wrap_conn ~source:uri conn - - module Type_spec = struct - type (_, _) t = -@@ -25,7 +75,7 @@ module Type_spec = struct - | [] -> - (Caqti_type.unit : tuple Caqti_type.t) - | rep :: spec -> -- Caqti_type.tup2 rep (to_rep spec) -+ Caqti_type.t2 rep (to_rep spec) - - let rec hlist_to_tuple : - 'hlist 'tuple. -@@ -93,59 +143,14 @@ let make_coding (type a) ~(elem_to_string : a -> string) - in - (encode, decode) - --(* register coding for nullable int arrays *) --let () = -- let open Caqti_type.Field in -- let rep = Caqti_type.String in -+(** this type may require type annotations in queries, eg. -+ `SELECT id FROM zkapp_states WHERE element_ids = ?::int[]` -+*) -+let array_nullable_int_typ = - let encode, decode = - make_coding ~elem_to_string:Int.to_string ~elem_of_string:Int.of_string - in -- let get_coding : type a. _ -> a t -> a coding = -- fun _ -> function -- | Array_nullable_int -> -- Coding { rep; encode; decode } -- | _ -> -- assert false -- in -- define_coding Array_nullable_int { get_coding } -- --(* register coding for nullable int64 arrays *) --let () = -- let open Caqti_type.Field in -- let rep = Caqti_type.String in -- let encode, decode = -- make_coding ~elem_to_string:Int64.to_string ~elem_of_string:Int64.of_string -- in -- let get_coding : type a. _ -> a t -> a coding = -- fun _ -> function -- | Array_nullable_int64 -> -- Coding { rep; encode; decode } -- | _ -> -- assert false -- in -- define_coding Array_nullable_int64 { get_coding } -- --(* register coding for nullable string arrays *) --let () = -- let open Caqti_type.Field in -- let rep = Caqti_type.String in -- let encode, decode = -- make_coding ~elem_to_string:Fn.id ~elem_of_string:Fn.id -- in -- let get_coding : type a. _ -> a t -> a coding = -- fun _ -> function -- | Array_nullable_string -> -- Coding { rep; encode; decode } -- | _ -> -- assert false -- in -- define_coding Array_nullable_string { get_coding } -- --(* this type may require type annotations in queries, eg. -- `SELECT id FROM zkapp_states WHERE element_ids = ?::int[]` --*) --let array_nullable_int_typ : int option array Caqti_type.t = -- Caqti_type.field Array_nullable_int -+ Caqti_type.custom ~encode ~decode Caqti_type.string - - let array_int_typ : int array Caqti_type.t = - let open Result.Let_syntax in -@@ -158,11 +163,14 @@ let array_int_typ : int array Caqti_type.t = - in - Caqti_type.custom array_nullable_int_typ ~encode ~decode - --(* this type may require type annotations in queries, eg. -+(** this type may require type annotations in queries, eg. - `SELECT id FROM zkapp_states WHERE element_ids = ?::bigint[]` - *) --let array_nullable_int64_typ : int64 option array Caqti_type.t = -- Caqti_type.field Array_nullable_int64 -+let array_nullable_int64_typ = -+ let encode, decode = -+ make_coding ~elem_to_string:Int64.to_string ~elem_of_string:Int64.of_string -+ in -+ Caqti_type.custom ~encode ~decode Caqti_type.string - - let array_int64_typ : int64 array Caqti_type.t = - let open Result.Let_syntax in -@@ -175,11 +183,14 @@ let array_int64_typ : int64 array Caqti_type.t = - in - Caqti_type.custom array_nullable_int64_typ ~encode ~decode - --(* this type may require type annotations in queries, e.g. -+(** this type may require type annotations in queries, e.g. - `SELECT id FROM zkapp_states WHERE element_ids = ?::string[]` - *) --let array_nullable_string_typ : string option array Caqti_type.t = -- Caqti_type.field Array_nullable_string -+let array_nullable_string_typ = -+ let encode, decode = -+ make_coding ~elem_to_string:Fn.id ~elem_of_string:Fn.id -+ in -+ Caqti_type.custom ~encode ~decode Caqti_type.string - - let array_string_typ : string array Caqti_type.t = - let open Result.Let_syntax in -@@ -278,12 +289,24 @@ let insert_into_cols ~(returning : string) ~(table_name : string) - (String.concat ~sep:", " cols) - values returning - -+let insert_assuming_new ~(select : string * 'select Caqti_type.t) -+ ~(table_name : string) ?tannot ~(cols : string list * 'cols Caqti_type.t) -+ (module Conn : CONNECTION) (value : 'cols) = -+ Conn.find -+ ( find_req (snd cols) (snd select) -+ @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot -+ ~cols:(fst cols) () ) -+ value -+ -+(* run `select_cols` and return the result, if found -+ if not found, run `insert_into_cols` and return the result -+*) - let select_insert_into_cols ~(select : string * 'select Caqti_type.t) - ~(table_name : string) ?tannot ~(cols : string list * 'cols Caqti_type.t) - (module Conn : CONNECTION) (value : 'cols) = - let open Deferred.Result.Let_syntax in - Conn.find_opt -- ( Caqti_request.find_opt (snd cols) (snd select) -+ ( Caqti_request.Infix.(snd cols ->? snd select) - @@ select_cols ~select:(fst select) ~table_name ?tannot ~cols:(fst cols) () - ) - value -@@ -292,7 +315,7 @@ let select_insert_into_cols ~(select : string * 'select Caqti_type.t) - return id - | None -> - Conn.find -- ( Caqti_request.find (snd cols) (snd select) -+ ( Caqti_request.Infix.(snd cols ->! snd select) - @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot - ~cols:(fst cols) () ) - value -@@ -314,7 +337,11 @@ let insert_multi_into_col ~(table_name : string) - (sep_by_comma ~parenthesis:true values) - (fst col) - in -- let%bind () = Conn.exec (Caqti_request.exec Caqti_type.unit insert) () in -+ let%bind () = -+ Conn.exec -+ (Caqti_request.Infix.(Caqti_type.unit ->. Caqti_type.unit) insert) -+ () -+ in - let search = - sprintf - {sql| SELECT %s, id FROM %s -@@ -322,13 +349,12 @@ let insert_multi_into_col ~(table_name : string) - (fst col) table_name (fst col) (sep_by_comma values) - in - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (snd col) int) -- search ) -+ Caqti_request.Infix.( -+ (Caqti_type.unit ->* Caqti_type.(t2 (snd col) int)) search) - () - - let query ~f pool = -- match%bind Caqti_async.Pool.use f pool with -+ match%bind Pool.use f pool with - | Ok v -> - return v - | Error msg -> -diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml -index a71ec6c5f8..711415c4bc 100644 ---- a/src/lib/runtime_config/runtime_config.ml -+++ b/src/lib/runtime_config/runtime_config.ml -@@ -50,9 +50,13 @@ let result_opt ~f x = - | None -> - Result.return None - -+let truncate_long_string s = -+ if String.length s > 100_000 then "... too long ..." else s -+ - let dump_on_error yojson x = - Result.map_error x ~f:(fun str -> -- str ^ "\n\nCould not parse JSON:\n" ^ Yojson.Safe.pretty_to_string yojson ) -+ str ^ "\n\nCould not parse JSON:\n" ^ truncate_long_string -+ @@ Yojson.Safe.pretty_to_string yojson ) - - let of_yojson_generic ~fields of_yojson json = - dump_on_error json @@ of_yojson diff --git a/buildkite/scripts/caqti-upgrade.patch b/buildkite/scripts/caqti-upgrade.patch deleted file mode 100644 index ba44547344a..00000000000 --- a/buildkite/scripts/caqti-upgrade.patch +++ /dev/null @@ -1,2637 +0,0 @@ -diff --git a/opam.export b/opam.export -index 655b623d35..130eb77140 100644 ---- a/opam.export -+++ b/opam.export -@@ -69,9 +69,9 @@ installed: [ - "camlp4.4.14+1" - "camomile.1.0.2" - "capnp.3.4.0" -- "caqti.1.5.0" -- "caqti-async.1.3.0" -- "caqti-driver-postgresql.1.5.1" -+ "caqti.2.0.1" -+ "caqti-async.2.0.1" -+ "caqti-driver-postgresql.2.0.1" - "charInfo_width.1.1.0" - "check_opam_switch.~dev" - "cmdliner.1.0.3" -@@ -138,6 +138,7 @@ installed: [ - "lmdb.1.0" - "logs.0.7.0" - "lwt.5.4.0" -+ "lwt-dllist.1.0.1" - "lwt_log.1.1.1" - "lwt_react.1.1.2" - "macaddr.5.0.1" -@@ -155,6 +156,7 @@ installed: [ - "mirage-crypto-rng.0.11.0" - "mirage-crypto-rng-async.0.11.0" - "mmap.1.1.0" -+ "mtime.2.0.0" - "num.1.1" - "ocaml.4.14.0" - "ocaml-base-compiler.4.14.0" -diff --git a/src/app/archive/lib/metrics.ml b/src/app/archive/lib/metrics.ml -index c69e1805d1..aac3552028 100644 ---- a/src/app/archive/lib/metrics.ml -+++ b/src/app/archive/lib/metrics.ml -@@ -14,7 +14,7 @@ let default_missing_blocks_width = 2000 - - module Max_block_height = struct - let query = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Caqti_request.Infix.(Caqti_type.unit ->! Caqti_type.int) - "SELECT GREATEST(0, MAX(height)) FROM blocks" - - let update (module Conn : Caqti_async.CONNECTION) metric_server = -@@ -30,7 +30,7 @@ end - module Missing_blocks = struct - (*A block is missing if there is no entry for a specific height. However, if there is an entry then it doesn't necessarily mean that it is part of the main chain. Unparented_blocks will show value > 1 in that case. Look for the last 2000 blocks*) - let query missing_blocks_width = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Caqti_request.Infix.(Caqti_type.unit ->! Caqti_type.int) - (Core_kernel.sprintf - {sql| - SELECT COUNT( * ) -@@ -55,7 +55,7 @@ module Unparented_blocks = struct - (* parent_hashes represent ends of chains leading to an orphan block *) - - let query = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Caqti_request.Infix.(Caqti_type.unit ->! Caqti_type.int) - {sql| - SELECT COUNT( * ) FROM blocks - WHERE parent_id IS NULL -diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml -index bec5544e41..be30438975 100644 ---- a/src/app/archive/lib/processor.ml -+++ b/src/app/archive/lib/processor.ml -@@ -29,6 +29,7 @@ open Mina_block - open Pipe_lib - open Signature_lib - open Pickles_types -+open Mina_caqti - - let applied_str = "applied" - -@@ -42,20 +43,20 @@ module Public_key = struct - let find (module Conn : CONNECTION) (t : Public_key.Compressed.t) = - let public_key = Public_key.Compressed.to_base58_check t in - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (Caqti_request.Infix.(Caqti_type.string ->! Caqti_type.int) - "SELECT id FROM public_keys WHERE value = ?" ) - public_key - - let find_opt (module Conn : CONNECTION) (t : Public_key.Compressed.t) = - let public_key = Public_key.Compressed.to_base58_check t in - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ (Caqti_request.Infix.(Caqti_type.string ->? Caqti_type.int) - "SELECT id FROM public_keys WHERE value = ?" ) - public_key - - let find_by_id (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (Caqti_request.Infix.(Caqti_type.int ->! Caqti_type.string) - "SELECT value FROM public_keys WHERE id = ?" ) - id - -@@ -68,7 +69,7 @@ module Public_key = struct - | None -> - let public_key = Public_key.Compressed.to_base58_check t in - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (Caqti_request.Infix.(Caqti_type.string ->! Caqti_type.int) - "INSERT INTO public_keys (value) VALUES (?) RETURNING id" ) - public_key - end -@@ -107,7 +108,7 @@ module Token = struct - - let find_by_id (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - -@@ -117,15 +118,15 @@ module Token = struct - (Mina_caqti.select_cols ~table_name ~select:"id" ~cols:[ "value" ] ()) ) - (Token_id.to_string token_id) - -- let find (module Conn : CONNECTION) = make_finder Conn.find Caqti_request.find -+ let find (module Conn : CONNECTION) = make_finder Conn.find find_req - - let find_opt (module Conn : CONNECTION) = -- make_finder Conn.find_opt Caqti_request.find_opt -+ make_finder Conn.find_opt find_opt_req - - let find_no_owner_opt (module Conn : CONNECTION) token_id = - let value = Token_id.to_string token_id in - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ (find_opt_req Caqti_type.string Caqti_type.int - {sql| SELECT id - FROM tokens - WHERE value = $1 -@@ -137,8 +138,8 @@ module Token = struct - let set_owner (module Conn : CONNECTION) ~id ~owner_public_key_id - ~owner_token_id = - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - Caqti_type.int - {sql| UPDATE tokens - SET owner_public_key_id = $2, owner_token_id = $3 -@@ -202,7 +203,7 @@ module Voting_for = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -223,7 +224,7 @@ module Token_symbols = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -269,8 +270,8 @@ module Account_identifiers = struct - return None - | Some tok_id -> - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 int int) -+ (find_opt_req -+ Caqti_type.(t2 int int) - Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name - ~cols:Fields.names () ) ) -@@ -283,15 +284,15 @@ module Account_identifiers = struct - let token = Account_id.token_id account_id in - let%bind token_id = Token.find (module Conn) token in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup2 int int) -+ (find_req -+ Caqti_type.(t2 int int) - Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name ~cols:Fields.names ()) ) - (public_key_id, token_id) - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -309,7 +310,7 @@ module Zkapp_field = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "field" ]) ) - id - end -@@ -334,7 +335,7 @@ module Zkapp_field_array = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Mina_caqti.array_int_typ -+ (find_req Caqti_type.int Mina_caqti.array_int_typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "element_ids" ]) ) - id - end -@@ -406,7 +407,7 @@ module Zkapp_states_nullable = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -467,7 +468,7 @@ module Zkapp_states = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -509,7 +510,7 @@ module Zkapp_action_states = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -531,7 +532,7 @@ module Zkapp_verification_key_hashes = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -566,7 +567,7 @@ module Zkapp_verification_keys = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -596,22 +597,22 @@ module Protocol_versions = struct - - let find (module Conn : CONNECTION) ~transaction ~network ~patch = - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name ~cols:Fields.names ()) ) - (transaction, network, patch) - - let find_txn_version (module Conn : CONNECTION) ~transaction = - Conn.collect_list -- (Caqti_request.collect Caqti_type.int Caqti_type.int -+ (collect_req Caqti_type.int Caqti_type.int - {sql| SELECT id FROM protocol_versions WHERE transaction = ? - |sql} ) - transaction - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -710,7 +711,7 @@ module Zkapp_permissions = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -763,7 +764,7 @@ module Zkapp_timing_info = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -784,7 +785,7 @@ module Zkapp_uri = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) - id - end -@@ -877,7 +878,7 @@ module Zkapp_updates = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -905,7 +906,7 @@ module Zkapp_balance_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -934,7 +935,7 @@ module Zkapp_nonce_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1018,7 +1019,7 @@ module Zkapp_account_precondition = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1046,7 +1047,7 @@ module Zkapp_token_id_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1078,7 +1079,7 @@ module Zkapp_timestamp_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1106,7 +1107,7 @@ module Zkapp_length_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1134,7 +1135,7 @@ module Zkapp_amount_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1169,7 +1170,7 @@ module Zkapp_global_slot_bounds = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1198,7 +1199,7 @@ module Timing_info = struct - Account_identifiers.find (module Conn) account_id - in - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - {sql| SELECT account_identifier_id, initial_minimum_balance, - cliff_time, cliff_amount, - vesting_period, vesting_increment -@@ -1210,7 +1211,7 @@ module Timing_info = struct - let find_by_account_identifier_id_opt (module Conn : CONNECTION) - account_identifier_id = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.int typ -+ (find_opt_req Caqti_type.int typ - {sql| SELECT account_identifier_id, initial_minimum_balance, - cliff_time, cliff_amount, - vesting_period, vesting_increment -@@ -1253,7 +1254,7 @@ module Timing_info = struct - in - match%bind - Conn.find_opt -- (Caqti_request.find_opt typ Caqti_type.int -+ (find_opt_req typ Caqti_type.int - {sql| SELECT id FROM timing_info - WHERE account_identifier_id = ? - AND initial_minimum_balance = ? -@@ -1267,7 +1268,7 @@ module Timing_info = struct - return id - | None -> - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - {sql| INSERT INTO timing_info - (account_identifier_id,initial_minimum_balance, - cliff_time, cliff_amount, vesting_period, vesting_increment) -@@ -1278,13 +1279,13 @@ module Timing_info = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - - let load_opt (module Conn : CONNECTION) id = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.int typ -+ (find_opt_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1293,13 +1294,13 @@ module Snarked_ledger_hash = struct - let find (module Conn : CONNECTION) (t : Frozen_ledger_hash.t) = - let hash = Frozen_ledger_hash.to_base58_check t in - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (find_req Caqti_type.string Caqti_type.int - "SELECT id FROM snarked_ledger_hashes WHERE value = ?" ) - hash - - let find_by_id (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) - id - -@@ -1309,7 +1310,7 @@ module Snarked_ledger_hash = struct - let hash = Frozen_ledger_hash.to_base58_check t in - match%bind - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ (find_opt_req Caqti_type.string Caqti_type.int - "SELECT id FROM snarked_ledger_hashes WHERE value = ?" ) - hash - with -@@ -1317,13 +1318,13 @@ module Snarked_ledger_hash = struct - return id - | None -> - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (find_req Caqti_type.string Caqti_type.int - "INSERT INTO snarked_ledger_hashes (value) VALUES (?) RETURNING id" ) - hash - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Caqti_type.string -+ (find_req Caqti_type.int Caqti_type.string - "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) - id - end -@@ -1359,7 +1360,7 @@ module Zkapp_epoch_ledger = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1419,7 +1420,7 @@ module Zkapp_epoch_data = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1494,7 +1495,7 @@ module Zkapp_network_precondition = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1586,7 +1587,7 @@ module Zkapp_events = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int Mina_caqti.array_int_typ -+ (find_req Caqti_type.int Mina_caqti.array_int_typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "element_ids" ]) ) - id - end -@@ -1738,7 +1739,7 @@ module Zkapp_account_update_body = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1769,7 +1770,7 @@ module Zkapp_account_update = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1813,7 +1814,7 @@ module Zkapp_fee_payer_body = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1870,7 +1871,7 @@ module Epoch_data = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -1911,13 +1912,13 @@ module User_command = struct - let find (module Conn : CONNECTION) ~(transaction_hash : Transaction_hash.t) - ~v1_transaction_hash = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ (find_opt_req Caqti_type.string Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name ~cols:[ "hash" ] ()) ) - (txn_hash_to_base58_check transaction_hash ~v1_transaction_hash) - - let load (module Conn : CONNECTION) ~(id : int) = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - -@@ -1961,7 +1962,7 @@ module User_command = struct - in - (* TODO: Converting these uint64s to int64 can overflow; see #5419 *) - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2008,7 +2009,7 @@ module User_command = struct - Public_key.add_if_doesn't_exist (module Conn) user_cmd.receiver - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2049,14 +2050,14 @@ module User_command = struct - let find_opt (module Conn : CONNECTION) - ~(transaction_hash : Transaction_hash.t) = - Conn.find_opt -- ( Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ ( find_opt_req Caqti_type.string Caqti_type.int - @@ Mina_caqti.select_cols ~select:"id" ~table_name ~cols:[ "hash" ] () - ) - (Transaction_hash.to_base58_check transaction_hash) - - let load (module Conn : CONNECTION) id = - Conn.find -- ( Caqti_request.find Caqti_type.int typ -+ ( find_req Caqti_type.int typ - @@ Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names ) - id - -@@ -2132,8 +2133,8 @@ module Internal_command = struct - let find_opt (module Conn : CONNECTION) ~(v1_transaction_hash : bool) - ~(transaction_hash : Transaction_hash.t) ~(command_type : string) = - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 string string) -+ (find_opt_req -+ Caqti_type.(t2 string string) - Caqti_type.int - (Mina_caqti.select_cols ~select:"id" ~table_name - ~tannot:(function -@@ -2144,7 +2145,7 @@ module Internal_command = struct - - let load (module Conn : CONNECTION) ~(id : int) = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - -@@ -2165,7 +2166,7 @@ module Internal_command = struct - Public_key.add_if_doesn't_exist (module Conn) internal_cmd.receiver - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "command_type" -> Some "internal_command_type" | _ -> None -@@ -2210,7 +2211,7 @@ module Fee_transfer = struct - in - Ok { kind; receiver_id; fee; hash } - in -- let rep = Caqti_type.(tup4 string int int64 string) in -+ let rep = Caqti_type.(t4 string int int64 string) in - Caqti_type.custom ~encode ~decode rep - - let add_if_doesn't_exist (module Conn : CONNECTION) -@@ -2232,7 +2233,7 @@ module Fee_transfer = struct - Public_key.add_if_doesn't_exist (module Conn) pk - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - {sql| INSERT INTO internal_commands - (command_type, receiver_id, fee, hash) - VALUES (?::internal_command_type, ?, ?, ?) -@@ -2260,7 +2261,7 @@ module Coinbase = struct - let decode (_, receiver_id, amount, hash) = - Ok { receiver_id; amount; hash } - in -- let rep = Caqti_type.(tup4 string int int64 string) in -+ let rep = Caqti_type.(t4 string int int64 string) in - Caqti_type.custom ~encode ~decode rep - - let add_if_doesn't_exist (module Conn : CONNECTION) -@@ -2281,7 +2282,7 @@ module Coinbase = struct - Public_key.add_if_doesn't_exist (module Conn) pk - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - {sql| INSERT INTO internal_commands - (command_type, receiver_id, fee, hash) - VALUES (?::internal_command_type, ?, ?, ?) -@@ -2320,7 +2321,7 @@ module Block_and_internal_command = struct - Option.map ~f:Transaction_status.Failure.to_string failure_reason - in - Conn.exec -- (Caqti_request.exec typ -+ (exec_req typ - {sql| INSERT INTO blocks_internal_commands - (block_id, - internal_command_id, -@@ -2341,8 +2342,8 @@ module Block_and_internal_command = struct - let find (module Conn : CONNECTION) ~block_id ~internal_command_id - ~sequence_no ~secondary_sequence_no = - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup4 int int int int) -+ (find_opt_req -+ Caqti_type.(t4 int int int int) - Caqti_type.string - {sql| SELECT 'exists' FROM blocks_internal_commands - WHERE block_id = $1 -@@ -2373,8 +2374,8 @@ module Block_and_internal_command = struct - ~sequence_no ~secondary_sequence_no = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup4 int int int int) -+ (find_req -+ Caqti_type.(t4 int int int int) - typ - (sprintf - {sql| SELECT %s FROM blocks_internal_commands -@@ -2409,7 +2410,7 @@ module Block_and_signed_command = struct - Option.map ~f:Transaction_status.Failure.to_string failure_reason - in - Conn.exec -- (Caqti_request.exec typ -+ (exec_req typ - {sql| INSERT INTO blocks_user_commands - (block_id, - user_command_id, -@@ -2439,8 +2440,8 @@ module Block_and_signed_command = struct - let open Deferred.Result.Let_syntax in - match%bind - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup3 int int int) -+ (find_opt_req -+ Caqti_type.(t3 int int int) - Caqti_type.string - {sql| SELECT 'exists' FROM blocks_user_commands - WHERE block_id = $1 -@@ -2459,8 +2460,8 @@ module Block_and_signed_command = struct - let load (module Conn : CONNECTION) ~block_id ~user_command_id ~sequence_no = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - typ - (sprintf - {sql| SELECT %s FROM blocks_user_commands -@@ -2494,7 +2495,7 @@ module Zkapp_account_update_failures = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name - ~cols:[ "index"; "failures" ] ) ) - id -@@ -2537,8 +2538,7 @@ module Block_and_zkapp_command = struct - in - Mina_caqti.select_insert_into_cols - ~select: -- ( "block_id, zkapp_command_id, sequence_no" -- , Caqti_type.(tup3 int int int) ) -+ ("block_id, zkapp_command_id, sequence_no", Caqti_type.(t3 int int int)) - ~table_name - ~cols: - ( [ "block_id" -@@ -2561,8 +2561,8 @@ module Block_and_zkapp_command = struct - let load (module Conn : CONNECTION) ~block_id ~zkapp_command_id ~sequence_no = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int int) -+ (find_req -+ Caqti_type.(t3 int int int) - typ - (Mina_caqti.select_cols ~table_name ~select:comma_cols - ~cols:[ "block_id"; "zkapp_command_id"; "sequence_no" ] -@@ -2572,7 +2572,7 @@ module Block_and_zkapp_command = struct - let all_from_block (module Conn : CONNECTION) ~block_id = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.collect_list -- (Caqti_request.collect Caqti_type.int typ -+ (collect_req Caqti_type.int typ - (Mina_caqti.select_cols ~table_name ~select:comma_cols - ~cols:[ "block_id" ] () ) ) - block_id -@@ -2644,7 +2644,7 @@ module Zkapp_account = struct - - let load (module Conn : CONNECTION) id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) - id - end -@@ -2688,8 +2688,8 @@ module Accounts_accessed = struct - let find_opt (module Conn : CONNECTION) ~block_id ~account_identifier_id = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 int int) -+ (find_opt_req -+ Caqti_type.(t2 int int) - typ - (sprintf - {sql| SELECT %s -@@ -2760,7 +2760,7 @@ module Accounts_accessed = struct - } - in - Mina_caqti.select_insert_into_cols -- ~select:("block_id,account_identifier_id", Caqti_type.(tup2 int int)) -+ ~select:("block_id,account_identifier_id", Caqti_type.(t2 int int)) - ~table_name ~cols:(Fields.names, typ) - (module Conn) - account_accessed -@@ -2776,7 +2776,7 @@ module Accounts_accessed = struct - let all_from_block (module Conn : CONNECTION) block_id = - let comma_cols = String.concat Fields.names ~sep:"," in - Conn.collect_list -- (Caqti_request.collect Caqti_type.int typ -+ (collect_req Caqti_type.int typ - (Mina_caqti.select_cols ~select:comma_cols ~table_name - ~cols:[ "block_id" ] () ) ) - block_id -@@ -2801,7 +2801,7 @@ module Accounts_created = struct - in - let creation_fee = Currency.Fee.to_string creation_fee in - Mina_caqti.select_insert_into_cols -- ~select:("block_id,account_identifier_id", Caqti_type.(tup2 int int)) -+ ~select:("block_id,account_identifier_id", Caqti_type.(t2 int int)) - ~table_name ~cols:(Fields.names, typ) - (module Conn) - { block_id; account_identifier_id; creation_fee } -@@ -2816,7 +2816,7 @@ module Accounts_created = struct - - let all_from_block (module Conn : CONNECTION) block_id = - Conn.collect_list -- (Caqti_request.collect Caqti_type.int typ -+ (collect_req Caqti_type.int typ - {sql| SELECT block_id, account_identifier_id, creation_fee - FROM accounts_created - WHERE block_id = ? -@@ -2882,14 +2882,14 @@ module Block = struct - "SELECT id FROM blocks WHERE state_hash = ?" ) - (State_hash.to_base58_check state_hash) - -- let find (module Conn : CONNECTION) = make_finder Conn.find Caqti_request.find -+ let find (module Conn : CONNECTION) = make_finder Conn.find find_req - - let find_opt (module Conn : CONNECTION) = -- make_finder Conn.find_opt Caqti_request.find_opt -+ make_finder Conn.find_opt find_opt_req - - let load (module Conn : CONNECTION) ~id = - Conn.find -- (Caqti_request.find Caqti_type.int typ -+ (find_req Caqti_type.int typ - (Mina_caqti.select_cols_from_id ~table_name:"blocks" ~cols:Fields.names) ) - id - -@@ -2994,7 +2994,7 @@ module Block = struct - let blockchain_state = Protocol_state.blockchain_state protocol_state in - let%bind block_id = - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "chain_status" -> -@@ -3376,9 +3376,7 @@ module Block = struct - in - - (* we don't need to specify all types here, just the ones that sql may infer incorrectly *) -- let field_name : type a. a Caqti_type.Field.t -> string option = -- let open Caqti_type in -- function -+ let field_name : type a. a Caqti_type.Field.t -> string option = function - | Bool -> - Some "BOOL" - | Int -> -@@ -3391,39 +3389,36 @@ module Block = struct - Some "BIGINT" - | Float -> - Some "FLOAT" -+ | String -> -+ Some "TEXT" - | Enum name -> - Some name -- | _ -> -- None -+ | Octets | Pdate | Ptime | Ptime_span -> -+ failwith "unsupported field type" - in - - let rec type_field_names : type a. a Caqti_type.t -> string option list = -+ let open Caqti_type in - function -- | Unit -> -- [] - | Field f -> - [ field_name f ] - | Option t -> - type_field_names t -- | Tup2 (at, bt) -> -- List.concat [ type_field_names at; type_field_names bt ] -- | Tup3 (at, bt, ct) -> -- List.concat -- [ type_field_names at; type_field_names bt; type_field_names ct ] -- | Tup4 (at, bt, ct, dt) -> -- List.concat -- [ type_field_names at -- ; type_field_names bt -- ; type_field_names ct -- ; type_field_names dt -- ] -- | Custom custom -> -- type_field_names custom.rep -- in -- -- let rec render_field : type a. a Caqti_type.Field.t -> a -> string = -+ | Product (_, prod) -> -+ let rec loop : type a i. _ -> (a, i) product -> _ = -+ fun acc -> function -+ | Proj_end -> -+ List.concat (List.rev acc) -+ | Proj (t, _, prod) -> -+ loop (type_field_names t :: acc) prod -+ in -+ loop [] prod -+ | Annot (_, t) -> -+ type_field_names t -+ in -+ -+ let render_field : type a. a Caqti_type.Field.t -> a -> string = - fun typ value -> -- let open Caqti_type in - match typ with - | Bool -> - Bool.to_string value -@@ -3451,45 +3446,30 @@ module Block = struct - (* we are ignoring the enum annotation in this context because it's not always valid to apply *) - (* NOTE: we assume enum values do not contain special characters (eg "'") *) - "'" ^ value ^ "'" -- | _ -> ( -- match Caqti_type.Field.coding Conn.driver_info typ with -- | None -> -- failwithf "unable to render caqti field: %s" -- (Caqti_type.Field.to_string typ) -- () -- | Some (Coding coding) -> -- render_field coding.rep -- (Result.ok_or_failwith @@ coding.encode value) ) - in - let rec render_type : type a. a Caqti_type.t -> a -> string list = -- fun typ value -> -- match typ with -- | Unit -> -- [] -- | Field f -> -- [ render_field f value ] -- | Option t -> ( -- match value with -- | None -> -- List.init (Caqti_type.length typ) ~f:(Fn.const "NULL") -- | Some x -> -- render_type t x ) -- | Tup2 (at, bt) -> -- let a, b = value in -- List.concat [ render_type at a; render_type bt b ] -- | Tup3 (at, bt, ct) -> -- let a, b, c = value in -- List.concat [ render_type at a; render_type bt b; render_type ct c ] -- | Tup4 (at, bt, ct, dt) -> -- let a, b, c, d = value in -- List.concat -- [ render_type at a -- ; render_type bt b -- ; render_type ct c -- ; render_type dt d -- ] -- | Custom custom -> -- render_type custom.rep (Result.ok_or_failwith @@ custom.encode value) -+ let open Caqti_type in -+ fun typ value -> -+ match typ with -+ | Field f -> -+ [ render_field f value ] -+ | Option t -> ( -+ match value with -+ | None -> -+ List.init (Caqti_type.length typ) ~f:(Fn.const "NULL") -+ | Some x -> -+ render_type t x ) -+ | Product (_, prod) -> -+ let rec loop : type i. _ -> (a, i) product -> _ = -+ fun acc -> function -+ | Proj_end -> -+ List.concat (List.rev acc) -+ | Proj (t, prj, prod) -> -+ loop (render_type t (prj value) :: acc) prod -+ in -+ loop [] prod -+ | Annot (_, t) -> -+ render_type t value - in - let render_row (type a) (typ : a Caqti_type.t) (value : a) : string = - "(" ^ String.concat ~sep:"," (render_type typ value) ^ ")" -@@ -3540,8 +3520,8 @@ module Block = struct - in - let%map entries = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 typ int) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 typ int) - query ) - () - in -@@ -3559,7 +3539,7 @@ module Block = struct - String.concat ~sep:"," @@ List.map ~f:(render_row typ) values - in - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit Caqti_type.int -+ (Mina_caqti.collect_req Caqti_type.unit Caqti_type.int - (sprintf "INSERT INTO %s (%s) VALUES %s RETURNING id" table - fields_sql values_sql ) ) - () ) -@@ -4050,7 +4030,7 @@ module Block = struct - let ids_sql = String.concat ~sep:"," ids in - let parent_ids_sql = String.concat ~sep:"," parent_ids in - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - "UPDATE %s AS b SET parent_id = data.parent_id FROM (SELECT \ - unnest(array[%s]) as id, unnest(array[%s]) as parent_id) AS \ -@@ -4119,7 +4099,7 @@ module Block = struct - Some id ) - in - Conn.find -- (Caqti_request.find typ Caqti_type.int -+ (find_req typ Caqti_type.int - (Mina_caqti.insert_into_cols ~returning:"id" ~table_name - ~tannot:(function - | "sub_window_densities" -> -@@ -4280,8 +4260,8 @@ module Block = struct - let set_parent_id_if_null (module Conn : CONNECTION) ~parent_hash - ~(parent_id : int) = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int string) -+ (exec_req -+ Caqti_type.(t2 int string) - {sql| UPDATE blocks SET parent_id = ? - WHERE parent_hash = ? - AND parent_id IS NULL -@@ -4297,8 +4277,8 @@ module Block = struct - in - let columns = concat Fields.names in - Conn.collect_list -- (Caqti_request.collect -- Caqti_type.(tup2 int int) -+ (collect_req -+ Caqti_type.(t2 int int) - typ - (sprintf - {sql| WITH RECURSIVE chain AS ( -@@ -4324,37 +4304,37 @@ module Block = struct - - let get_highest_canonical_block_opt (module Conn : CONNECTION) = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.unit -- Caqti_type.(tup2 int int64) -+ (find_opt_req Caqti_type.unit -+ Caqti_type.(t2 int int64) - "SELECT id,height FROM blocks WHERE chain_status='canonical' ORDER BY \ - height DESC LIMIT 1" ) - - let get_nearest_canonical_block_above (module Conn : CONNECTION) height = - Conn.find -- (Caqti_request.find Caqti_type.int64 -- Caqti_type.(tup2 int int64) -+ (find_req Caqti_type.int64 -+ Caqti_type.(t2 int int64) - "SELECT id,height FROM blocks WHERE chain_status='canonical' AND \ - height > ? ORDER BY height ASC LIMIT 1" ) - height - - let get_nearest_canonical_block_below (module Conn : CONNECTION) height = - Conn.find -- (Caqti_request.find Caqti_type.int64 -- Caqti_type.(tup2 int int64) -+ (find_req Caqti_type.int64 -+ Caqti_type.(t2 int int64) - "SELECT id,height FROM blocks WHERE chain_status='canonical' AND \ - height < ? ORDER BY height DESC LIMIT 1" ) - height - - let mark_as_canonical (module Conn : CONNECTION) ~state_hash = - Conn.exec -- (Caqti_request.exec Caqti_type.string -+ (exec_req Caqti_type.string - "UPDATE blocks SET chain_status='canonical' WHERE state_hash = ?" ) - state_hash - - let mark_as_orphaned (module Conn : CONNECTION) ~state_hash ~height = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 string int64) -+ (exec_req -+ Caqti_type.(t2 string int64) - {sql| UPDATE blocks SET chain_status='orphaned' - WHERE height = $2 - AND state_hash <> $1 -@@ -4443,7 +4423,7 @@ module Block = struct - | None, Some num_blocks -> ( - match%map - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.unit Caqti_type.int -+ (find_opt_req Caqti_type.unit Caqti_type.int - "SELECT MAX(height) FROM blocks" ) - () - with -@@ -4459,8 +4439,8 @@ module Block = struct - let%bind () = - (* Delete user commands from old blocks. *) - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int int64) -+ (exec_req -+ Caqti_type.(t2 int int64) - "DELETE FROM user_commands\n\ - WHERE id IN\n\ - (SELECT user_command_id FROM blocks_user_commands\n\ -@@ -4471,8 +4451,8 @@ module Block = struct - let%bind () = - (* Delete old blocks. *) - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int int64) -+ (exec_req -+ Caqti_type.(t2 int int64) - "DELETE FROM blocks WHERE blocks.height < ? OR blocks.timestamp < \ - ?" ) - (height, timestamp) -@@ -4480,7 +4460,7 @@ module Block = struct - let%bind () = - (* Delete orphaned internal commands. *) - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (exec_req Caqti_type.unit - "DELETE FROM internal_commands\n\ - WHERE id NOT IN\n\ - (SELECT internal_commands.id FROM internal_commands\n\ -@@ -4491,7 +4471,7 @@ module Block = struct - let%bind () = - (* Delete orphaned snarked ledger hashes. *) - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (exec_req Caqti_type.unit - "DELETE FROM snarked_ledger_hashes\n\ - WHERE id NOT IN\n\ - (SELECT snarked_ledger_hash_id FROM blocks)" ) -@@ -4500,7 +4480,7 @@ module Block = struct - let%bind () = - (* Delete orphaned public keys. *) - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (exec_req Caqti_type.unit - "DELETE FROM public_keys\n\ - WHERE id NOT IN (SELECT fee_payer_id FROM user_commands)\n\ - AND id NOT IN (SELECT source_id FROM user_commands)\n\ -@@ -4893,7 +4873,13 @@ let setup_server ~metrics_server_port ~constraint_constants ~logger - Strict_pipe.Writer.write extensional_block_writer extensional_block ) - ] - in -- match Caqti_async.connect_pool ~max_size:30 postgres_address with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:30 ())) -+ postgres_address -+ with - | Error e -> - [%log error] - "Failed to create a Caqti pool for Postgresql, see error: $error" -diff --git a/src/app/best_tip_merger/best_tip_merger.ml b/src/app/best_tip_merger/best_tip_merger.ml -index 5682f29ec5..90f418c153 100644 ---- a/src/app/best_tip_merger/best_tip_merger.ml -+++ b/src/app/best_tip_merger/best_tip_merger.ml -@@ -205,7 +205,7 @@ module Compact_display = struct - { current : State_hash.t - ; parent : State_hash.t - ; blockchain_length : Mina_numbers.Length.t -- ; global_slot : Mina_numbers.Global_slot_since_genesis.t -+ ; global_slot : Mina_numbers.Global_slot_since_hard_fork.t - } - [@@deriving yojson] - -@@ -243,7 +243,7 @@ module Graph_node = struct - | Node of - { current : State_hash.t - ; length : Mina_numbers.Length.t -- ; slot : Mina_numbers.Global_slot_since_genesis.t -+ ; slot : Mina_numbers.Global_slot_since_hard_fork.t - } - [@@deriving yojson, equal, hash] - -@@ -267,7 +267,7 @@ module Graph_node = struct - ("NA", "NA") - | Node s -> - ( Mina_numbers.Length.to_string s.length -- , Mina_numbers.Global_slot_since_genesis.to_string s.slot ) -+ , Mina_numbers.Global_slot_since_hard_fork.to_string s.slot ) - in - { state; slot; length; peers = t.peers } - -diff --git a/src/app/delegation_compliance/delegation_compliance.ml b/src/app/delegation_compliance/delegation_compliance.ml -index 8a0e92a6c5..0216fbd1e3 100644 ---- a/src/app/delegation_compliance/delegation_compliance.ml -+++ b/src/app/delegation_compliance/delegation_compliance.ml -@@ -343,7 +343,13 @@ let main ~input_file ~csv_file ~preliminary_csv_file_opt ~archive_uri - csv_datas - in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ archive_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/delegation_compliance/sql.ml b/src/app/delegation_compliance/sql.ml -index 3d214eb9c9..ea264fdeb6 100644 ---- a/src/app/delegation_compliance/sql.ml -+++ b/src/app/delegation_compliance/sql.ml -@@ -16,7 +16,7 @@ module Block_info = struct - - (* find all blocks, working back from block with given state hash *) - let query = -- Caqti_request.collect Caqti_type.string typ -+ Mina_caqti.collect_req Caqti_type.string typ - {sql| WITH RECURSIVE chain AS ( - - SELECT id,parent_id,global_slot,state_hash,ledger_hash FROM blocks b WHERE b.state_hash = ? -@@ -122,7 +122,7 @@ module User_command = struct - Caqti_type.custom ~encode ~decode (to_rep spec) - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT type,fee_payer_id, source_id,receiver_id,fee,fee_token,token,amount,valid_until,memo,nonce, - blocks.id,blocks.global_slot,parent.global_slot_since_genesis, - sequence_no,status,created_token, -@@ -148,8 +148,8 @@ module User_command = struct - Conn.collect_list query user_cmd_id - - let query_payments_by_source_and_receiver = -- Caqti_request.collect -- Caqti_type.(tup2 int int) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 int int) - typ - {sql| SELECT type,fee_payer_id, source_id, receiver_id, fee,fee_token, - token, amount, valid_until, memo, nonce, blocks.id, blocks.global_slot, -@@ -182,7 +182,7 @@ module User_command = struct - (source_id, receiver_id) - - let query_payments_by_receiver = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT type,fee_payer_id, source_id, receiver_id, fee,fee_token, - token, amount, valid_until, memo, nonce, blocks.id, blocks.global_slot, - parent.global_slot_since_genesis, sequence_no, status, created_token, -@@ -214,7 +214,7 @@ end - - module Public_key = struct - let query = -- Caqti_request.find_opt Caqti_type.int Caqti_type.string -+ Mina_caqti.find_opt_req Caqti_type.int Caqti_type.string - {sql| SELECT value FROM public_keys - WHERE id = ? - |sql} -@@ -223,7 +223,7 @@ module Public_key = struct - Conn.find_opt query pk_id - - let query_for_id = -- Caqti_request.find_opt Caqti_type.string Caqti_type.int -+ Mina_caqti.find_opt_req Caqti_type.string Caqti_type.int - {sql| SELECT id FROM public_keys - WHERE value = ? - |sql} -@@ -234,7 +234,7 @@ end - - module Block = struct - let max_slot_query = -- Caqti_request.find Caqti_type.unit Caqti_type.int -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int - {sql| SELECT MAX(global_slot) FROM blocks - |sql} - -@@ -242,7 +242,7 @@ module Block = struct - Conn.find max_slot_query () - - let state_hashes_by_slot_query = -- Caqti_request.collect Caqti_type.int Caqti_type.string -+ Mina_caqti.collect_req Caqti_type.int Caqti_type.string - {sql| SELECT state_hash FROM blocks WHERE global_slot = $1 - |sql} - -@@ -250,8 +250,8 @@ module Block = struct - Conn.collect_list state_hashes_by_slot_query slot - - let creator_slot_bounds_query = -- Caqti_request.collect -- Caqti_type.(tup3 int int64 int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t3 int int64 int64) - Caqti_type.int - {sql| SELECT id FROM blocks - WHERE creator_id = $1 -@@ -268,7 +268,7 @@ module Coinbase_receivers_for_block_creator = struct - with given creator_id, where the receiver distinct from the creator_id - *) - let query = -- Caqti_request.collect Caqti_type.int Caqti_type.int -+ Mina_caqti.collect_req Caqti_type.int Caqti_type.int - {sql| SELECT DISTINCT ic.receiver_id - - FROM blocks b -diff --git a/src/app/extract_blocks/extract_blocks.ml b/src/app/extract_blocks/extract_blocks.ml -index 3605538ae4..1bf72965c9 100644 ---- a/src/app/extract_blocks/extract_blocks.ml -+++ b/src/app/extract_blocks/extract_blocks.ml -@@ -479,7 +479,13 @@ let main ~archive_uri ~start_state_hash_opt ~end_state_hash_opt ~all_blocks () = - (* sanity-check input state hashes *) - check_state_hash ~logger start_state_hash_opt ; - check_state_hash ~logger end_state_hash_opt ; -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ archive_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/extract_blocks/sql.ml b/src/app/extract_blocks/sql.ml -index 2c4b659973..0f614da7c7 100644 ---- a/src/app/extract_blocks/sql.ml -+++ b/src/app/extract_blocks/sql.ml -@@ -31,12 +31,12 @@ module Subchain = struct - fields b_fields join_condition fields - - let query_unparented = -- Caqti_request.collect Caqti_type.string Archive_lib.Processor.Block.typ -+ Mina_caqti.collect_req Caqti_type.string Archive_lib.Processor.Block.typ - (make_sql ~join_condition:"b.id = chain.parent_id") - - let query_from_start = -- Caqti_request.collect -- Caqti_type.(tup2 string string) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string string) - Archive_lib.Processor.Block.typ - (make_sql - ~join_condition: -@@ -56,7 +56,7 @@ module Subchain = struct - let comma_fields = - String.concat Archive_lib.Processor.Block.Fields.names ~sep:"," - in -- Caqti_request.collect Caqti_type.unit Archive_lib.Processor.Block.typ -+ Mina_caqti.collect_req Caqti_type.unit Archive_lib.Processor.Block.typ - (sprintf "SELECT %s FROM blocks" comma_fields) - - let all_blocks (module Conn : Caqti_async.CONNECTION) = -@@ -67,8 +67,8 @@ end - - module Blocks_and_user_commands = struct - let query = -- Caqti_request.collect Caqti_type.int -- Caqti_type.(tup2 int int) -+ Mina_caqti.collect_req Caqti_type.int -+ Caqti_type.(t2 int int) - {sql| SELECT user_command_id, sequence_no - FROM blocks_user_commands - WHERE block_id = ? -@@ -91,7 +91,7 @@ module Block_user_command_tokens = struct - Caqti_type.[ string; option int; option int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT tokens.value, owner_public_key_id, owner_token_id - FROM (blocks_user_commands buc - INNER JOIN blocks -@@ -122,7 +122,7 @@ module Blocks_and_internal_commands = struct - Caqti_type.[ int; int; int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT internal_command_id, sequence_no, secondary_sequence_no - FROM (blocks_internal_commands - INNER JOIN blocks -@@ -147,7 +147,7 @@ module Block_internal_command_tokens = struct - Caqti_type.[ string; option int; option int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT tokens.value, owner_public_key_id, owner_token_id - FROM (blocks_internal_commands bic - INNER JOIN blocks -@@ -167,8 +167,8 @@ end - - module Blocks_and_zkapp_commands = struct - let query = -- Caqti_request.collect Caqti_type.int -- Caqti_type.(tup2 int int) -+ Mina_caqti.collect_req Caqti_type.int -+ Caqti_type.(t2 int int) - {sql| SELECT zkapp_command_id, sequence_no - FROM blocks_zkapp_commands - WHERE block_id = ? -@@ -191,7 +191,7 @@ module Block_zkapp_command_tokens = struct - Caqti_type.[ string; option int; option int ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT tokens.value, owner_public_key_id, owner_token_id - FROM (blocks_zkapp_commands bzkc - INNER JOIN blocks -diff --git a/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml b/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml -index 65b34d6b55..efe0668cc2 100644 ---- a/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml -+++ b/src/app/genesis_ledger_from_tsv/genesis_ledger_from_tsv.ml -@@ -67,15 +67,11 @@ let amount_geq_min_balance ~amount ~initial_min_balance = - let generate_delegate_account ~logger delegatee_pk = - [%log info] "Generating account for delegatee $delegatee" - ~metadata:[ ("delegatee", `String delegatee_pk) ] ; -- let pk = Some delegatee_pk in -- let balance = Currency.Balance.zero in -- let timing = None in -- let delegate = None in - { Runtime_config.Json_layout.Accounts.Single.default with -- pk -- ; balance -- ; timing -- ; delegate -+ pk = delegatee_pk -+ ; balance = Currency.Balance.zero -+ ; timing = None -+ ; delegate = None - } - - let generate_missing_delegate_accounts ~logger = -@@ -97,7 +93,6 @@ let runtime_config_account ~logger ~wallet_pk ~amount ~initial_min_balance - ~delegatee_pk = - [%log info] "Processing record for $wallet_pk" - ~metadata:[ ("wallet_pk", `String wallet_pk) ] ; -- let pk = Some wallet_pk in - let balance = Currency.Balance.of_mina_string_exn amount in - let initial_minimum_balance = - (* if omitted in the TSV, use balance *) -@@ -135,7 +130,9 @@ let runtime_config_account ~logger ~wallet_pk ~amount ~initial_min_balance - .initial_minimum_balance - ; cliff_time - ; cliff_amount -- ; vesting_period -+ ; vesting_period = -+ Global_slot_since_genesis.( -+ diff vesting_period zero |> Option.value_exn) - ; vesting_increment - } - in -@@ -144,7 +141,7 @@ let runtime_config_account ~logger ~wallet_pk ~amount ~initial_min_balance - if no_delegatee delegatee_pk then None else Some delegatee_pk - in - { Runtime_config.Json_layout.Accounts.Single.default with -- pk -+ pk = wallet_pk - ; balance - ; timing - ; delegate -diff --git a/src/app/ledger_export_bench/ledger_export_benchmark.ml b/src/app/ledger_export_bench/ledger_export_benchmark.ml -index cd5846345a..5125f8ba90 100644 ---- a/src/app/ledger_export_bench/ledger_export_benchmark.ml -+++ b/src/app/ledger_export_bench/ledger_export_benchmark.ml -@@ -26,7 +26,7 @@ let () = - let cfg = load_daemon_cfg runtime_config () in - let accounts = - match cfg.ledger with -- | None | Some { base = Named _; _ } | Some { base = Hash _; _ } -> -+ | None | Some { base = Named _; _ } | Some { base = Hash; _ } -> - [] - | Some { base = Accounts accs; _ } -> - List.map ~f:Runtime_config.Accounts.Single.to_account accs -diff --git a/src/app/migrate-balances-table/dune b/src/app/migrate-balances-table/dune -index dbb463c78c..dbe6d09aad 100644 ---- a/src/app/migrate-balances-table/dune -+++ b/src/app/migrate-balances-table/dune -@@ -18,6 +18,7 @@ - ;; local libraries - logger - bounded_types -+ mina_caqti - ) - (preprocessor_deps ../../config.mlh) - (instrumentation (backend bisect_ppx)) -diff --git a/src/app/migrate-balances-table/migrate_balances_table.ml b/src/app/migrate-balances-table/migrate_balances_table.ml -index 76d0549c2a..ea8f83f841 100644 ---- a/src/app/migrate-balances-table/migrate_balances_table.ml -+++ b/src/app/migrate-balances-table/migrate_balances_table.ml -@@ -14,7 +14,13 @@ let query_db pool ~f ~item = - let main ~archive_uri () = - let logger = Logger.create () in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ archive_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/migrate-balances-table/sql.ml b/src/app/migrate-balances-table/sql.ml -index 4c0a8c86dd..f0bc5f8c81 100644 ---- a/src/app/migrate-balances-table/sql.ml -+++ b/src/app/migrate-balances-table/sql.ml -@@ -4,7 +4,7 @@ open Core_kernel - - let create_temp_balances_table (module Conn : Caqti_async.CONNECTION) = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - {sql| CREATE TABLE IF NOT EXISTS balances_temp - ( id serial PRIMARY KEY - , public_key_id int NOT NULL REFERENCES public_keys(id) -@@ -19,7 +19,7 @@ let create_temp_balances_table (module Conn : Caqti_async.CONNECTION) = - - let copy_table_to_temp_table (module Conn : Caqti_async.CONNECTION) table = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE TABLE IF NOT EXISTS %s_temp AS (SELECT * FROM %s) - |sql} -@@ -27,7 +27,7 @@ let copy_table_to_temp_table (module Conn : Caqti_async.CONNECTION) table = - - let create_table_index (module Conn : Caqti_async.CONNECTION) table col = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE INDEX IF NOT EXISTS idx_%s_%s ON %s(%s) - |sql} -@@ -39,7 +39,7 @@ let create_temp_table_index (module Conn : Caqti_async.CONNECTION) table col = - let create_table_named_index (module Conn : Caqti_async.CONNECTION) table col - name = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE INDEX IF NOT EXISTS idx_%s_%s ON %s(%s) - |sql} -@@ -51,7 +51,7 @@ let create_temp_table_named_index (module Conn : Caqti_async.CONNECTION) table - - let drop_table_index (module Conn : Caqti_async.CONNECTION) table col = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf {sql| DROP INDEX IF EXISTS idx_%s_%s - |sql} table col ) ) - -@@ -60,7 +60,7 @@ let drop_temp_table_index (module Conn : Caqti_async.CONNECTION) table col = - - let create_cursor (module Conn : Caqti_async.CONNECTION) name = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| CREATE TABLE IF NOT EXISTS %s_cursor - ( value int NOT NULL) -@@ -69,7 +69,7 @@ let create_cursor (module Conn : Caqti_async.CONNECTION) name = - - let initialize_cursor (module Conn : Caqti_async.CONNECTION) name = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| INSERT INTO %s_cursor (value) VALUES (0) - |sql} -@@ -77,13 +77,13 @@ let initialize_cursor (module Conn : Caqti_async.CONNECTION) name = - - let current_cursor (module Conn : Caqti_async.CONNECTION) name = - Conn.find_opt -- (Caqti_request.find_opt Caqti_type.unit Caqti_type.int -+ (Mina_caqti.find_opt_req Caqti_type.unit Caqti_type.int - (sprintf {sql| SELECT value FROM %s_cursor - |sql} name ) ) - - let update_cursor (module Conn : Caqti_async.CONNECTION) name ndx = - Conn.exec -- (Caqti_request.exec Caqti_type.int -+ (Mina_caqti.exec_req Caqti_type.int - (sprintf - {sql| UPDATE %s_cursor SET value = $1 - |sql} -@@ -99,7 +99,7 @@ let drop_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) table - |sql} - table foreign_key - in -- Conn.exec (Caqti_request.exec Caqti_type.unit sql) -+ Conn.exec (Mina_caqti.exec_req Caqti_type.unit sql) - - let add_balances_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) - table col foreign_key = -@@ -112,7 +112,7 @@ let add_balances_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) - |sql} - table foreign_key col - in -- Conn.exec (Caqti_request.exec Caqti_type.unit sql) -+ Conn.exec (Mina_caqti.exec_req Caqti_type.unit sql) - - let add_blocks_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) - table col foreign_key = -@@ -125,14 +125,14 @@ let add_blocks_foreign_key_constraint (module Conn : Caqti_async.CONNECTION) - |sql} - table foreign_key col - in -- Conn.exec (Caqti_request.exec Caqti_type.unit sql) -+ Conn.exec (Mina_caqti.exec_req Caqti_type.unit sql) - - let find_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id - ~balance ~block_id ~block_height ~block_sequence_no - ~block_secondary_sequence_no = - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup3 int int64 (tup4 int int int int)) -+ (Mina_caqti.find_opt_req -+ Caqti_type.(t3 int int64 (t4 int int int int)) - Caqti_type.int - {sql| SELECT id - FROM balances_temp -@@ -152,8 +152,8 @@ let insert_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id - ~balance ~block_id ~block_height ~block_sequence_no - ~block_secondary_sequence_no = - Conn.find -- (Caqti_request.find -- Caqti_type.(tup3 int int64 (tup4 int int int int)) -+ (Mina_caqti.find_req -+ Caqti_type.(t3 int int64 (t4 int int int int)) - Caqti_type.int - {sql| INSERT INTO balances_temp - ( public_key_id -@@ -178,8 +178,8 @@ let insert_balance_entry (module Conn : Caqti_async.CONNECTION) ~public_key_id - - let get_internal_commands (module Conn : Caqti_async.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup4 int int64 (tup4 int int int int) int) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t4 int int64 (t4 int int int int) int) - {sql| SELECT bal.public_key_id,bal.balance,bic.block_id,blocks.height,bic.sequence_no,bic.secondary_sequence_no, - internal_command_id - FROM blocks_internal_commands bic -@@ -193,8 +193,8 @@ let update_internal_command_receiver_balance - (module Conn : Caqti_async.CONNECTION) ~new_balance_id ~block_id - ~internal_command_id ~block_sequence_no ~block_secondary_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup4 int int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t4 int int int int)) - {sql| UPDATE blocks_internal_commands_temp SET receiver_balance = $1 - WHERE block_id = $2 - AND internal_command_id = $3 -@@ -209,8 +209,8 @@ let update_internal_command_receiver_balance - - let get_user_command_fee_payers (module Conn : Caqti_async.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (tup4 int int int int) (tup2 int int64)) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 (t4 int int int int) (t2 int int64)) - {sql| SELECT buc.block_id,blocks.height,buc.sequence_no,user_command_id, - bal_fee_payer.public_key_id,bal_fee_payer.balance - FROM blocks_user_commands buc -@@ -222,8 +222,8 @@ let get_user_command_fee_payers (module Conn : Caqti_async.CONNECTION) = - - let get_user_command_sources (module Conn : Caqti_async.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (tup4 int int int int) (tup2 int int64)) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 (t4 int int int int) (t2 int int64)) - {sql| SELECT buc.block_id,blocks.height,buc.sequence_no,user_command_id, - bal_source.public_key_id,bal_source.balance - FROM blocks_user_commands buc -@@ -236,8 +236,8 @@ let get_user_command_sources (module Conn : Caqti_async.CONNECTION) = - - let get_user_command_receivers (module Conn : Caqti_async.CONNECTION) = - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (tup4 int int int int) (tup2 int int64)) -+ (Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t2 (t4 int int int int) (t2 int int64)) - {sql| SELECT buc.block_id,blocks.height,buc.sequence_no,user_command_id, - bal_receiver.public_key_id,bal_receiver.balance - FROM blocks_user_commands buc -@@ -251,8 +251,8 @@ let get_user_command_receivers (module Conn : Caqti_async.CONNECTION) = - let update_user_command_fee_payer_balance (module Conn : Caqti_async.CONNECTION) - ~new_balance_id ~block_id ~user_command_id ~block_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup3 int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t3 int int int)) - {sql| UPDATE blocks_user_commands_temp SET fee_payer_balance = $1 - WHERE block_id = $2 - AND user_command_id = $3 -@@ -263,8 +263,8 @@ let update_user_command_fee_payer_balance (module Conn : Caqti_async.CONNECTION) - let update_user_command_source_balance (module Conn : Caqti_async.CONNECTION) - ~new_balance_id ~block_id ~user_command_id ~block_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup3 int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t3 int int int)) - {sql| UPDATE blocks_user_commands_temp SET source_balance = $1 - WHERE block_id = $2 - AND user_command_id = $3 -@@ -276,8 +276,8 @@ let update_user_command_source_balance (module Conn : Caqti_async.CONNECTION) - let update_user_command_receiver_balance (module Conn : Caqti_async.CONNECTION) - ~new_balance_id ~block_id ~user_command_id ~block_sequence_no = - Conn.exec -- (Caqti_request.exec -- Caqti_type.(tup2 int (tup3 int int int)) -+ (Mina_caqti.exec_req -+ Caqti_type.(t2 int (t3 int int int)) - {sql| UPDATE blocks_user_commands_temp SET receiver_balance = $1 - WHERE block_id = $2 - AND user_command_id = $3 -@@ -288,13 +288,13 @@ let update_user_command_receiver_balance (module Conn : Caqti_async.CONNECTION) - - let drop_table (module Conn : Caqti_async.CONNECTION) table = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf {sql| DROP TABLE %s - |sql} table ) ) - - let rename_temp_table (module Conn : Caqti_async.CONNECTION) table = - Conn.exec -- (Caqti_request.exec Caqti_type.unit -+ (Mina_caqti.exec_req Caqti_type.unit - (sprintf - {sql| ALTER TABLE %s_temp - RENAME TO %s -@@ -303,7 +303,7 @@ let rename_temp_table (module Conn : Caqti_async.CONNECTION) table = - - let get_column_count (module Conn : Caqti_async.CONNECTION) table = - Conn.find -- (Caqti_request.find Caqti_type.string Caqti_type.int -+ (Mina_caqti.find_req Caqti_type.string Caqti_type.int - {sql| SELECT COUNT(*) FROM information_schema.columns - WHERE table_name=$1 - |sql} ) -diff --git a/src/app/missing_blocks_auditor/dune b/src/app/missing_blocks_auditor/dune -index 623a5079ba..7ec1eae5af 100644 ---- a/src/app/missing_blocks_auditor/dune -+++ b/src/app/missing_blocks_auditor/dune -@@ -18,6 +18,7 @@ - ;; local libraries - logger - bounded_types -+ mina_caqti - ) - (preprocessor_deps ../../config.mlh) - (instrumentation (backend bisect_ppx)) -diff --git a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml -index 5bce995ea2..d2b54f3615 100644 ---- a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml -+++ b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml -@@ -22,7 +22,13 @@ let add_error, get_exit_code = - let main ~archive_uri () = - let logger = Logger.create () in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ archive_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/missing_blocks_auditor/sql.ml b/src/app/missing_blocks_auditor/sql.ml -index 3e2bf01e93..d8a47b926d 100644 ---- a/src/app/missing_blocks_auditor/sql.ml -+++ b/src/app/missing_blocks_auditor/sql.ml -@@ -4,8 +4,8 @@ module Unparented_blocks = struct - (* parent_hashes represent ends of chains leading to an orphan block *) - - let query = -- Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup4 int string int string) -+ Mina_caqti.collect_req Caqti_type.unit -+ Caqti_type.(t4 int string int string) - {sql| - SELECT id, state_hash, height, parent_hash FROM blocks - WHERE parent_id IS NULL -@@ -16,7 +16,7 @@ end - - module Missing_blocks_gap = struct - let query = -- Caqti_request.find Caqti_type.int Caqti_type.int -+ Mina_caqti.find_req Caqti_type.int Caqti_type.int - {sql| SELECT $1 - MAX(height) - 1 FROM blocks - WHERE height < $1 - |sql} -@@ -26,7 +26,7 @@ end - - module Chain_status = struct - let query_highest_canonical = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT max(height) FROM blocks - WHERE chain_status = 'canonical' - |sql} -@@ -35,7 +35,7 @@ module Chain_status = struct - Conn.find query_highest_canonical () - - let query_count_pending_below = -- Caqti_request.find Caqti_type.int64 Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.int64 Caqti_type.int64 - {sql| SELECT count(*) FROM blocks - WHERE chain_status = 'pending' - AND height <= ? -@@ -45,8 +45,8 @@ module Chain_status = struct - Conn.find query_count_pending_below height - - let query_canonical_chain = -- Caqti_request.collect Caqti_type.int64 -- Caqti_type.(tup3 int string string) -+ Mina_caqti.collect_req Caqti_type.int64 -+ Caqti_type.(t3 int string string) - {sql| WITH RECURSIVE chain AS ( - - (SELECT id, state_hash, parent_id, chain_status -diff --git a/src/app/patch_archive_test/patch_archive_test.ml b/src/app/patch_archive_test/patch_archive_test.ml -index 1fbc9e3104..017b6574fc 100644 ---- a/src/app/patch_archive_test/patch_archive_test.ml -+++ b/src/app/patch_archive_test/patch_archive_test.ml -@@ -124,7 +124,13 @@ let main ~archive_uri ~num_blocks_to_patch ~archive_blocks_path - let copy_uri = make_archive_copy_uri archive_uri in - [%log info] "Connecting to original database" ; - let%bind () = -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ archive_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -@@ -159,7 +165,13 @@ let main ~archive_uri ~num_blocks_to_patch ~archive_blocks_path - () - in - [%log info] "Connecting to copied database" ; -- match Caqti_async.connect_pool ~max_size:128 copy_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ copy_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/replayer/replayer.ml b/src/app/replayer/replayer.ml -index f9f22df42c..4498a21e9d 100644 ---- a/src/app/replayer/replayer.ml -+++ b/src/app/replayer/replayer.ml -@@ -661,7 +661,13 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri - msg ) - in - let archive_uri = Uri.of_string archive_uri in -- match Caqti_async.connect_pool ~max_size:128 archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) (create ~max_size:128 ())) -+ archive_uri -+ with - | Error e -> - [%log fatal] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/replayer/sql.ml b/src/app/replayer/sql.ml -index 40caaf5af1..54c32b7274 100644 ---- a/src/app/replayer/sql.ml -+++ b/src/app/replayer/sql.ml -@@ -18,8 +18,8 @@ module Block_info = struct - - (* find all blocks, working back from block with given state hash *) - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - typ - {sql| WITH RECURSIVE chain AS ( - SELECT id,parent_id,global_slot_since_genesis,state_hash,ledger_hash, snarked_ledger_hash_id FROM blocks b WHERE b.state_hash = $1 -@@ -71,7 +71,7 @@ let find_command_ids_query s = - - module Block = struct - let state_hash_query = -- Caqti_request.find Caqti_type.int Caqti_type.string -+ Mina_caqti.find_req Caqti_type.int Caqti_type.string - {sql| SELECT state_hash FROM blocks - WHERE id = ? - |sql} -@@ -80,7 +80,7 @@ module Block = struct - Conn.find state_hash_query id - - let parent_id_query = -- Caqti_request.find Caqti_type.int Caqti_type.int -+ Mina_caqti.find_req Caqti_type.int Caqti_type.int - {sql| SELECT parent_id FROM blocks - WHERE id = ? - |sql} -@@ -89,7 +89,7 @@ module Block = struct - Conn.find parent_id_query id - - let unparented_query = -- Caqti_request.collect Caqti_type.unit Caqti_type.int -+ Mina_caqti.collect_req Caqti_type.unit Caqti_type.int - {sql| SELECT id FROM blocks - WHERE parent_id IS NULL - |sql} -@@ -98,21 +98,21 @@ module Block = struct - Conn.collect_list unparented_query () - - let get_height_query = -- Caqti_request.find Caqti_type.int Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.int Caqti_type.int64 - {sql| SELECT height FROM blocks WHERE id = $1 |sql} - - let get_height (module Conn : Caqti_async.CONNECTION) ~block_id = - Conn.find get_height_query block_id - - let max_slot_query = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT MAX(global_slot_since_genesis) FROM blocks |sql} - - let get_max_slot (module Conn : Caqti_async.CONNECTION) () = - Conn.find max_slot_query () - - let max_canonical_slot_query = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT MAX(global_slot_since_genesis) FROM blocks - WHERE chain_status = 'canonical' - |sql} -@@ -121,7 +121,7 @@ module Block = struct - Conn.find max_canonical_slot_query () - - let next_slot_query = -- Caqti_request.find_opt Caqti_type.int64 Caqti_type.int64 -+ Mina_caqti.find_opt_req Caqti_type.int64 Caqti_type.int64 - {sql| SELECT global_slot_since_genesis FROM blocks - WHERE global_slot_since_genesis >= $1 - AND chain_status <> 'orphaned' -@@ -133,7 +133,7 @@ module Block = struct - Conn.find_opt next_slot_query slot - - let state_hashes_by_slot_query = -- Caqti_request.collect Caqti_type.int64 Caqti_type.string -+ Mina_caqti.collect_req Caqti_type.int64 Caqti_type.string - {sql| SELECT state_hash FROM blocks WHERE global_slot_since_genesis = $1 |sql} - - let get_state_hashes_by_slot (module Conn : Caqti_async.CONNECTION) slot = -@@ -141,7 +141,7 @@ module Block = struct - - (* find all blocks, working back from block with given state hash *) - let chain_query = -- Caqti_request.collect Caqti_type.string Caqti_type.string -+ Mina_caqti.collect_req Caqti_type.string Caqti_type.string - {sql| WITH RECURSIVE chain AS ( - - SELECT id,parent_id FROM blocks b WHERE b.state_hash = ? -@@ -171,7 +171,7 @@ module Block = struct - let pk, _ = Mina_state.Consensus_state_hooks.genesis_winner in - Signature_lib.Public_key.Compressed.to_base58_check pk - in -- Caqti_request.find Caqti_type.int64 Caqti_type.string -+ Mina_caqti.find_req Caqti_type.int64 Caqti_type.string - (sprintf - {sql| SELECT snarked_ledger_hashes.value - FROM blocks -@@ -192,8 +192,8 @@ end - - module User_command_ids = struct - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - Caqti_type.int - (find_command_ids_query "user") - -@@ -240,7 +240,7 @@ module User_command = struct - ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT command_type,fee_payer_id, source_id,receiver_id,fee,amount,valid_until,memo,nonce, - blocks.id,blocks.height,blocks.global_slot_since_genesis, - sequence_no,status -@@ -265,8 +265,8 @@ end - - module Zkapp_command_ids = struct - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - Caqti_type.int - (find_command_ids_query "zkapp") - -@@ -292,7 +292,7 @@ module Zkapp_command = struct - [ int; Mina_caqti.array_int_typ; string; int; int64; int; string ] - - let query = -- Caqti_request.collect Caqti_type.int typ -+ Mina_caqti.collect_req Caqti_type.int typ - {sql| SELECT zkapp_fee_payer_body_id,zkapp_account_updates_ids,memo, - blocks.id,blocks.global_slot_since_genesis, - sequence_no,hash -@@ -317,8 +317,8 @@ end - - module Internal_command_ids = struct - let query = -- Caqti_request.collect -- Caqti_type.(tup2 string int64) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int64) - Caqti_type.int - (find_command_ids_query "internal") - -@@ -347,8 +347,8 @@ module Internal_command = struct - the call to Staged_ledger.apply in Block_producer - *) - let query = -- Caqti_request.collect -- Caqti_type.(tup2 int64 int) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 int64 int) - typ - {sql| SELECT command_type,receiver_id,fee, - b.id,b.height,b.global_slot_since_genesis, -@@ -377,7 +377,7 @@ end - - module Public_key = struct - let query = -- Caqti_request.find_opt Caqti_type.int Caqti_type.string -+ Mina_caqti.find_opt_req Caqti_type.int Caqti_type.string - {sql| SELECT value FROM public_keys - WHERE id = ? - |sql} -@@ -388,7 +388,7 @@ end - - module Snarked_ledger_hashes = struct - let query = -- Caqti_request.find Caqti_type.int Caqti_type.string -+ Mina_caqti.find_req Caqti_type.int Caqti_type.string - {sql| SELECT value FROM snarked_ledger_hashes - WHERE id = ? - |sql} -@@ -404,11 +404,11 @@ module Epoch_data = struct - let decode (epoch_ledger_hash, epoch_data_seed) = - Ok { epoch_ledger_hash; epoch_data_seed } - in -- let rep = Caqti_type.(tup2 string string) in -+ let rep = Caqti_type.(t2 string string) in - Caqti_type.custom ~encode ~decode rep - - let query_epoch_data = -- Caqti_request.find Caqti_type.int epoch_data_typ -+ Mina_caqti.find_req Caqti_type.int epoch_data_typ - {sql| SELECT slh.value, ed.seed FROM snarked_ledger_hashes AS slh - - INNER JOIN -@@ -425,7 +425,7 @@ module Epoch_data = struct - Conn.find query_epoch_data epoch_ledger_id - - let query_staking_epoch_data_id = -- Caqti_request.find Caqti_type.string Caqti_type.int -+ Mina_caqti.find_req Caqti_type.string Caqti_type.int - {sql| SELECT staking_epoch_data_id FROM blocks - - WHERE state_hash = ? -@@ -437,7 +437,7 @@ module Epoch_data = struct - Conn.find query_staking_epoch_data_id state_hash - - let query_next_epoch_data_id = -- Caqti_request.find Caqti_type.string Caqti_type.int -+ Mina_caqti.find_req Caqti_type.string Caqti_type.int - {sql| SELECT next_epoch_data_id FROM blocks - - WHERE state_hash = ? -@@ -450,7 +450,7 @@ end - module Parent_block = struct - (* fork block is parent of block with the given state hash *) - let query_parent_state_hash = -- Caqti_request.find Caqti_type.string Caqti_type.string -+ Mina_caqti.find_req Caqti_type.string Caqti_type.string - {sql| SELECT parent.state_hash FROM blocks AS parent - - INNER JOIN -diff --git a/src/app/rosetta/lib/account.ml b/src/app/rosetta/lib/account.ml -index 0b2265c206..d9243fbecb 100644 ---- a/src/app/rosetta/lib/account.ml -+++ b/src/app/rosetta/lib/account.ml -@@ -36,9 +36,9 @@ module Sql = struct - .transaction_capacity_log_2 - - let query_pending = -- Caqti_request.find_opt -- Caqti_type.(tup3 string int64 string) -- Caqti_type.(tup2 (tup4 int64 int64 int64 int64) int) -+ Mina_caqti.find_opt_req -+ Caqti_type.(t3 string int64 string) -+ Caqti_type.(t2 (t4 int64 int64 int64 int64) int) - {sql| - WITH RECURSIVE pending_chain AS ( - -@@ -87,9 +87,9 @@ module Sql = struct - |sql} - - let query_canonical = -- Caqti_request.find_opt -- Caqti_type.(tup3 string int64 string) -- Caqti_type.(tup2 (tup4 int64 int64 int64 int64) int) -+ Mina_caqti.find_opt_req -+ Caqti_type.(t3 string int64 string) -+ Caqti_type.(t2 (t4 int64 int64 int64 int64) int) - {sql| - SELECT b.height,b.global_slot_since_genesis AS block_global_slot_since_genesis,balance,nonce,timing_id - -diff --git a/src/app/rosetta/lib/block.ml b/src/app/rosetta/lib/block.ml -index b6cf0ea4d0..b6108d049a 100644 ---- a/src/app/rosetta/lib/block.ml -+++ b/src/app/rosetta/lib/block.ml -@@ -104,7 +104,7 @@ module Sql = struct - String.concat ~sep:"," fields - - let query_count_canonical_at_height = -- Caqti_request.find Caqti_type.int64 Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.int64 Caqti_type.int64 - {sql| SELECT COUNT(*) FROM blocks - WHERE height = ? - AND chain_status = 'canonical' -@@ -112,7 +112,7 @@ module Sql = struct - - let query_height_canonical = - let c_fields = block_fields ~prefix:"c." () in -- Caqti_request.find_opt Caqti_type.int64 typ -+ Mina_caqti.find_opt_req Caqti_type.int64 typ - (* The archive database will only reconcile the canonical columns for - * blocks older than k + epsilon - *) -@@ -135,7 +135,7 @@ module Sql = struct - let fields = block_fields () in - let b_fields = block_fields ~prefix:"b." () in - let c_fields = block_fields ~prefix:"c." () in -- Caqti_request.find_opt Caqti_type.int64 typ -+ Mina_caqti.find_opt_req Caqti_type.int64 typ - (* According to the clarification of the Rosetta spec here - * https://community.rosetta-api.org/t/querying-block-by-just-its-index/84/3 , - * it is important to select only the block on the canonical chain for a -@@ -180,7 +180,7 @@ module Sql = struct - - let query_hash = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt Caqti_type.string typ -+ Mina_caqti.find_opt_req Caqti_type.string typ - [%string - {| - SELECT b.id, -@@ -197,8 +197,8 @@ module Sql = struct - - let query_both = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt -- Caqti_type.(tup2 string int64) -+ Mina_caqti.find_opt_req -+ Caqti_type.(t2 string int64) - typ - [%string - {| -@@ -217,7 +217,7 @@ module Sql = struct - - let query_by_id = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt Caqti_type.int typ -+ Mina_caqti.find_opt_req Caqti_type.int typ - [%string - {| - SELECT b.id, -@@ -234,7 +234,7 @@ module Sql = struct - - let query_best = - let b_fields = block_fields ~prefix:"b." () in -- Caqti_request.find_opt Caqti_type.unit typ -+ Mina_caqti.find_opt_req Caqti_type.unit typ - [%string - {| - SELECT b.id, -@@ -272,7 +272,7 @@ module Sql = struct - else - let%bind max_height = - Conn.find -- (Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ (Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT MAX(height) FROM blocks |sql} ) - () - in -@@ -323,8 +323,7 @@ module Sql = struct - - let typ = - Caqti_type.( -- tup3 int Archive_lib.Processor.User_command.Signed_command.typ -- Extras.typ) -+ t3 int Archive_lib.Processor.User_command.Signed_command.typ Extras.typ) - - let fields = - String.concat ~sep:"," -@@ -333,8 +332,8 @@ module Sql = struct - Archive_lib.Processor.User_command.Signed_command.Fields.names - - let query = -- Caqti_request.collect -- Caqti_type.(tup2 int string) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 int string) - typ - [%string - {| -@@ -541,7 +540,7 @@ module Sql = struct - - let run (module Conn : Caqti_async.CONNECTION) id = - Conn.collect_list -- (Caqti_request.collect Caqti_type.(tup2 int string) typ query) -+ (Mina_caqti.collect_req Caqti_type.(t2 int string) typ query) - (id, Mina_base.Token_id.(to_string default)) - - let to_info t = -@@ -682,7 +681,7 @@ module Sql = struct - |}] - - let query = -- Caqti_request.collect Caqti_type.(tup2 int string) typ query_string -+ Mina_caqti.collect_req Caqti_type.(t2 int string) typ query_string - - let run (module Conn : Caqti_async.CONNECTION) id = - Conn.collect_list query (id, Mina_base.Token_id.(to_string default)) -diff --git a/src/app/rosetta/lib/construction.ml b/src/app/rosetta/lib/construction.ml -index 90297f8b17..753f4e1f88 100644 ---- a/src/app/rosetta/lib/construction.ml -+++ b/src/app/rosetta/lib/construction.ml -@@ -866,7 +866,7 @@ module Submit = struct - Caqti_type.custom ~encode ~decode (to_rep spec) - - let query = -- Caqti_request.find_opt params_typ Caqti_type.string -+ Mina_caqti.find_opt_req params_typ Caqti_type.string - {sql| SELECT uc.id FROM user_commands uc - INNER JOIN public_keys AS pks ON pks.id = uc.source_id - INNER JOIN public_keys AS pkr ON pkr.id = uc.receiver_id -diff --git a/src/app/rosetta/lib/network.ml b/src/app/rosetta/lib/network.ml -index 420d3569ac..f7a5a8a247 100644 ---- a/src/app/rosetta/lib/network.ml -+++ b/src/app/rosetta/lib/network.ml -@@ -75,8 +75,8 @@ end - - module Sql = struct - let oldest_block_query = -- Caqti_request.find Caqti_type.unit -- Caqti_type.(tup2 int64 string) -+ Mina_caqti.find_req Caqti_type.unit -+ Caqti_type.(t2 int64 string) - "SELECT height, state_hash FROM blocks ORDER BY timestamp ASC, \ - state_hash ASC LIMIT 1" - -@@ -88,8 +88,8 @@ module Sql = struct - 0L - - let latest_block_query = -- Caqti_request.find Caqti_type.unit -- Caqti_type.(tup3 int64 string int64) -+ Mina_caqti.find_req Caqti_type.unit -+ Caqti_type.(t3 int64 string int64) - (sprintf - {sql| SELECT height, state_hash, timestamp FROM blocks b - WHERE height = (select MAX(height) - %Ld FROM blocks) -diff --git a/src/app/rosetta/lib/pg_data.ml b/src/app/rosetta/lib/pg_data.ml -index 15065b1fa3..699b661aa4 100644 ---- a/src/app/rosetta/lib/pg_data.ml -+++ b/src/app/rosetta/lib/pg_data.ml -@@ -1,7 +1,7 @@ - (* pg_data.ml -- Postgres data *) - - let query_connection_count = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT count(*) FROM pg_stat_activity - WHERE state = 'active' - |sql} -@@ -10,7 +10,7 @@ let run_connection_count (module Conn : Caqti_async.CONNECTION) = - Conn.find query_connection_count - - let query_lock_count = -- Caqti_request.find Caqti_type.unit Caqti_type.int64 -+ Mina_caqti.find_req Caqti_type.unit Caqti_type.int64 - {sql| SELECT count(*) FROM pg_locks - WHERE mode = 'SIReadLock' - |sql} -diff --git a/src/app/rosetta/lib/rosetta.ml b/src/app/rosetta/lib/rosetta.ml -index 135235db65..07b14fb095 100644 ---- a/src/app/rosetta/lib/rosetta.ml -+++ b/src/app/rosetta/lib/rosetta.ml -@@ -190,7 +190,14 @@ let command = - "MINA_ROSETTA_MAX_DB_POOL_SIZE not set or invalid. Please set \ - this to a number (try 64 or 128)" - in -- match Caqti_async.connect_pool ~max_size:max_pool_size archive_uri with -+ match -+ Caqti_async.connect_pool -+ ~pool_config: -+ Caqti_pool_config.( -+ merge_left (default_from_env ()) -+ (create ~max_size:max_pool_size ())) -+ archive_uri -+ with - | Error e -> - [%log error] - ~metadata:[ ("error", `String (Caqti_error.show e)) ] -diff --git a/src/app/rosetta/lib/search.ml b/src/app/rosetta/lib/search.ml -index a4d16f4921..31e44a46aa 100644 ---- a/src/app/rosetta/lib/search.ml -+++ b/src/app/rosetta/lib/search.ml -@@ -418,8 +418,8 @@ module Sql = struct - in - let offset = offset_sql offset in - let limit = limit_sql limit in -- Caqti_request.collect Params.typ -- Caqti_type.(tup2 int64 typ) -+ Mina_caqti.collect_req Params.typ -+ Caqti_type.(t2 int64 typ) - [%string - {sql| - WITH filtered_ids AS ( -@@ -783,7 +783,7 @@ module Sql = struct - in - match%map - Conn.collect_list -- (Caqti_request.collect Params.typ Caqti_type.(tup2 int64 typ) query) -+ (Mina_caqti.collect_req Params.typ Caqti_type.(t2 int64 typ) query) - params - with - | [] -> -@@ -939,7 +939,7 @@ module Sql = struct - ~address_fields:[ "pk_fee_payer.value"; "pk_update_body.value" ] - ~op_type_filters operator - in -- Caqti_request.collect Params.typ Caqti_type.(tup2 int64 typ) -+ Mina_caqti.collect_req Params.typ Caqti_type.(t2 int64 typ) - @@ query_string ~offset ~limit ~filters - - let run (module Conn : Caqti_async.CONNECTION) ~offset ~limit input = -diff --git a/src/app/swap_bad_balances/sql.ml b/src/app/swap_bad_balances/sql.ml -index 82ec136a10..d8d6172857 100644 ---- a/src/app/swap_bad_balances/sql.ml -+++ b/src/app/swap_bad_balances/sql.ml -@@ -5,8 +5,8 @@ open Async - module Receiver_balances = struct - (* find receiver balances for combined fee transfer *) - let query_ids_from_fee_transfer = -- Caqti_request.collect -- Caqti_type.(tup2 string int) -+ Mina_caqti.collect_req -+ Caqti_type.(t2 string int) - Caqti_type.(int) - {sql| SELECT bic.receiver_balance - FROM blocks_internal_commands bic -@@ -25,8 +25,8 @@ module Receiver_balances = struct - (* if duplicates, any is acceptable *) - match%bind - Conn.find_opt -- (Caqti_request.find_opt -- Caqti_type.(tup2 int int64) -+ (Mina_caqti.find_opt_req -+ Caqti_type.(t2 int int64) - Caqti_type.int - {sql| SELECT id - FROM balances -@@ -40,8 +40,8 @@ module Receiver_balances = struct - return id - | None -> - Conn.find -- (Caqti_request.find -- Caqti_type.(tup2 int int64) -+ (Mina_caqti.find_req -+ Caqti_type.(t2 int int64) - Caqti_type.int - "INSERT INTO balances (public_key_id,balance) VALUES ($1,$2) \ - RETURNING id" ) -@@ -49,9 +49,9 @@ module Receiver_balances = struct - - let load (module Conn : Caqti_async.CONNECTION) id = - Conn.find -- (Caqti_request.find -+ (Mina_caqti.find_req - Caqti_type.(int) -- Caqti_type.(tup2 int int64) -+ Caqti_type.(t2 int int64) - {sql| SELECT public_key_id,balance - FROM balances - WHERE id = $1 -@@ -59,8 +59,8 @@ module Receiver_balances = struct - id - - let query_swap_in_new_balance = -- Caqti_request.exec -- Caqti_type.(tup4 string int int int) -+ Mina_caqti.exec_req -+ Caqti_type.(t4 string int int int) - {sql| UPDATE blocks_internal_commands bic SET receiver_balance = $4 - FROM blocks b - WHERE b.id = bic.block_id -diff --git a/src/lib/mina_caqti/mina_caqti.ml b/src/lib/mina_caqti/mina_caqti.ml -index b1e49d9bcb..7dee307b08 100644 ---- a/src/lib/mina_caqti/mina_caqti.ml -+++ b/src/lib/mina_caqti/mina_caqti.ml -@@ -5,15 +5,6 @@ open Core_kernel - open Caqti_async - open Mina_base - --type _ Caqti_type.field += -- | Array_nullable_int : int option array Caqti_type.field -- --type _ Caqti_type.field += -- | Array_nullable_int64 : int64 option array Caqti_type.field -- --type _ Caqti_type.field += -- | Array_nullable_string : string option array Caqti_type.field -- - module Type_spec = struct - type (_, _) t = - | [] : (unit, unit) t -@@ -25,7 +16,7 @@ module Type_spec = struct - | [] -> - (Caqti_type.unit : tuple Caqti_type.t) - | rep :: spec -> -- Caqti_type.tup2 rep (to_rep spec) -+ Caqti_type.t2 rep (to_rep spec) - - let rec hlist_to_tuple : - 'hlist 'tuple. -@@ -93,59 +84,14 @@ let make_coding (type a) ~(elem_to_string : a -> string) - in - (encode, decode) - --(* register coding for nullable int arrays *) --let () = -- let open Caqti_type.Field in -- let rep = Caqti_type.String in -+(** this type may require type annotations in queries, eg. -+ `SELECT id FROM zkapp_states WHERE element_ids = ?::int[]` -+*) -+let array_nullable_int_typ = - let encode, decode = - make_coding ~elem_to_string:Int.to_string ~elem_of_string:Int.of_string - in -- let get_coding : type a. _ -> a t -> a coding = -- fun _ -> function -- | Array_nullable_int -> -- Coding { rep; encode; decode } -- | _ -> -- assert false -- in -- define_coding Array_nullable_int { get_coding } -- --(* register coding for nullable int64 arrays *) --let () = -- let open Caqti_type.Field in -- let rep = Caqti_type.String in -- let encode, decode = -- make_coding ~elem_to_string:Int64.to_string ~elem_of_string:Int64.of_string -- in -- let get_coding : type a. _ -> a t -> a coding = -- fun _ -> function -- | Array_nullable_int64 -> -- Coding { rep; encode; decode } -- | _ -> -- assert false -- in -- define_coding Array_nullable_int64 { get_coding } -- --(* register coding for nullable string arrays *) --let () = -- let open Caqti_type.Field in -- let rep = Caqti_type.String in -- let encode, decode = -- make_coding ~elem_to_string:Fn.id ~elem_of_string:Fn.id -- in -- let get_coding : type a. _ -> a t -> a coding = -- fun _ -> function -- | Array_nullable_string -> -- Coding { rep; encode; decode } -- | _ -> -- assert false -- in -- define_coding Array_nullable_string { get_coding } -- --(* this type may require type annotations in queries, eg. -- `SELECT id FROM zkapp_states WHERE element_ids = ?::int[]` --*) --let array_nullable_int_typ : int option array Caqti_type.t = -- Caqti_type.field Array_nullable_int -+ Caqti_type.custom ~encode ~decode Caqti_type.string - - let array_int_typ : int array Caqti_type.t = - let open Result.Let_syntax in -@@ -158,11 +104,14 @@ let array_int_typ : int array Caqti_type.t = - in - Caqti_type.custom array_nullable_int_typ ~encode ~decode - --(* this type may require type annotations in queries, eg. -+(** this type may require type annotations in queries, eg. - `SELECT id FROM zkapp_states WHERE element_ids = ?::bigint[]` - *) --let array_nullable_int64_typ : int64 option array Caqti_type.t = -- Caqti_type.field Array_nullable_int64 -+let array_nullable_int64_typ = -+ let encode, decode = -+ make_coding ~elem_to_string:Int64.to_string ~elem_of_string:Int64.of_string -+ in -+ Caqti_type.custom ~encode ~decode Caqti_type.string - - let array_int64_typ : int64 array Caqti_type.t = - let open Result.Let_syntax in -@@ -175,11 +124,14 @@ let array_int64_typ : int64 array Caqti_type.t = - in - Caqti_type.custom array_nullable_int64_typ ~encode ~decode - --(* this type may require type annotations in queries, e.g. -+(** this type may require type annotations in queries, e.g. - `SELECT id FROM zkapp_states WHERE element_ids = ?::string[]` - *) --let array_nullable_string_typ : string option array Caqti_type.t = -- Caqti_type.field Array_nullable_string -+let array_nullable_string_typ = -+ let encode, decode = -+ make_coding ~elem_to_string:Fn.id ~elem_of_string:Fn.id -+ in -+ Caqti_type.custom ~encode ~decode Caqti_type.string - - let array_string_typ : string array Caqti_type.t = - let open Result.Let_syntax in -@@ -283,7 +235,7 @@ let select_insert_into_cols ~(select : string * 'select Caqti_type.t) - (module Conn : CONNECTION) (value : 'cols) = - let open Deferred.Result.Let_syntax in - Conn.find_opt -- ( Caqti_request.find_opt (snd cols) (snd select) -+ ( Caqti_request.Infix.(snd cols ->? snd select) - @@ select_cols ~select:(fst select) ~table_name ?tannot ~cols:(fst cols) () - ) - value -@@ -292,7 +244,7 @@ let select_insert_into_cols ~(select : string * 'select Caqti_type.t) - return id - | None -> - Conn.find -- ( Caqti_request.find (snd cols) (snd select) -+ ( Caqti_request.Infix.(snd cols ->! snd select) - @@ insert_into_cols ~returning:(fst select) ~table_name ?tannot - ~cols:(fst cols) () ) - value -@@ -314,7 +266,11 @@ let insert_multi_into_col ~(table_name : string) - (sep_by_comma ~parenthesis:true values) - (fst col) - in -- let%bind () = Conn.exec (Caqti_request.exec Caqti_type.unit insert) () in -+ let%bind () = -+ Conn.exec -+ (Caqti_request.Infix.(Caqti_type.unit ->. Caqti_type.unit) insert) -+ () -+ in - let search = - sprintf - {sql| SELECT %s, id FROM %s -@@ -322,9 +278,8 @@ let insert_multi_into_col ~(table_name : string) - (fst col) table_name (fst col) (sep_by_comma values) - in - Conn.collect_list -- (Caqti_request.collect Caqti_type.unit -- Caqti_type.(tup2 (snd col) int) -- search ) -+ Caqti_request.Infix.( -+ (Caqti_type.unit ->* Caqti_type.(t2 (snd col) int)) search) - () - - let query ~f pool = -@@ -361,3 +316,11 @@ let get_zkapp_or_ignore (item_opt : 'arg option) - let get_opt_item (arg_opt : 'arg option) - ~(f : 'arg -> ('res, _) Deferred.Result.t) : 'res option Deferred.t = - make_get_opt ~of_option:Fn.id ~f arg_opt -+ -+let find_req t u s = Caqti_request.Infix.(t ->! u) s -+ -+let find_opt_req t u s = Caqti_request.Infix.(t ->? u) s -+ -+let collect_req t u s = Caqti_request.Infix.(t ->* u) s -+ -+let exec_req t s = Caqti_request.Infix.(t ->. Caqti_type.unit) s From 5e97dc699bdbb9e87ea1b9ed9f3ae75e7654afe1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 30 Jul 2024 11:56:25 +0200 Subject: [PATCH 007/178] add spaces before and after logo --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ee70b7c60c6..b7c73df3815 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,13 @@ | ------- | ---------- | ---------- | | [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) -
-

Mina logo - +
Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy! From af365cc42891818555399c2b10b04333d3a158cf Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 2 Aug 2024 15:33:56 +0200 Subject: [PATCH 008/178] format --- buildkite/src/Command/MinaArtifact.dhall | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index a7ceb81781e..59a7bbb5876 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -146,11 +146,7 @@ let docker_step } ) networks - - - , TestExecutive = - []: List DockerImage.ReleaseSpec.Type - + , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps From 5530a83c2bbc09334a9a9d0059fba2f409beb6fd Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 12 Aug 2024 19:42:52 -0400 Subject: [PATCH 009/178] swapping out the s3 bucket link for snark-keys --- buildkite/scripts/build-hardfork-package.sh | 6 +++--- scripts/mina-verify-packaged-fork-config | 3 +-- src/lib/cache_dir/fake/cache_dir.ml | 2 +- src/lib/cache_dir/native/cache_dir.ml | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/buildkite/scripts/build-hardfork-package.sh b/buildkite/scripts/build-hardfork-package.sh index 6a83ea82f5a..fc6d0396599 100755 --- a/buildkite/scripts/build-hardfork-package.sh +++ b/buildkite/scripts/build-hardfork-package.sh @@ -58,18 +58,18 @@ _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --confi echo "--- Create hardfork config" FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json -existing_files=$(aws s3 ls s3://snark-keys.o1test.net/ | awk '{print $4}') +existing_files=$(aws s3 ls s3://snark-keys-ro.o1test.net/ | awk '{print $4}') for file in hardfork_ledgers/*; do filename=$(basename "$file") if echo "$existing_files" | grep -q "$filename"; then echo "Info: $filename already exists in the bucket, packaging it instead." oldhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}') - aws s3 cp "s3://snark-keys.o1test.net/$filename" "$file" + aws s3 cp "s3://snark-keys-ro.o1test.net/$filename" "$file" newhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}') sed -i "s/$oldhash/$newhash/g" new_config.json else - aws s3 cp --acl public-read "$file" s3://snark-keys.o1test.net/ + aws s3 cp --acl public-read "$file" s3://snark-keys-ro.o1test.net/ fi done diff --git a/scripts/mina-verify-packaged-fork-config b/scripts/mina-verify-packaged-fork-config index b19eaf560b0..fe1bf79a904 100755 --- a/scripts/mina-verify-packaged-fork-config +++ b/scripts/mina-verify-packaged-fork-config @@ -212,8 +212,7 @@ for file in "$workdir"/ledgers/*.tar.gz; do mkdir -p "$tardir"/{packaged,generated,web} tar -xzf "$file" -C "$tardir/generated" tar -xzf "$GENESIS_LEDGER_DIR/$tarname.tar.gz" -C "$tardir/packaged" - curl "https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net/$tarname.tar.gz" | tar -xz -C "$tardir/web" - + curl "https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net/$tarname.tar.gz" | tar -xz -C "$tardir/web" $ldb_cmd --hex --db="$tardir/packaged" scan > "$workdir/packaged.scan" $ldb_cmd --hex --db="$tardir/web" scan > "$workdir/web.scan" $ldb_cmd --hex --db="$tardir/generated" scan > "$workdir/generated.scan" diff --git a/src/lib/cache_dir/fake/cache_dir.ml b/src/lib/cache_dir/fake/cache_dir.ml index fe607a422e7..96c21eb6ff9 100644 --- a/src/lib/cache_dir/fake/cache_dir.ml +++ b/src/lib/cache_dir/fake/cache_dir.ml @@ -6,7 +6,7 @@ let autogen_path = "/tmp/coda_cache_dir" let s3_install_path = "/tmp/s3_cache_dir" let s3_keys_bucket_prefix = - "https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net" + "https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net" let manual_install_path = "/var/lib/coda" diff --git a/src/lib/cache_dir/native/cache_dir.ml b/src/lib/cache_dir/native/cache_dir.ml index dd8f21a49eb..8450d40bfd7 100644 --- a/src/lib/cache_dir/native/cache_dir.ml +++ b/src/lib/cache_dir/native/cache_dir.ml @@ -8,7 +8,7 @@ let s3_install_path = "/tmp/s3_cache_dir" let s3_keys_bucket_prefix = Option.value (Sys.getenv "MINA_LEDGER_S3_BUCKET") - ~default:"https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net" + ~default:"https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net" let manual_install_path = "/var/lib/coda" From 13122000e4becebe2b948a2f723eba1c5eb3149b Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 12 Aug 2024 20:09:41 -0400 Subject: [PATCH 010/178] removing berkeley connection test in buildkite --- buildkite/scripts/build-artifact.sh | 2 +- buildkite/scripts/connect-to-berkeley.sh | 68 ------------------- buildkite/scripts/connect-to-testnet.sh | 5 +- .../src/Jobs/Test/BerkeleyCompatibility.dhall | 48 ------------- .../src/Jobs/Test/ConnectToBerkeley.dhall | 46 ------------- 5 files changed, 3 insertions(+), 166 deletions(-) delete mode 100755 buildkite/scripts/connect-to-berkeley.sh delete mode 100644 buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall delete mode 100644 buildkite/src/Jobs/Test/ConnectToBerkeley.dhall diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 0a6aa47ae29..a58f39d910c 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -24,7 +24,7 @@ make -C src/app/libp2p_helper MAINNET_TARGETS="" [[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe" -echo "--- Build all major tagets required for packaging" +echo "--- Build all major targets required for packaging" echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" echo "Rust Version: $(rustc --version)" dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \ diff --git a/buildkite/scripts/connect-to-berkeley.sh b/buildkite/scripts/connect-to-berkeley.sh deleted file mode 100755 index 1b2aa7cfe82..00000000000 --- a/buildkite/scripts/connect-to-berkeley.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - -# Don't prompt for answers during apt-get install -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl - -TESTNET_VERSION_NAME="berkeley" -TESTNET_NAME="testworld-2-0" - -git config --global --add safe.directory /workdir - -source buildkite/scripts/export-git-env-vars.sh - -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" - -# Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: - -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key -# Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ - -# Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background - -# Attempt to connect to the GraphQL client every 30s for up to 12 minutes -num_status_retries=24 -for ((i=1;i<=$num_status_retries;i++)); do - sleep 30s - set +e - mina client status - status_exit_code=$? - set -e - if [ $status_exit_code -eq 0 ]; then - break - elif [ $i -eq $num_status_retries ]; then - exit $status_exit_code - fi -done - -# Check that the daemon has connected to peers and is still up after 2 mins -sleep 2m -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi - diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 3cfd7a4aa7b..57fc9688383 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -7,7 +7,6 @@ if [[ $# -ne 3 ]]; then exit 1 fi -TESTNET_VERSION_NAME="berkeley" TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 @@ -24,13 +23,13 @@ esac export DEBIAN_FRONTEND=noninteractive apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl +apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" +source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1 # Remove lockfile if present rm ~/.mina-config/.mina-lock ||: diff --git a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall b/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall deleted file mode 100644 index b44d2b6ea96..00000000000 --- a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Command = ../../Command/Base.dhall - -let Docker = ../../Command/Docker/Type.dhall - -let Size = ../../Command/Size.dhall - -let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/check-compatibility" "sh" - , S.exactly "buildkite/src/Jobs/Test/BerkeleyCompatibility" "dhall" - ] - , path = "Test" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - , name = "BerkeleyCompatibility" - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run "buildkite/scripts/check-compatibility.sh berkeley" ] - , label = "Test: berkeley compatibilty test" - , key = "berkeley-compatibilty-test" - , target = Size.XLarge - , docker = None Docker.Type - , depends_on = dependsOn - , timeout_in_minutes = Some +60 - } - ] - } diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall deleted file mode 100644 index 2396f299df0..00000000000 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ /dev/null @@ -1,46 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let B = ../../External/Buildkite.dhall - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let Dockers = ../../Constants/DockerVersions.dhall - -let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - Profiles.Type.Standard - "daemon-berkeley" - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" - , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" - ] - , path = "Test" - , name = "ConnectToBerkeley" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - } - , steps = - [ ConnectToTestnet.step - dependsOn - "berkeley" - "40s" - "2m" - (B/SoftFail.Boolean True) - ] - } From e9bf6d99b4744de326f06f42dbf1de5fa845e397 Mon Sep 17 00:00:00 2001 From: sai Date: Wed, 14 Aug 2024 18:48:48 -0400 Subject: [PATCH 011/178] first pass at changin ci deps from #15731 --- automation/scripts/gen-keys-ledger.sh | 2 +- buildkite/scripts/build-release.sh | 2 +- buildkite/scripts/connect-to-testnet.sh | 51 +++++++++++-------- buildkite/scripts/debian/install.sh | 2 +- buildkite/scripts/dump-mina-type-shapes.sh | 2 +- .../scripts/rosetta-integration-tests.sh | 6 ++- .../scripts/run-snark-transaction-profiler.sh | 2 +- buildkite/scripts/run-test-executive-cloud.sh | 2 +- buildkite/scripts/run-test-executive-local.sh | 4 +- buildkite/scripts/run_promote_build_job.sh | 2 +- .../scripts/run_verify_promoted_build_job.sh | 2 +- buildkite/scripts/single-node-tests.sh | 2 +- buildkite/scripts/version-linter.sh | 2 +- buildkite/src/Command/ConnectToTestnet.dhall | 10 ++-- buildkite/src/Command/MinaArtifact.dhall | 8 +-- buildkite/src/Command/PromotePackage.dhall | 4 +- .../Release/ItnOrchestratorArtifact.dhall | 2 +- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 45 ++++++++++++++++ .../Jobs/Test/TestnetIntegrationTests.dhall | 2 +- .../Test/TestnetIntegrationTestsLong.dhall | 2 +- scripts/deb-builder-helpers.sh | 27 +++------- scripts/rebuild-deb.sh | 1 - 22 files changed, 109 insertions(+), 73 deletions(-) create mode 100644 buildkite/src/Jobs/Test/ConnectToDevnet.dhall diff --git a/automation/scripts/gen-keys-ledger.sh b/automation/scripts/gen-keys-ledger.sh index 2ed5edcc9d9..33c881516fc 100755 --- a/automation/scripts/gen-keys-ledger.sh +++ b/automation/scripts/gen-keys-ledger.sh @@ -1,5 +1,5 @@ #! /bin/bash -MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-berkeley" +MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-devnet" while [ $# -gt 0 ]; do case "$1" in diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index f1242fb94f5..6bf49f48231 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -9,7 +9,7 @@ source ~/.profile ./buildkite/scripts/build-artifact.sh echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" -echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" +echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 57fc9688383..45f26a8f742 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -22,8 +22,8 @@ esac # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq +sudo apt-get update +sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq git config --global --add safe.directory /workdir @@ -32,27 +32,29 @@ source buildkite/scripts/export-git-env-vars.sh source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1 # Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: +sudo rm ~/.mina-config/.mina-lock ||: + +sudo mkdir -p /root/libp2p-keys/ -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ +sudo chmod -R 0700 /root/libp2p-keys/ +# Pre-generated random password for this quick test +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \ + TESTNET_NAME=$TESTNET_NAME \ + bash -c "mina daemon \ + --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \ + --libp2p-keypair \"/root/libp2p-keys/key\" \ + --seed &" # -background # Attempt to connect to the GraphQL client every 10s for up to 8 minutes num_status_retries=24 for ((i=1;i<=$num_status_retries;i++)); do sleep $WAIT_BETWEEN_POLLING_GRAPHQL set +e - mina client status + sudo mina client status status_exit_code=$? set -e if [ $status_exit_code -eq 0 ]; then @@ -62,13 +64,18 @@ for ((i=1;i<=$num_status_retries;i++)); do fi done -# Check that the daemon has connected to peers and is still up after 2 mins -sleep $WAIT_AFTER_FINAL_CHECK -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi +peer_retries=10 +for ((i=1;i<=$peer_retries;i++)); do + peer_count=$(sudo mina advanced get-peers | wc -l) + sudo mina client status + + if [ "$peer_count" -gt 0 ]; then + echo "Found some peers" + exit 0; + else + echo "No peers found" + fi + sleep $WAIT_AFTER_FINAL_CHECK +done +exit 1; \ No newline at end of file diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index 93313f4a2dd..bd9a863f8ef 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -34,7 +34,7 @@ else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do case $i in - mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet) + mina-devnet|mina-mainnet|mina-devnet-lightnet) # Downaload mina-logproc too source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER ;; diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh index ff920e7b143..23579a89917 100755 --- a/buildkite/scripts/dump-mina-type-shapes.sh +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index e9219071677..ca976ac17de 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then send_zkapp_transactions & fi +mina client status --json + next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) +sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} +sleep $sleep_time # Mina Rosetta Checks (spec construction data perf) echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" diff --git a/buildkite/scripts/run-snark-transaction-profiler.sh b/buildkite/scripts/run-snark-transaction-profiler.sh index 802cd730632..70c20a92167 100755 --- a/buildkite/scripts/run-snark-transaction-profiler.sh +++ b/buildkite/scripts/run-snark-transaction-profiler.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/run-test-executive-cloud.sh b/buildkite/scripts/run-test-executive-cloud.sh index 884b8222cbc..8a3df280401 100755 --- a/buildkite/scripts/run-test-executive-cloud.sh +++ b/buildkite/scripts/run-test-executive-cloud.sh @@ -2,7 +2,7 @@ set -o pipefail -x TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 7312a19941a..087ad7cc1e2 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -26,7 +26,7 @@ cleanup TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then @@ -40,7 +40,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 26feee3e23a..9191cf5bb84 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index ed093b1a44b..b758f4e661a 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/single-node-tests.sh b/buildkite/scripts/single-node-tests.sh index 93b224c5ba8..dcd6752ae23 100755 --- a/buildkite/scripts/single-node-tests.sh +++ b/buildkite/scripts/single-node-tests.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1 +source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1 export MINA_LIBP2P_PASS="naughty blue worm" export MINA_PRIVKEY_PASS="naughty blue worm" diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index 183a8db9894..5a1862a0e86 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then exit 1 fi -TESTNET_NAME="${TESTNET_NAME:-berkeley}" +TESTNET_NAME="${TESTNET_NAME:-devnet}" # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index ba4bba1b974..e16edfd1d7b 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -4,9 +4,9 @@ let Command = ./Base.dhall let Size = ./Size.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let RunInToolchain = ./RunInToolchain.dhall -let Cmd = ../Lib/Cmds.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type in { step = \(dependsOn : List Command.TaggedKey.Type) @@ -17,10 +17,8 @@ in { step = -> Command.build Command.Config::{ , commands = - [ Cmd.runInDocker - Cmd.Docker::{ - , image = (../Constants/ContainerImages.dhall).ubuntu2004 - } + RunInToolchain.runInToolchain + ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" ] , label = "Connect to ${testnet}" diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index f80115fa1ca..01a4efb78fe 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -52,7 +52,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Berkeley ] + , networks = [ Network.Type.Devnet ] , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -164,7 +164,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-batch-txn" - , network = "berkeley" + , network = "devnet" , deb_codename = "${DebianVersions.lowerName debVersion}" , deb_profile = profile , build_flags = buildFlags @@ -194,7 +194,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-rosetta" - , network = "berkeley" + , network = "devnet" , build_flags = buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = profile @@ -233,7 +233,7 @@ let docker_step debVersion}${Profiles.toLabelSegment profile}${BuildFlags.toLabelSegment buildFlags}--docker-image" - , network = "berkeley" + , network = "devnet" } ] } diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall index b4f0db530df..9303532e9e2 100644 --- a/buildkite/src/Command/PromotePackage.dhall +++ b/buildkite/src/Command/PromotePackage.dhall @@ -56,7 +56,7 @@ let PromoteDebianSpec = , version = "" , new_version = "" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.Nightly @@ -88,7 +88,7 @@ let PromoteDockerSpec = , new_tag = "" , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall index 5c88037f8d1..4b6b0598287 100644 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall @@ -18,7 +18,7 @@ let spec = DockerImage.ReleaseSpec::{ , service = "itn-orchestrator" , step_key = "itn-orchestrator-docker-image" - , network = "berkeley" + , network = "devnet" , deb_repo = DebianRepo.Type.Local , deps = DebianVersions.dependsOn diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall new file mode 100644 index 00000000000..9f744d1f382 --- /dev/null +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -0,0 +1,45 @@ +let S = ../../Lib/SelectFiles.dhall + +let B = ../../External/Buildkite.dhall + +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let DebianVersions = ../../Constants/DebianVersions.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" + , S.exactly "buildkite/src/Jobs/Test/ConnectToDevnet" "dhall" + , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" + ] + , path = "Test" + , name = "ConnectToDevnet" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = + [ ConnectToTestnet.step + dependsOn + "devnet" + "40s" + "2m" + (B/SoftFail.Boolean True) + ] + } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index e356ad8f3fb..0d7099909ad 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -18,7 +18,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard - "daemon-berkeley" + "daemon-devnet" # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" in Pipeline.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 4e6de0140c0..55df0195024 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -18,7 +18,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard - "daemon-berkeley" + "daemon-devnet" # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" in Pipeline.build diff --git a/scripts/deb-builder-helpers.sh b/scripts/deb-builder-helpers.sh index d07d358c823..3562e4063db 100755 --- a/scripts/deb-builder-helpers.sh +++ b/scripts/deb-builder-helpers.sh @@ -2,7 +2,7 @@ # Helper script to include when building deb archives. -echo "--- Setting up the envrionment to build debian packages..." +echo "--- Setting up the environment to build debian packages..." set -euo pipefail @@ -54,19 +54,19 @@ esac # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" + MINA_DEB_NAME="mina-devnet" DEB_SUFFIX="" ;; *) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" + MINA_DEB_NAME="mina-devnet-${_SUFFIX}" DEB_SUFFIX="-${_SUFFIX}" ;; esac -#ADd suffix to debian to distinguish instrumented packages +#Add suffix to debian to distinguish instrumented packages if [[ -v DUNE_INSTRUMENT_WITH ]]; then INSTRUMENTED_SUFFIX=instrumented MINA_DEB_NAME="${MINA_DEB_NAME}-${INSTRUMENTED_SUFFIX}" @@ -296,28 +296,13 @@ build_daemon_devnet_deb() { echo "------------------------------------------------------------" echo "--- Building testnet signatures deb without keys:" - create_control_file mina-devnet "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' "${SUGGESTED_DEPS}" + create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' copy_common_daemon_configs devnet testnet 'seed-lists/devnet_seeds.txt' - build_deb mina-devnet -} -##################################### END DEVNET PACKAGE ####################################### - -##################################### BERKELEY PACKAGE ####################################### -build_daemon_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building Mina Berkeley testnet signatures deb without keys:" - - create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon' - - copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt' - build_deb "${MINA_DEB_NAME}" - } -##################################### END BERKELEY PACKAGE ####################################### +##################################### END DEVNET PACKAGE ####################################### ##################################### ARCHIVE PACKAGE ########################################## build_archive_deb () { diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index e90f125f967..9c59b017131 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -16,7 +16,6 @@ if [ $# -eq 0 ] build_keypair_deb build_archive_deb build_batch_txn_deb - build_daemon_berkeley_deb build_mainnet_daemon_deb build_devnet_daemon_deb build_test_executive_deb From b31fd50d23e98490aa2fc394c19650a58fcb7799 Mon Sep 17 00:00:00 2001 From: sai Date: Wed, 14 Aug 2024 18:56:38 -0400 Subject: [PATCH 012/178] fixing dhall syntax --- buildkite/src/Command/ConnectToTestnet.dhall | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index e16edfd1d7b..91bf6ee8bea 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -20,7 +20,6 @@ in { step = RunInToolchain.runInToolchain ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" - ] , label = "Connect to ${testnet}" , soft_fail = Some soft_fail , key = "connect-to-${testnet}" From 332563c53211299ce49f9425ab21d40979385fa7 Mon Sep 17 00:00:00 2001 From: sai Date: Thu, 15 Aug 2024 00:22:58 -0400 Subject: [PATCH 013/178] removing guard to run test on all branches --- buildkite/scripts/connect-to-testnet.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 45f26a8f742..cba27c93759 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -11,14 +11,6 @@ TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive From bc6d04fc5ed801f76f7b89b731f9d4731cfe2871 Mon Sep 17 00:00:00 2001 From: sai Date: Mon, 12 Aug 2024 19:42:52 -0400 Subject: [PATCH 014/178] swapping out the s3 bucket link for snark-keys --- buildkite/scripts/build-hardfork-package.sh | 6 +++--- scripts/mina-verify-packaged-fork-config | 3 +-- src/lib/cache_dir/fake/cache_dir.ml | 2 +- src/lib/cache_dir/native/cache_dir.ml | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/buildkite/scripts/build-hardfork-package.sh b/buildkite/scripts/build-hardfork-package.sh index 6a83ea82f5a..fc6d0396599 100755 --- a/buildkite/scripts/build-hardfork-package.sh +++ b/buildkite/scripts/build-hardfork-package.sh @@ -58,18 +58,18 @@ _build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --confi echo "--- Create hardfork config" FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json -existing_files=$(aws s3 ls s3://snark-keys.o1test.net/ | awk '{print $4}') +existing_files=$(aws s3 ls s3://snark-keys-ro.o1test.net/ | awk '{print $4}') for file in hardfork_ledgers/*; do filename=$(basename "$file") if echo "$existing_files" | grep -q "$filename"; then echo "Info: $filename already exists in the bucket, packaging it instead." oldhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}') - aws s3 cp "s3://snark-keys.o1test.net/$filename" "$file" + aws s3 cp "s3://snark-keys-ro.o1test.net/$filename" "$file" newhash=$(openssl dgst -r -sha3-256 "$file" | awk '{print $1}') sed -i "s/$oldhash/$newhash/g" new_config.json else - aws s3 cp --acl public-read "$file" s3://snark-keys.o1test.net/ + aws s3 cp --acl public-read "$file" s3://snark-keys-ro.o1test.net/ fi done diff --git a/scripts/mina-verify-packaged-fork-config b/scripts/mina-verify-packaged-fork-config index b19eaf560b0..fe1bf79a904 100755 --- a/scripts/mina-verify-packaged-fork-config +++ b/scripts/mina-verify-packaged-fork-config @@ -212,8 +212,7 @@ for file in "$workdir"/ledgers/*.tar.gz; do mkdir -p "$tardir"/{packaged,generated,web} tar -xzf "$file" -C "$tardir/generated" tar -xzf "$GENESIS_LEDGER_DIR/$tarname.tar.gz" -C "$tardir/packaged" - curl "https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net/$tarname.tar.gz" | tar -xz -C "$tardir/web" - + curl "https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net/$tarname.tar.gz" | tar -xz -C "$tardir/web" $ldb_cmd --hex --db="$tardir/packaged" scan > "$workdir/packaged.scan" $ldb_cmd --hex --db="$tardir/web" scan > "$workdir/web.scan" $ldb_cmd --hex --db="$tardir/generated" scan > "$workdir/generated.scan" diff --git a/src/lib/cache_dir/fake/cache_dir.ml b/src/lib/cache_dir/fake/cache_dir.ml index fe607a422e7..96c21eb6ff9 100644 --- a/src/lib/cache_dir/fake/cache_dir.ml +++ b/src/lib/cache_dir/fake/cache_dir.ml @@ -6,7 +6,7 @@ let autogen_path = "/tmp/coda_cache_dir" let s3_install_path = "/tmp/s3_cache_dir" let s3_keys_bucket_prefix = - "https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net" + "https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net" let manual_install_path = "/var/lib/coda" diff --git a/src/lib/cache_dir/native/cache_dir.ml b/src/lib/cache_dir/native/cache_dir.ml index dd8f21a49eb..8450d40bfd7 100644 --- a/src/lib/cache_dir/native/cache_dir.ml +++ b/src/lib/cache_dir/native/cache_dir.ml @@ -8,7 +8,7 @@ let s3_install_path = "/tmp/s3_cache_dir" let s3_keys_bucket_prefix = Option.value (Sys.getenv "MINA_LEDGER_S3_BUCKET") - ~default:"https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net" + ~default:"https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net" let manual_install_path = "/var/lib/coda" From a6212a62a0e79210b7d8e56854430eb391683068 Mon Sep 17 00:00:00 2001 From: sai Date: Thu, 15 Aug 2024 14:27:49 -0400 Subject: [PATCH 015/178] improving error logging --- src/lib/cache_dir/native/cache_dir.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/cache_dir/native/cache_dir.ml b/src/lib/cache_dir/native/cache_dir.ml index 8450d40bfd7..d76cd00aa38 100644 --- a/src/lib/cache_dir/native/cache_dir.ml +++ b/src/lib/cache_dir/native/cache_dir.ml @@ -55,7 +55,7 @@ let load_from_s3 s3_bucket_prefix s3_install_path ~logger = @@ Monitor.try_with ~here:[%here] (fun () -> let each_uri (uri_string, file_path) = let open Deferred.Let_syntax in - [%log trace] "Downloading file from S3" + [%log trace] "Downloading file from S3: $url to $local_file_path" ~metadata: [ ("url", `String uri_string) ; ("local_file_path", `String file_path) From 361ba3d0575be71dc4cd8cf6a047990bb1d1f9c3 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 19 Aug 2024 18:00:33 +0200 Subject: [PATCH 016/178] soft fail for rosetta tets --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 5 +++++ buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 2e2dc9c79a9..067a536251b 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -1,3 +1,5 @@ +let B = ../../External/Buildkite.dhall + let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall @@ -20,6 +22,8 @@ let Artifacts = ../../Constants/Artifacts.dhall let RunWithPostgres = ../../Command/RunWithPostgres.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -59,6 +63,7 @@ in Pipeline.build ] , label = "Rosetta integration tests Bullseye" , key = "rosetta-integration-tests-bullseye" + , soft_fail = Some (B/SoftFail.Boolean True) , target = Size.Small , depends_on = Dockers.dependsOn diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index ab4eed58015..eaee44624b3 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -1,5 +1,7 @@ let Cmd = ../../Lib/Cmds.dhall +let B = ../../External/Buildkite.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall @@ -14,6 +16,8 @@ let Command = ../../Command/Base.dhall let Size = ../../Command/Size.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -49,6 +53,7 @@ in Pipeline.build ] , label = "Rosetta integration tests Bullseye Long" , key = "rosetta-integration-tests-bullseye-long" + , soft_fail = Some (B/SoftFail.Boolean True) , target = Size.Small , depends_on = [ { name = "MinaArtifactBullseye" From e44821b3ba600772e06d2eca39da3b58afe3aad2 Mon Sep 17 00:00:00 2001 From: sai Date: Wed, 21 Aug 2024 17:58:56 -0400 Subject: [PATCH 017/178] removing unecessary index --- automation/scripts/gen-keys-ledger.sh | 2 +- buildkite/scripts/rosetta-integration-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/scripts/gen-keys-ledger.sh b/automation/scripts/gen-keys-ledger.sh index 33c881516fc..2ed5edcc9d9 100755 --- a/automation/scripts/gen-keys-ledger.sh +++ b/automation/scripts/gen-keys-ledger.sh @@ -1,5 +1,5 @@ #! /bin/bash -MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-devnet" +MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-berkeley" while [ $# -gt 0 ]; do case "$1" in diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index ca976ac17de..b6c4979a376 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -252,7 +252,7 @@ fi mina client status --json -next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." sleep $sleep_time From 56a406dbd5fe0b7f5db74d2fe391f7b4efab43a9 Mon Sep 17 00:00:00 2001 From: sai Date: Wed, 14 Aug 2024 18:48:48 -0400 Subject: [PATCH 018/178] first pass at changin ci deps from #15731 --- automation/scripts/gen-keys-ledger.sh | 2 +- buildkite/scripts/build-release.sh | 2 +- buildkite/scripts/connect-to-testnet.sh | 51 +++++++++++-------- buildkite/scripts/debian/install.sh | 2 +- buildkite/scripts/dump-mina-type-shapes.sh | 2 +- .../scripts/rosetta-integration-tests.sh | 6 ++- .../scripts/run-snark-transaction-profiler.sh | 2 +- buildkite/scripts/run-test-executive-cloud.sh | 2 +- buildkite/scripts/run-test-executive-local.sh | 4 +- buildkite/scripts/run_promote_build_job.sh | 2 +- .../scripts/run_verify_promoted_build_job.sh | 2 +- buildkite/scripts/single-node-tests.sh | 2 +- buildkite/scripts/version-linter.sh | 2 +- buildkite/src/Command/ConnectToTestnet.dhall | 10 ++-- buildkite/src/Command/MinaArtifact.dhall | 8 +-- .../Release/ItnOrchestratorArtifact.dhall | 2 +- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 45 ++++++++++++++++ .../Jobs/Test/TestnetIntegrationTests.dhall | 2 +- .../Test/TestnetIntegrationTestsLong.dhall | 2 +- scripts/deb-builder-helpers.sh | 27 +++------- scripts/rebuild-deb.sh | 8 +-- 21 files changed, 109 insertions(+), 76 deletions(-) create mode 100644 buildkite/src/Jobs/Test/ConnectToDevnet.dhall diff --git a/automation/scripts/gen-keys-ledger.sh b/automation/scripts/gen-keys-ledger.sh index 2ed5edcc9d9..33c881516fc 100755 --- a/automation/scripts/gen-keys-ledger.sh +++ b/automation/scripts/gen-keys-ledger.sh @@ -1,5 +1,5 @@ #! /bin/bash -MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-berkeley" +MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-devnet" while [ $# -gt 0 ]; do case "$1" in diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index f1242fb94f5..6bf49f48231 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -9,7 +9,7 @@ source ~/.profile ./buildkite/scripts/build-artifact.sh echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" -echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" +echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 3cfd7a4aa7b..7dfd054f470 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -23,8 +23,8 @@ esac # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl +sudo apt-get update +sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq git config --global --add safe.directory /workdir @@ -33,27 +33,29 @@ source buildkite/scripts/export-git-env-vars.sh source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" # Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: +sudo rm ~/.mina-config/.mina-lock ||: + +sudo mkdir -p /root/libp2p-keys/ -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ +sudo chmod -R 0700 /root/libp2p-keys/ +# Pre-generated random password for this quick test +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \ + TESTNET_NAME=$TESTNET_NAME \ + bash -c "mina daemon \ + --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \ + --libp2p-keypair \"/root/libp2p-keys/key\" \ + --seed &" # -background # Attempt to connect to the GraphQL client every 10s for up to 8 minutes num_status_retries=24 for ((i=1;i<=$num_status_retries;i++)); do sleep $WAIT_BETWEEN_POLLING_GRAPHQL set +e - mina client status + sudo mina client status status_exit_code=$? set -e if [ $status_exit_code -eq 0 ]; then @@ -63,13 +65,18 @@ for ((i=1;i<=$num_status_retries;i++)); do fi done -# Check that the daemon has connected to peers and is still up after 2 mins -sleep $WAIT_AFTER_FINAL_CHECK -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi +peer_retries=10 +for ((i=1;i<=$peer_retries;i++)); do + peer_count=$(sudo mina advanced get-peers | wc -l) + sudo mina client status + + if [ "$peer_count" -gt 0 ]; then + echo "Found some peers" + exit 0; + else + echo "No peers found" + fi + sleep $WAIT_AFTER_FINAL_CHECK +done +exit 1; \ No newline at end of file diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index 93313f4a2dd..bd9a863f8ef 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -34,7 +34,7 @@ else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do case $i in - mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet) + mina-devnet|mina-mainnet|mina-devnet-lightnet) # Downaload mina-logproc too source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER ;; diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh index ff920e7b143..23579a89917 100755 --- a/buildkite/scripts/dump-mina-type-shapes.sh +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index c3fb9dd378a..a8317183961 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then send_zkapp_transactions & fi +mina client status --json + next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) +sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} +sleep $sleep_time # Mina Rosetta Checks (spec construction data perf) echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" diff --git a/buildkite/scripts/run-snark-transaction-profiler.sh b/buildkite/scripts/run-snark-transaction-profiler.sh index 802cd730632..70c20a92167 100755 --- a/buildkite/scripts/run-snark-transaction-profiler.sh +++ b/buildkite/scripts/run-snark-transaction-profiler.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/run-test-executive-cloud.sh b/buildkite/scripts/run-test-executive-cloud.sh index 884b8222cbc..8a3df280401 100755 --- a/buildkite/scripts/run-test-executive-cloud.sh +++ b/buildkite/scripts/run-test-executive-cloud.sh @@ -2,7 +2,7 @@ set -o pipefail -x TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 7312a19941a..087ad7cc1e2 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -26,7 +26,7 @@ cleanup TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then @@ -40,7 +40,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 26feee3e23a..9191cf5bb84 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index ed093b1a44b..b758f4e661a 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/single-node-tests.sh b/buildkite/scripts/single-node-tests.sh index 93b224c5ba8..dcd6752ae23 100755 --- a/buildkite/scripts/single-node-tests.sh +++ b/buildkite/scripts/single-node-tests.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1 +source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1 export MINA_LIBP2P_PASS="naughty blue worm" export MINA_PRIVKEY_PASS="naughty blue worm" diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index ce3da4eb1a0..c6007c27dd9 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then exit 1 fi -TESTNET_NAME="${TESTNET_NAME:-berkeley}" +TESTNET_NAME="${TESTNET_NAME:-devnet}" # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index ba4bba1b974..e16edfd1d7b 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -4,9 +4,9 @@ let Command = ./Base.dhall let Size = ./Size.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let RunInToolchain = ./RunInToolchain.dhall -let Cmd = ../Lib/Cmds.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type in { step = \(dependsOn : List Command.TaggedKey.Type) @@ -17,10 +17,8 @@ in { step = -> Command.build Command.Config::{ , commands = - [ Cmd.runInDocker - Cmd.Docker::{ - , image = (../Constants/ContainerImages.dhall).ubuntu2004 - } + RunInToolchain.runInToolchain + ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" ] , label = "Connect to ${testnet}" diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 59a7bbb5876..06bc2131e4b 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -52,7 +52,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Berkeley ] + , networks = [ Network.Type.Devnet ] , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -151,7 +151,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-batch-txn" - , network = "berkeley" + , network = "devnet" , deb_codename = "${DebianVersions.lowerName debVersion}" , deb_profile = profile , build_flags = buildFlags @@ -181,7 +181,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-rosetta" - , network = "berkeley" + , network = "devnet" , build_flags = buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = profile @@ -220,7 +220,7 @@ let docker_step debVersion}${Profiles.toLabelSegment profile}${BuildFlags.toLabelSegment buildFlags}--docker-image" - , network = "berkeley" + , network = "devnet" } ] } diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall index 5c88037f8d1..4b6b0598287 100644 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall @@ -18,7 +18,7 @@ let spec = DockerImage.ReleaseSpec::{ , service = "itn-orchestrator" , step_key = "itn-orchestrator-docker-image" - , network = "berkeley" + , network = "devnet" , deb_repo = DebianRepo.Type.Local , deps = DebianVersions.dependsOn diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall new file mode 100644 index 00000000000..9f744d1f382 --- /dev/null +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -0,0 +1,45 @@ +let S = ../../Lib/SelectFiles.dhall + +let B = ../../External/Buildkite.dhall + +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let DebianVersions = ../../Constants/DebianVersions.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" + , S.exactly "buildkite/src/Jobs/Test/ConnectToDevnet" "dhall" + , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" + ] + , path = "Test" + , name = "ConnectToDevnet" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = + [ ConnectToTestnet.step + dependsOn + "devnet" + "40s" + "2m" + (B/SoftFail.Boolean True) + ] + } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index e356ad8f3fb..0d7099909ad 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -18,7 +18,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard - "daemon-berkeley" + "daemon-devnet" # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" in Pipeline.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 4e6de0140c0..55df0195024 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -18,7 +18,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard - "daemon-berkeley" + "daemon-devnet" # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" in Pipeline.build diff --git a/scripts/deb-builder-helpers.sh b/scripts/deb-builder-helpers.sh index 8c2f418b76d..68cfc372920 100755 --- a/scripts/deb-builder-helpers.sh +++ b/scripts/deb-builder-helpers.sh @@ -2,7 +2,7 @@ # Helper script to include when building deb archives. -echo "--- Setting up the envrionment to build debian packages..." +echo "--- Setting up the environment to build debian packages..." set -euo pipefail @@ -54,19 +54,19 @@ esac # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" + MINA_DEB_NAME="mina-devnet" DEB_SUFFIX="" ;; *) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" + MINA_DEB_NAME="mina-devnet-${_SUFFIX}" DEB_SUFFIX="-${_SUFFIX}" ;; esac -#ADd suffix to debian to distinguish instrumented packages +#Add suffix to debian to distinguish instrumented packages if [[ -v DUNE_INSTRUMENT_WITH ]]; then INSTRUMENTED_SUFFIX=instrumented MINA_DEB_NAME="${MINA_DEB_NAME}-${INSTRUMENTED_SUFFIX}" @@ -333,28 +333,13 @@ build_daemon_devnet_deb() { echo "------------------------------------------------------------" echo "--- Building testnet signatures deb without keys:" - create_control_file mina-devnet "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' "${SUGGESTED_DEPS}" + create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' copy_common_daemon_configs devnet testnet 'seed-lists/devnet_seeds.txt' - build_deb mina-devnet -} -##################################### END DEVNET PACKAGE ####################################### - -##################################### BERKELEY PACKAGE ####################################### -build_daemon_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building Mina Berkeley testnet signatures deb without keys:" - - create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon' - - copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt' - build_deb "${MINA_DEB_NAME}" - } -##################################### END BERKELEY PACKAGE ####################################### +##################################### END DEVNET PACKAGE ####################################### ##################################### ARCHIVE PACKAGE ########################################## build_archive_deb () { diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index db063592a64..9c59b017131 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -16,12 +16,8 @@ if [ $# -eq 0 ] build_keypair_deb build_archive_deb build_batch_txn_deb - build_daemon_berkeley_deb - build_daemon_mainnet_deb - build_daemon_devnet_deb - build_rosetta_berkeley_deb - build_rosetta_mainnet_deb - build_rosetta_devnet_deb + build_mainnet_daemon_deb + build_devnet_daemon_deb build_test_executive_deb build_functional_test_suite_deb build_zkapp_test_transaction_deb From 1a0abb4f0a8fffbb576ea2b10e0bf9e2401cc510 Mon Sep 17 00:00:00 2001 From: sai Date: Wed, 14 Aug 2024 18:56:38 -0400 Subject: [PATCH 019/178] fixing dhall syntax --- buildkite/src/Command/ConnectToTestnet.dhall | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index e16edfd1d7b..91bf6ee8bea 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -20,7 +20,6 @@ in { step = RunInToolchain.runInToolchain ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" - ] , label = "Connect to ${testnet}" , soft_fail = Some soft_fail , key = "connect-to-${testnet}" From bdf2a465bb5b7ac402d73c28f35270784f53e1b8 Mon Sep 17 00:00:00 2001 From: sai Date: Thu, 15 Aug 2024 00:22:58 -0400 Subject: [PATCH 020/178] removing guard to run test on all branches --- buildkite/scripts/connect-to-testnet.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 7dfd054f470..8f2ee367b02 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -12,14 +12,6 @@ TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive From 613cfd5f7bfa8d1564fb20fd9135ea537f0d1439 Mon Sep 17 00:00:00 2001 From: sai Date: Wed, 21 Aug 2024 17:58:56 -0400 Subject: [PATCH 021/178] removing unecessary index --- automation/scripts/gen-keys-ledger.sh | 2 +- buildkite/scripts/rosetta-integration-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/scripts/gen-keys-ledger.sh b/automation/scripts/gen-keys-ledger.sh index 33c881516fc..2ed5edcc9d9 100755 --- a/automation/scripts/gen-keys-ledger.sh +++ b/automation/scripts/gen-keys-ledger.sh @@ -1,5 +1,5 @@ #! /bin/bash -MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-devnet" +MINA_DAEMON_IMAGE="minaprotocol/mina-daemon:1.3.2beta2-release-2.0.0-6f9d956-focal-berkeley" while [ $# -gt 0 ]; do case "$1" in diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index a8317183961..e091ca39269 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -252,7 +252,7 @@ fi mina client status --json -next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." sleep $sleep_time From acbd3f65fed48c8e1be8cc6ca8dcf0935cb30f1d Mon Sep 17 00:00:00 2001 From: sai Date: Thu, 22 Aug 2024 22:52:27 -0400 Subject: [PATCH 022/178] changing field type to snark param version --- src/lib/mina_graphql/types.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/mina_graphql/types.ml b/src/lib/mina_graphql/types.ml index f8fb79b3894..8e13606f7b3 100644 --- a/src/lib/mina_graphql/types.ml +++ b/src/lib/mina_graphql/types.ml @@ -205,7 +205,7 @@ let block_producer_timing : ] ) let merkle_path_element : - (_, [ `Left of Zkapp_basic.F.t | `Right of Zkapp_basic.F.t ] option) typ = + (_, [ `Left of Snark_params.Tick.Field.t | `Right of Snark_params.Tick.Field.t ] option) typ = obj "MerklePathElement" ~fields:(fun _ -> [ field "left" ~typ:field_elem ~args:Arg.[] @@ -653,13 +653,13 @@ let local_state : (Mina_lib.t, Mina_state.Local_state.t option) typ = ~args:Arg.[] ~doc:"Stack frame component of local state" ~typ:(non_null field_elem) ~resolve:(fun _ t -> - (M.stack_frame t : Stack_frame.Digest.t :> Zkapp_basic.F.Stable.V1.t) + (M.stack_frame t : Stack_frame.Digest.t :> Snark_params.Tick.Field.t) ) ; field "callStack" ~args:Arg.[] ~doc:"Call stack component of local state" ~typ:(non_null field_elem) ~resolve:(fun _ t -> - (M.call_stack t : Call_stack_digest.t :> Zkapp_basic.F.Stable.V1.t) + (M.call_stack t : Call_stack_digest.t :> Snark_params.Tick.Field.t) ) ; field "transactionCommitment" ~args:Arg.[] From c1e07dac700e698503b3af0333daceb8b5fc875e Mon Sep 17 00:00:00 2001 From: sai Date: Thu, 22 Aug 2024 23:08:39 -0400 Subject: [PATCH 023/178] format --- src/lib/mina_graphql/types.ml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/mina_graphql/types.ml b/src/lib/mina_graphql/types.ml index 8e13606f7b3..5070e1ff75d 100644 --- a/src/lib/mina_graphql/types.ml +++ b/src/lib/mina_graphql/types.ml @@ -205,7 +205,11 @@ let block_producer_timing : ] ) let merkle_path_element : - (_, [ `Left of Snark_params.Tick.Field.t | `Right of Snark_params.Tick.Field.t ] option) typ = + ( _ + , [ `Left of Snark_params.Tick.Field.t + | `Right of Snark_params.Tick.Field.t ] + option ) + typ = obj "MerklePathElement" ~fields:(fun _ -> [ field "left" ~typ:field_elem ~args:Arg.[] From a6e67a1c0e2e6721808d1296766c91da855e26cd Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 24 Aug 2024 14:26:40 +0200 Subject: [PATCH 024/178] Implement first test on mina_automation - path archive test --- buildkite/scripts/build-artifact.sh | 3 +- buildkite/src/Command/MinaArtifact.dhall | 14 +- buildkite/src/Command/PatchArchiveTest.dhall | 27 +++ buildkite/src/Command/ReplayerTest.dhall | 2 +- .../src/Jobs/Test/PatchArchiveTest.dhall | 39 +++++ scripts/debian/builder-helpers.sh | 7 +- scripts/patch-archive-test.sh | 17 ++ scripts/replayer-test.sh | 2 +- src/test/archive/patch_archive_test/dune | 31 ++++ .../patch_archive_test/patch_archive_test.ml | 160 ++++++++++++++++++ src/test/archive/patch_archive_test/sql.ml | 29 ++++ 11 files changed, 320 insertions(+), 11 deletions(-) create mode 100644 buildkite/src/Command/PatchArchiveTest.dhall create mode 100644 buildkite/src/Jobs/Test/PatchArchiveTest.dhall create mode 100755 scripts/patch-archive-test.sh create mode 100644 src/test/archive/patch_archive_test/dune create mode 100644 src/test/archive/patch_archive_test/patch_archive_test.ml create mode 100644 src/test/archive/patch_archive_test/sql.ml diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 263a258ef3c..d01fe36b3b6 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -45,4 +45,5 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \ src/app/rosetta/indexer_test/indexer_test.exe \ src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \ src/app/test_executive/test_executive.exe \ - src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log + src/test/command_line_tests/command_line_tests.exe \ + src/test/archive/patch_archive_test/patch_archive_test.exe diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 4fe0ed95ffd..870834fe89e 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -188,7 +188,7 @@ let docker_step , step_key = "batch-txn-${DebianVersions.lowerName spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + spec.buildFlags}-docker-image" } ] , Archive = @@ -234,9 +234,9 @@ let docker_step "${DebianVersions.lowerName spec.debVersion}" , step_key = "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + debVersion}${Profiles.toLabelSegment + profile}${BuildFlags.toLabelSegment + buildFlags}-docker-image" } ] , FunctionalTestSuite = @@ -250,9 +250,9 @@ let docker_step , deb_profile = spec.profile , step_key = "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + debVersion}${Profiles.toLabelSegment + profile}${BuildFlags.toLabelSegment + buildFlags}-docker-image" , network = "berkeley" } ] diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall new file mode 100644 index 00000000000..55d01eab194 --- /dev/null +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -0,0 +1,27 @@ +let Artifacts = ../Constants/Artifacts.dhall + +let Command = ./Base.dhall + +let Size = ./Size.dhall + +let RunWithPostgres = ./RunWithPostgres.dhall + +in { step = + \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + [ RunWithPostgres.runInDockerWithPostgresConn + [ "PATCH_ARCHIVE_TEST_APP=mina-patch-archive-test" + , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" + ] + "./src/test/archive/sample_db/archive_db.sql" + Artifacts.Type.FunctionalTestSuite + "./scripts/patch-archive-test.sh" + ] + , label = "Archive: Patch Archive test" + , key = "patch-archive-test" + , target = Size.Large + , depends_on = dependsOn + } + } diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 02c02978836..e1fa220de90 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -15,7 +15,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - "./buildkite/scripts/replayer-test.sh" + "./scripts/replayer-test.sh -i src/test/archive/sample_db/replayer_input_file.json -a mina-replayer" ] , label = "Archive: Replayer test" , key = "replayer-test" diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall new file mode 100644 index 00000000000..8fc433daa57 --- /dev/null +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -0,0 +1,39 @@ +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let Dockers = ../../Constants/DockerVersions.dhall + +let dependsOn = + Dockers.dependsOn + Dockers.Type.Bullseye + Profiles.Type.Standard + "test-suite" + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "scripts/path-archive-test" "sh" + , S.exactly "buildkite/src/Jobs/Test/PatchArchiveTest" "dhall" + , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall" + ] + , path = "Test" + , name = "PatchArchiveTest" + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + ] + } + , steps = [ PatchArchiveTest.step dependsOn ] + } diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 0e34178731d..58cbaa80a26 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -239,9 +239,14 @@ build_batch_txn_deb() { build_functional_test_suite_deb() { create_control_file mina-test-suite "${SHARED_DEPS}" 'Test suite apps for mina.' + mkdir -p "${BUILDDIR}/etc/mina/test/archive" + + cp -r ../src/test/archive/* ${BUILDDIR}/etc/mina/test/archive/ + # Binaries cp ./default/src/test/command_line_tests/command_line_tests.exe "${BUILDDIR}/usr/local/bin/mina-command-line-tests" - + cp ./default/src/test/archive/patch_archive_test/patch_archive_test.exe "${BUILDDIR}/usr/local/bin/mina-patch-archive-test" + build_deb mina-test-suite } diff --git a/scripts/patch-archive-test.sh b/scripts/patch-archive-test.sh new file mode 100755 index 00000000000..48e1c61861a --- /dev/null +++ b/scripts/patch-archive-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -x +# test replayer on known archive db + +NETWORK_DATA_FOLDER=src/test/archive/sample_db +PATCH_ARCHIVE_TEST_APP=${PATCH_ARCHIVE_TEST_APP:-_build/default/src/test/archive/patch_archive_test/patch_archive_test.exe} +PG_PORT=${PG_PORT:-5432} +POSTGRES_USER=${POSTGRES_USER:-postgres} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + +CONN=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${PG_PORT} + + +echo "Running patch archive test" +$PATCH_ARCHIVE_TEST_APP --source-uri $CONN \ + --network-data-folder $NETWORK_DATA_FOLDER diff --git a/scripts/replayer-test.sh b/scripts/replayer-test.sh index b75c32f4b08..8b6f511295b 100755 --- a/scripts/replayer-test.sh +++ b/scripts/replayer-test.sh @@ -5,7 +5,7 @@ set -x INPUT_FILE=src/test/archive/sample_db/replayer_input_file.json REPLAYER_APP=_build/default/src/app/replayer/replayer.exe -PG_CONN=postgres://postgres:postgres@localhost:5433/archive +PG_CONN=postgres://postgres:postgres@localhost:5432/archive while [[ "$#" -gt 0 ]]; do case $1 in -i|--input-file) INPUT_FILE="$2"; shift;; diff --git a/src/test/archive/patch_archive_test/dune b/src/test/archive/patch_archive_test/dune new file mode 100644 index 00000000000..109d27b4a77 --- /dev/null +++ b/src/test/archive/patch_archive_test/dune @@ -0,0 +1,31 @@ +(executable + (package patch_archive_test) + (name patch_archive_test) + (public_name patch_archive_test) + (libraries + async + async.async_command + core_kernel + caqti + caqti-async + caqti-driver-postgresql + integration_test_lib + archive_lib + block_time + mina_numbers + logger + mina_base + uri + base + async_kernel + core + async_unix + stdio + base.caml + result + mina_automation + bounded_types + ) + (preprocessor_deps ../../../config.mlh) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv h_list.ppx))) diff --git a/src/test/archive/patch_archive_test/patch_archive_test.ml b/src/test/archive/patch_archive_test/patch_archive_test.ml new file mode 100644 index 00000000000..ea61373c72b --- /dev/null +++ b/src/test/archive/patch_archive_test/patch_archive_test.ml @@ -0,0 +1,160 @@ +(* patch_archive_test.ml *) + +(* test patching of archive databases + + test structure: + - import reference database for comparision (for example with 100 blocks) + - create new schema and export blocks from reference db with some missing ones + - patch the database with missing precomputed blocks + - compare original and copy +*) + +module Network_Data = struct + type t = + { init_script : String.t + ; precomputed_blocks_zip : String.t + ; genesis_ledger_file : String.t + ; replayer_input_file : String.t + ; folder : String.t + } + + let create folder = + { init_script = "archive_db.sql" + ; genesis_ledger_file = "input.json" + ; precomputed_blocks_zip = "precomputed_blocks.zip" + ; replayer_input_file = "replayer_input_file.json" + ; folder + } +end + +open Core_kernel +open Async +open Mina_automation + +let main ~db_uri ~network_data_folder () = + let open Deferred.Let_syntax in + let missing_blocks_count = 3 in + let network_name = "dummy" in + + let network_data = Network_Data.create network_data_folder in + + let output_folder = Filename.temp_dir_name ^ "/output" in + + let%bind output_folder = Unix.mkdtemp output_folder in + + let connection = Psql.Conn_str db_uri in + + let source_db_name = "patch_archive_test_source" in + let target_db_name = "patch_archive_test_target" in + let%bind _ = Psql.create_empty_db ~connection ~db:source_db_name in + let%bind _ = + Psql.run_script ~connection ~db:source_db_name + (network_data.folder ^ "/" ^ network_data.init_script) + in + let%bind () = Psql.create_mina_db ~connection ~db:target_db_name in + + let source_db = db_uri ^ "/" ^ source_db_name in + let target_db = db_uri ^ "/" ^ target_db_name in + + let extract_blocks = Extract_blocks.of_context Executor.AutoDetect in + let config = + { Extract_blocks.Config.archive_uri = source_db + ; range = Extract_blocks.Config.AllBlocks + ; output_folder = Some output_folder + ; network = Some network_name + ; include_block_height_in_name = true + } + in + let%bind _ = Extract_blocks.run extract_blocks ~config in + + let archive_blocks = Archive_blocks.of_context Executor.AutoDetect in + + let%bind extensional_files = + Sys.ls_dir output_folder + >>= Deferred.List.map ~f:(fun e -> + Deferred.return (output_folder ^ "/" ^ e) ) + in + + let n = + List.range 0 missing_blocks_count + |> List.map ~f:(fun _ -> Random.int (List.length extensional_files)) + in + + let unpatched_extensional_files = + List.filteri extensional_files ~f:(fun i _ -> + not (List.mem n i ~equal:Int.equal) ) + |> List.dedup_and_sort ~compare:(fun left right -> + let scan_height item = + let item = + Filename.basename item |> Str.global_replace (Str.regexp "-") " " + in + Scanf.sscanf item "%s %d %s" (fun _ height _ -> height) + in + + let left_height = scan_height left in + let right_height = scan_height right in + + Int.compare left_height right_height ) + in + + let%bind _ = + Archive_blocks.run archive_blocks ~blocks:unpatched_extensional_files + ~archive_uri:target_db ~format:Extensional + in + + let%bind missing_blocks_auditor_path = + Missing_blocks_auditor.of_context Executor.AutoDetect + |> Missing_blocks_auditor.path + in + + let%bind archive_blocks_path = Archive_blocks.path archive_blocks in + + let config = + { Missing_blocks_guardian.Config.archive_uri = Uri.of_string target_db + ; precomputed_blocks = Uri.make ~scheme:"file" ~path:output_folder () + ; network = network_name + ; run_mode = Run + ; missing_blocks_auditor = missing_blocks_auditor_path + ; archive_blocks = archive_blocks_path + ; block_format = Extensional + } + in + + let missing_blocks_guardian = + Missing_blocks_guardian.of_context Executor.AutoDetect + in + + let%bind _ = Missing_blocks_guardian.run missing_blocks_guardian ~config in + + let replayer = Replayer.of_context Executor.AutoDetect in + + let%bind _ = + Replayer.run replayer ~archive_uri:target_db + ~input_config: + (network_data.folder ^ "/" ^ network_data.replayer_input_file) + ~interval_checkpoint:10 ~output_ledger:"./output_ledger" () + in + + Deferred.unit + +let () = + Command.( + run + (let open Let_syntax in + async ~summary:"Test patching of blocks in an archive database" + (let%map db_uri = + Param.flag "--source-uri" + ~doc: + "URI URI for connecting to the database (e.g., \ + postgres://$USER@localhost:5432)" + Param.(required string) + and network_data_folder = + Param.( + flag "--network-data-folder" ~aliases:[ "network-data-folder" ] + Param.(required string)) + ~doc: + "Path Path to folder containing network data. Usually it's sql \ + for db import, genesis ledger and zipped precomputed blocks \ + archive" + in + main ~db_uri ~network_data_folder ))) diff --git a/src/test/archive/patch_archive_test/sql.ml b/src/test/archive/patch_archive_test/sql.ml new file mode 100644 index 00000000000..b8ce7abf09a --- /dev/null +++ b/src/test/archive/patch_archive_test/sql.ml @@ -0,0 +1,29 @@ +module Block = struct + let run_state_hash (module Conn : Caqti_async.CONNECTION) = + Conn.collect_list + (Caqti_request.collect Caqti_type.unit Caqti_type.string + {sql| SELECT state_hash from blocks + |sql} ) + () + + let run (module Conn : Caqti_async.CONNECTION) ~state_hash = + Conn.find + (Caqti_request.find Caqti_type.string Caqti_type.int + {sql| SELECT id from blocks where state_hash = ? + |sql} ) + state_hash + + let run_unset_parent (module Conn : Caqti_async.CONNECTION) id = + Conn.exec + (Caqti_request.exec Caqti_type.int + {sql| UPDATE blocks SET parent_hash = NULL where id = ? + |sql} ) + id + + let run_delete (module Conn : Caqti_async.CONNECTION) ~state_hash = + Conn.exec + (Caqti_request.exec Caqti_type.string + {sql| DELETE FROM blocks where state_hash = ? CASCADE + |sql} ) + state_hash +end From 0457bbbf4f0a939dfc1f64339077ef9587c66140 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 26 Aug 2024 10:29:01 +0200 Subject: [PATCH 025/178] remove /usr/local/bin/from official paths --- scripts/debian/builder-helpers.sh | 3 +-- src/test/mina_automation/archive_blocks.ml | 2 +- src/test/mina_automation/extract_blocks.ml | 2 +- src/test/mina_automation/missing_blocks_auditor.ml | 2 +- src/test/mina_automation/missing_blocks_guardian.ml | 2 +- src/test/mina_automation/replayer.ml | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 58cbaa80a26..bf37db05df7 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -369,8 +369,7 @@ build_archive_deb () { cp ./default/src/app/extract_blocks/extract_blocks.exe "${BUILDDIR}/usr/local/bin/mina-extract-blocks" mkdir -p "${BUILDDIR}/etc/mina/archive" - cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/etc/mina/archive" - + cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/usr/local/bin/mina-missing-blocks-guardian" cp ./default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe "${BUILDDIR}/usr/local/bin/mina-missing-blocks-auditor" cp ./default/src/app/replayer/replayer.exe "${BUILDDIR}/usr/local/bin/mina-replayer" diff --git a/src/test/mina_automation/archive_blocks.ml b/src/test/mina_automation/archive_blocks.ml index 2c0dc47c89d..31e99b28a0e 100644 --- a/src/test/mina_automation/archive_blocks.ml +++ b/src/test/mina_automation/archive_blocks.ml @@ -9,7 +9,7 @@ include Executor let of_context context = Executor.of_context ~context ~dune_name:"src/app/archive_blocks/archive_blocks.exe" - ~official_name:"/usr/local/bin/mina-archive-blocks" + ~official_name:"mina-archive-blocks" type format = Precomputed | Extensional diff --git a/src/test/mina_automation/extract_blocks.ml b/src/test/mina_automation/extract_blocks.ml index 7b05e9b00cb..bfb3ec9e2ae 100644 --- a/src/test/mina_automation/extract_blocks.ml +++ b/src/test/mina_automation/extract_blocks.ml @@ -55,6 +55,6 @@ end let of_context context = Executor.of_context ~context ~dune_name:"src/app/extract_blocks/extract_blocks.exe" - ~official_name:"/usr/local/bin/mina-extract-blocks" + ~official_name:"mina-extract-blocks" let run t ~config = run t ~args:(Config.to_args config) () diff --git a/src/test/mina_automation/missing_blocks_auditor.ml b/src/test/mina_automation/missing_blocks_auditor.ml index f28983711bb..4c6116a4c24 100644 --- a/src/test/mina_automation/missing_blocks_auditor.ml +++ b/src/test/mina_automation/missing_blocks_auditor.ml @@ -8,4 +8,4 @@ include Executor let of_context context = Executor.of_context ~context ~dune_name:"src/app/missing_blocks_auditor/missing_blocks_auditor.exe" - ~official_name:"/usr/local/bin/mina-missing-blocks-auditor" + ~official_name:"mina-missing-blocks-auditor" diff --git a/src/test/mina_automation/missing_blocks_guardian.ml b/src/test/mina_automation/missing_blocks_guardian.ml index 42bbf0ec844..b74ea406855 100644 --- a/src/test/mina_automation/missing_blocks_guardian.ml +++ b/src/test/mina_automation/missing_blocks_guardian.ml @@ -45,7 +45,7 @@ end let of_context context = Executor.of_context ~context ~dune_name:"scripts/archive/missing-blocks-guardian.sh" - ~official_name:"/etc/mina/archive/missing-blocks-guardian.sh" + ~official_name:"mina-missing-blocks-guardian" let run t ~config = run t ~args:(Config.to_args config) ~env:(Config.to_envs config) () diff --git a/src/test/mina_automation/replayer.ml b/src/test/mina_automation/replayer.ml index 049f8071d95..fbdfe133d1c 100644 --- a/src/test/mina_automation/replayer.ml +++ b/src/test/mina_automation/replayer.ml @@ -68,7 +68,7 @@ include Executor let of_context context = Executor.of_context ~context ~dune_name:"src/app/replayer/replayer.exe" - ~official_name:"/usr/local/bin/mina-replayer" + ~official_name:"mina-replayer" let run t ~archive_uri ~input_config ~interval_checkpoint ?checkpoint_output_folder ?checkpoint_file_prefix ~output_ledger = From 79c3c6eb6a8108370f6d0ae377806cae5abbad7f Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 27 Aug 2024 00:30:44 +0200 Subject: [PATCH 026/178] replace string with method usage --- buildkite/src/Command/DockerImage.dhall | 4 +++- buildkite/src/Command/MinaArtifact.dhall | 6 +++--- buildkite/src/Command/PromotePackage.dhall | 2 +- buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall | 4 +++- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 4 +++- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index cc5131772c4..286cb7710df 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -18,6 +18,8 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let Network = ../Constants/Network.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type @@ -38,7 +40,7 @@ let ReleaseSpec = } , default = { deps = [] : List Command.TaggedKey.Type - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" , service = "\\\${MINA_SERVICE}" , branch = "\\\${BUILDKITE_BRANCH}" diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 4c6310998d1..b0ffb84d3f8 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -179,7 +179,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-batch-txn" - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , deb_codename = "${DebianVersions.lowerName spec.debVersion}" , deb_profile = spec.profile @@ -211,7 +211,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-rosetta" - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile @@ -253,7 +253,7 @@ let docker_step spec.debVersion}${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment spec.buildFlags}--docker-image" - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" } ] } diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall index 9303532e9e2..4989004d394 100644 --- a/buildkite/src/Command/PromotePackage.dhall +++ b/buildkite/src/Command/PromotePackage.dhall @@ -59,7 +59,7 @@ let PromoteDebianSpec = , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable - , to_channel = DebianChannel.Type.Nightly + , to_channel = DebianChannel.Type.NightlyDevelop , profile = Profiles.Type.Standard , remove_profile_from_name = False , step_key = "promote-debian-package" diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall index 4b6b0598287..8e9d8271d88 100644 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall @@ -14,11 +14,13 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let DebianRepo = ../../Constants/DebianRepo.dhall +let Network = ../../Constants/Network.dhall + let spec = DockerImage.ReleaseSpec::{ , service = "itn-orchestrator" , step_key = "itn-orchestrator-docker-image" - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , deb_repo = DebianRepo.Type.Local , deps = DebianVersions.dependsOn diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 9f744d1f382..97f05e0e19f 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -16,6 +16,8 @@ let Profiles = ../../Constants/Profiles.dhall let DebianVersions = ../../Constants/DebianVersions.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye @@ -37,7 +39,7 @@ in Pipeline.build , steps = [ ConnectToTestnet.step dependsOn - "devnet" + "${Network.lowerName Network.Type.Devnet}" "40s" "2m" (B/SoftFail.Boolean True) From 7b62ae355a9ce8ab7278bb4514fcf113569dee9d Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 27 Aug 2024 22:38:57 +0200 Subject: [PATCH 027/178] fix MinaArtifact.dhall --- buildkite/src/Command/MinaArtifact.dhall | 101 ++++++++++++----------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 280604a8cf9..5d6acaf03d2 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -105,36 +105,6 @@ let build_artifacts ] } -let publish_to_debian_repo = - \(spec : MinaBuildSpec.Type) - -> Command.build - Command.Config::{ - , commands = - Toolchain.select - spec.toolchainSelectMode - spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] - "./buildkite/scripts/debian/publish.sh" - , label = - "Publish Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile}" - , key = "publish-deb-pkg" - , depends_on = - DebianVersions.dependsOnStep - (Some spec.prefix) - spec.debVersion - spec.profile - spec.buildFlags - "build" - , target = Size.Small - } - let docker_step : Artifacts.Type -> MinaBuildSpec.Type -> List DockerImage.ReleaseSpec.Type = \(artifact : Artifacts.Type) @@ -181,23 +151,25 @@ let docker_step , deps = deps , service = Artifacts.dockerName Artifacts.Type.BatchTxn , network = Network.lowerName Network.Type.Berkeley - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , build_flags = buildFlags + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" + , deb_profile = spec.profile + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , step_key = "batch-txn-${DebianVersions.lowerName spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + spec.buildFlags}-docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps , service = Artifacts.dockerName Artifacts.Type.Archive - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , build_flags = buildFlags + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" + , deb_profile = spec.profile + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , step_key = "archive-${DebianVersions.lowerName @@ -217,23 +189,23 @@ let docker_step Artifacts.dockerName Artifacts.Type.Rosetta , network = Network.lowerName n , deb_codename = - "${DebianVersions.lowerName debVersion}" - , deb_profile = profile + "${DebianVersions.lowerName spec.debVersion}" + , deb_profile = spec.profile , deb_repo = DebianRepo.Type.Local , step_key = "rosetta-${Network.lowerName n}-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}-docker-image" } ) - networks + spec.networks , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps , service = Artifacts.dockerName Artifacts.Type.ZkappTestTransaction - , build_flags = buildFlags + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile , deb_codename = @@ -242,7 +214,7 @@ let docker_step "zkapp-test-transaction-${DebianVersions.lowerName spec.debVersion}${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + spec.buildFlags}-docker-image" } ] , FunctionalTestSuite = @@ -251,15 +223,16 @@ let docker_step , service = Artifacts.dockerName Artifacts.Type.FunctionalTestSuite , network = Network.lowerName Network.Type.Berkeley - , deb_codename = "${DebianVersions.lowerName debVersion}" - , build_flags = buildFlags + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile , step_key = "test-suite-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}--docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" } ] } @@ -315,6 +288,7 @@ let publish_to_debian_repo = , key = "publish-deb-pkg" , depends_on = DebianVersions.dependsOnStep + (Some spec.prefix) spec.debVersion spec.profile spec.buildFlags @@ -322,6 +296,35 @@ let publish_to_debian_repo = , target = Size.Small } +let pipelineBuilder + : MinaBuildSpec.Type -> List Command.Type -> Pipeline.Config.Type + = \(spec : MinaBuildSpec.Type) + -> \(steps : List Command.Type) + -> Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion + , path = "Release" + , name = + "${spec.prefix}${DebianVersions.capitalName + spec.debVersion}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + , tags = spec.tags + , mode = spec.mode + } + , steps = steps + } + +let onlyDebianPipeline + : MinaBuildSpec.Type -> Pipeline.Config.Type + = \(spec : MinaBuildSpec.Type) + -> pipelineBuilder + spec + [ Libp2p.step spec.debVersion spec.buildFlags + , build_artifacts spec + , publish_to_debian_repo spec + ] + let pipeline : MinaBuildSpec.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) From 9a85c3647917e72d8c6a32817feebcfb4ffaca6e Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 27 Aug 2024 22:56:29 +0200 Subject: [PATCH 028/178] fix MinaArtifact.dhall --- buildkite/src/Command/MinaArtifact.dhall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 5d6acaf03d2..0cb61c0c39b 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -279,8 +279,9 @@ let publish_to_debian_repo = , "AWS_SECRET_ACCESS_KEY" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" ] - "./buildkite/scripts/publish-deb.sh" + "./buildkite/scripts/debian/publish.sh" , label = "Publish Mina for ${DebianVersions.capitalName spec.debVersion} ${Profiles.toSuffixUppercase From e9f0463f1214c0267796ce5bb4d673bc1f24b514 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 27 Aug 2024 23:18:02 +0200 Subject: [PATCH 029/178] fix MinaArtifact.dhall --- buildkite/src/Command/MinaArtifact.dhall | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index f7e1144ac37..e4f15a74db3 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -222,7 +222,7 @@ let docker_step , deps = deps , service = Artifacts.dockerName Artifacts.Type.FunctionalTestSuite - , network = Network.lowerName Network.Type.Berkeley + , network = Network.lowerName Network.Type.Devnet , deb_codename = "${DebianVersions.lowerName spec.debVersion}" , build_flags = spec.buildFlags @@ -233,7 +233,6 @@ let docker_step spec.debVersion}${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment spec.buildFlags}-docker-image" - , network = "${Network.lowerName Network.Type.Devnet}" } ] } From 3510a79187fd88130812116e434c1b7213669e33 Mon Sep 17 00:00:00 2001 From: sai Date: Tue, 27 Aug 2024 17:57:44 -0400 Subject: [PATCH 030/178] replacing compatibility test berkeley hard code with devnet --- buildkite/src/Jobs/Test/DevelopCompatibility.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 99132224a5b..8660fdfa790 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -16,7 +16,7 @@ let Size = ../../Command/Size.dhall let dependsOn = [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" + , key = "daemon-devnet-bullseye-docker-image" } ] From 05ed10461dc61ae84b63118a5d7fc98969fea1e8 Mon Sep 17 00:00:00 2001 From: sai Date: Tue, 27 Aug 2024 18:57:39 -0400 Subject: [PATCH 031/178] Fixing compat script --- buildkite/scripts/check-compatibility.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index f1e571fdb9b..ca4acb81c08 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -11,7 +11,7 @@ function get_shas { function image_tag { SHA=$1 - IMAGE_TAG="$SHA-bullseye-berkeley" + IMAGE_TAG="$SHA-bullseye-devnet" } function download-docker { From 744d018954dd1f8b7c9acc8039f7ee50f88db708 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 28 Aug 2024 18:21:30 +0200 Subject: [PATCH 032/178] fix MinaArtifacts.dhal (add spec. prefix) --- buildkite/src/Command/MinaArtifact.dhall | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 870834fe89e..24f02c24147 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -234,9 +234,9 @@ let docker_step "${DebianVersions.lowerName spec.debVersion}" , step_key = "zkapp-test-transaction-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}-docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" } ] , FunctionalTestSuite = @@ -250,9 +250,9 @@ let docker_step , deb_profile = spec.profile , step_key = "test-suite-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}-docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" , network = "berkeley" } ] From a418936536db9e438ebf6c912ce38e6dd034536f Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 28 Aug 2024 20:55:38 +0200 Subject: [PATCH 033/178] Use PATH env whtn selecting paths for app find --- src/test/mina_automation/executor.ml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/test/mina_automation/executor.ml b/src/test/mina_automation/executor.ml index 2d52f940955..6fba161bf17 100644 --- a/src/test/mina_automation/executor.ml +++ b/src/test/mina_automation/executor.ml @@ -38,13 +38,23 @@ module Executor = struct let built_name t = Printf.sprintf "_build/default/%s" t.dune_name + let paths = + Option.value_map ~f:(String.split ~on:':') ~default:[] (Sys.getenv "PATH") + + let exists_at_path t prefix = + match%bind Sys.file_exists (prefix ^ "/" ^ t.official_name) with + | `Yes -> + Deferred.return (Some prefix) + | _ -> + Deferred.return None + let path t = match%bind Sys.file_exists (built_name t) with | `Yes -> Deferred.return (built_name t) | _ -> ( - match%bind Sys.file_exists t.official_name with - | `Yes -> + match%bind Deferred.List.find_map ~f:(exists_at_path t) paths with + | Some _ -> Deferred.return t.official_name | _ -> Deferred.return t.dune_name ) @@ -60,18 +70,7 @@ module Executor = struct ~metadata:[ ("app", `String (built_name t)) ] ; run_from_local t ~args ?env () | _ -> ( - let paths = - Option.value_map ~f:(String.split ~on:':') ~default:[] - (Sys.getenv "PATH") - in - let exists_at_path prefix = - match%bind Sys.file_exists (prefix ^ "/" ^ t.official_name) with - | `Yes -> - Deferred.return (Some prefix) - | _ -> - Deferred.return None - in - match%bind Deferred.List.find_map ~f:exists_at_path paths with + match%bind Deferred.List.find_map ~f:(exists_at_path t) paths with | Some prefix -> [%log debug] "running from %s" prefix ~metadata:[ ("app", `String t.official_name) ] ; From e81e748cc616af8b2427c25c317439df7ad06cc4 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 29 Aug 2024 00:06:27 +0200 Subject: [PATCH 034/178] add default for minimum_user_command_fee --- src/lib/runtime_config/runtime_config.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml index 2d8dfa305d5..f1e9097f636 100644 --- a/src/lib/runtime_config/runtime_config.ml +++ b/src/lib/runtime_config/runtime_config.ml @@ -473,7 +473,7 @@ module Json_layout = struct ; zkapp_cmd_limit_hardcap : int option [@default None] ; slot_tx_end : int option [@default None] ; slot_chain_end : int option [@default None] - ; minimum_user_command_fee : Currency.Fee.t option + ; minimum_user_command_fee : Currency.Fee.t option [@default None] ; network_id : string option [@default None] } [@@deriving yojson, fields] From 96dfa3bf27de2e7aa8bbb3ae799e6924b6155faf Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 29 Aug 2024 10:54:22 +0200 Subject: [PATCH 035/178] use constants in dhall rather than strings --- buildkite/src/Command/PatchArchiveTest.dhall | 3 +++ buildkite/src/Command/ReplayerTest.dhall | 1 + buildkite/src/Jobs/Test/PatchArchiveTest.dhall | 7 ++++++- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 4 ++-- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index 55d01eab194..4f4ce6a4c2c 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -6,6 +6,8 @@ let Size = ./Size.dhall let RunWithPostgres = ./RunWithPostgres.dhall +let Network = ../Constants/Network.dhall + in { step = \(dependsOn : List Command.TaggedKey.Type) -> Command.build @@ -17,6 +19,7 @@ in { step = ] "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.FunctionalTestSuite + (Some Network.Type.Devnet) "./scripts/patch-archive-test.sh" ] , label = "Archive: Patch Archive test" diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 4d2469079c1..846ecf859c9 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,6 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive + (Some Network.Type.Devnet) "./scripts/replayer-test.sh -i src/test/archive/sample_db/replayer_input_file.json -a mina-replayer" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index 8fc433daa57..f9eca18c7d5 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -12,11 +12,16 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye + (None Network.Type) Profiles.Type.Standard - "test-suite" + Artifacts.Type.FunctionalTestSuite in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index a336603b2a1..66e49e00544 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -55,12 +55,12 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Berkeley) + (Some Network.Type.Devnet) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-devnet" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] From e15b88bdbacd536542269cd8aaaad830be4ddc59 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 29 Aug 2024 11:58:30 +0200 Subject: [PATCH 036/178] make minimum_user_command_fee by default None --- src/lib/runtime_config/runtime_config.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml index 2d8dfa305d5..f1e9097f636 100644 --- a/src/lib/runtime_config/runtime_config.ml +++ b/src/lib/runtime_config/runtime_config.ml @@ -473,7 +473,7 @@ module Json_layout = struct ; zkapp_cmd_limit_hardcap : int option [@default None] ; slot_tx_end : int option [@default None] ; slot_chain_end : int option [@default None] - ; minimum_user_command_fee : Currency.Fee.t option + ; minimum_user_command_fee : Currency.Fee.t option [@default None] ; network_id : string option [@default None] } [@@deriving yojson, fields] From fe7630899545bc549ffc90f1382818c24876f719 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 29 Aug 2024 12:01:08 +0200 Subject: [PATCH 037/178] remove ConnectToBerkeley.dhall test job --- buildkite/scripts/connect-to-testnet.sh | 2 +- .../src/Jobs/Test/ConnectToBerkeley.dhall | 46 ------------------- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 buildkite/src/Jobs/Test/ConnectToBerkeley.dhall diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 8f2ee367b02..2a7934e5c48 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -7,7 +7,7 @@ if [[ $# -ne 3 ]]; then exit 1 fi -TESTNET_VERSION_NAME="berkeley" +TESTNET_VERSION_NAME="devnet" TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall deleted file mode 100644 index 2396f299df0..00000000000 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ /dev/null @@ -1,46 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let B = ../../External/Buildkite.dhall - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let Dockers = ../../Constants/DockerVersions.dhall - -let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - Profiles.Type.Standard - "daemon-berkeley" - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" - , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" - ] - , path = "Test" - , name = "ConnectToBerkeley" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - } - , steps = - [ ConnectToTestnet.step - dependsOn - "berkeley" - "40s" - "2m" - (B/SoftFail.Boolean True) - ] - } From b675e24c9787d2c09f019361d0bfd63f2336e0bb Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 29 Aug 2024 20:09:51 +0200 Subject: [PATCH 038/178] remove dust after BUILDKITE_BUILD_CHECKOUT_PATH --- buildkite/src/Command/RunWithPostgres.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index 5269fda5619..cd582fa8280 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -67,7 +67,7 @@ let runInDockerWithPostgresConn let outerDir : Text - = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH${networkOrDefault}" + = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" let minaDockerTag : Text From 43b1dee18833f8514a52eba9d60272bc5e09c4aa Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 29 Aug 2024 20:20:49 +0200 Subject: [PATCH 039/178] dhall lint --- buildkite/src/Command/RunWithPostgres.dhall | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index cd582fa8280..c953576bd10 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -2,10 +2,6 @@ let Prelude = ../External/Prelude.dhall let P = Prelude -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Text/concatMap = P.Text.concatMap let Cmd = ../Lib/Cmds.dhall @@ -40,15 +36,6 @@ let runInDockerWithPostgresConn let dbName = "archive" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - let pg_conn = "postgres://${user}:${password}@localhost:${port}/${dbName}" From 126ab66cf47af51f2840c522d8eb736b4011cb65 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 4 Sep 2024 21:13:51 +0200 Subject: [PATCH 040/178] Remove berkeley codename and fix step dependencies --- buildkite/src/Command/DockerImage.dhall | 57 ++++-- .../Command/HardforkPackageGeneration.dhall | 52 ++--- buildkite/src/Command/MinaArtifact.dhall | 179 ++++++++---------- .../src/Command/Promotion/PromoteDebian.dhall | 2 +- .../src/Command/Promotion/PromoteDocker.dhall | 2 +- buildkite/src/Command/ReplayerTest.dhall | 2 +- buildkite/src/Command/RunWithPostgres.dhall | 24 +-- buildkite/src/Constants/Artifacts.dhall | 47 +++-- buildkite/src/Constants/DebianVersions.dhall | 46 ++--- buildkite/src/Constants/DockerVersions.dhall | 46 +++-- buildkite/src/Constants/Network.dhall | 32 +++- .../Release/ItnOrchestratorArtifact.dhall | 48 ----- .../Jobs/Release/LeaderboardArtifact.dhall | 54 ------ ...dhall => MinaArtifactBullseyeDevnet.dhall} | 0 ...aArtifactBullseyeDevnetInstrumented.dhall} | 0 ... MinaArtifactBullseyeDevnetLightnet.dhall} | 0 ...hall => MinaArtifactBullseyeMainnet.dhall} | 3 +- ...al.dhall => MinaArtifactFocalDevnet.dhall} | 0 ...l.dhall => MinaArtifactFocalMainnet.dhall} | 3 +- ...inaArtifactOnlyDebianBullseyeDevnet.dhall} | 0 .../MinaToolchainArtifactBullseye.dhall | 9 +- .../Release/MinaToolchainArtifactFocal.dhall | 13 +- .../src/Jobs/Test/CheckGraphQLSchema.dhall | 3 + .../src/Jobs/Test/ConnectToBerkeley.dhall | 51 ----- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 1 + .../src/Jobs/Test/DevelopCompatibility.dhall | 14 +- buildkite/src/Jobs/Test/ReplayerTest.dhall | 2 +- .../Jobs/Test/RosettaIntegrationTests.dhall | 9 +- .../Test/RosettaIntegrationTestsLong.dhall | 7 +- .../src/Jobs/Test/RunSnarkProfiler.dhall | 3 + buildkite/src/Jobs/Test/SingleNodeTest.dhall | 4 + .../Jobs/Test/TestnetIntegrationTests.dhall | 4 +- .../Test/TestnetIntegrationTestsLong.dhall | 4 +- buildkite/src/Jobs/Test/VersionLint.dhall | 12 +- scripts/docker/helper.sh | 2 +- 35 files changed, 305 insertions(+), 430 deletions(-) delete mode 100644 buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall delete mode 100644 buildkite/src/Jobs/Release/LeaderboardArtifact.dhall rename buildkite/src/Jobs/Release/{MinaArtifactBullseye.dhall => MinaArtifactBullseyeDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeInstrumented.dhall => MinaArtifactBullseyeDevnetInstrumented.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeLightnet.dhall => MinaArtifactBullseyeDevnetLightnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactMainnetBullseye.dhall => MinaArtifactBullseyeMainnet.dhall} (88%) rename buildkite/src/Jobs/Release/{MinaArtifactFocal.dhall => MinaArtifactFocalDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactMainnetFocal.dhall => MinaArtifactFocalMainnet.dhall} (89%) rename buildkite/src/Jobs/Release/{MinaArtifactOnlyDebianBullseye.dhall => MinaArtifactOnlyDebianBullseyeDevnet.dhall} (100%) delete mode 100644 buildkite/src/Jobs/Test/ConnectToBerkeley.dhall diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index fc0dd9165fe..917a50fb621 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -10,8 +10,6 @@ let Size = ./Size.dhall let Profiles = ../Constants/Profiles.dhall -let Artifacts = ../Constants/Artifacts.dhall - let BuildFlags = ../Constants/BuildFlags.dhall let Cmd = ../Lib/Cmds.dhall @@ -20,59 +18,81 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianVersions = ../Constants/DebianVersions.dhall + let Network = ../Constants/Network.dhall +let Artifacts = ../Constants/Artifacts.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type , network : Text - , service : Text + , service : Artifacts.Type , version : Text , branch : Text , repo : Text , no_cache : Bool - , deb_codename : Text + , deb_codename : DebianVersions.DebVersion , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , step_key : Text + , step_key_suffix : Text , if : Optional B/If } , default = { deps = [] : List Command.TaggedKey.Type , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" - , service = Artifacts.dockerName Artifacts.Type.Daemon + , service = Artifacts.Type.Daemon , branch = "\\\${BUILDKITE_BRANCH}" , repo = "\\\${BUILDKITE_REPO}" - , deb_codename = "bullseye" + , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test , no_cache = False - , step_key = "daemon-standard-docker-image" + , step_key_suffix = "-docker-image" , if = None B/If } } +let stepKey = + \(spec : ReleaseSpec.Type) + -> "${Artifacts.lowerName + spec.service}${Profiles.toLabelSegment + spec.deb_profile}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" + +let stepLabel = + \(spec : ReleaseSpec.Type) + -> "Docker: ${Artifacts.capitalName + spec.service} ${spec.network} ${DebianVersions.capitalName + spec.deb_codename} ${Profiles.toSuffixUppercase + spec.deb_profile} ${BuildFlags.toSuffixUppercase + spec.build_flags}" + let generateStep = \(spec : ReleaseSpec.Type) - -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + -> let exportMinaDebCmd = + "export MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.deb_codename}" let maybeCacheOption = if spec.no_cache then "--no-cache" else "" let buildDockerCmd = "./scripts/docker/build.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --network ${spec.network}" ++ " --version ${spec.version}" ++ " --branch ${spec.branch}" ++ " ${maybeCacheOption} " - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" @@ -83,10 +103,11 @@ let generateStep = let releaseDockerCmd = "./scripts/docker/release.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName @@ -123,12 +144,16 @@ let generateStep = in Command.build Command.Config::{ , commands = commands - , label = "Docker: ${spec.step_key}" - , key = spec.step_key + , label = "${stepLabel spec}" + , key = "${stepKey spec}" , target = Size.XLarge , docker_login = Some DockerLogin::{=} , depends_on = spec.deps , if = spec.if } -in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } +in { generateStep = generateStep + , ReleaseSpec = ReleaseSpec + , stepKey = stepKey + , stepLabel = stepLabel + } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 183826d81b4..a8b9edf84b7 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -98,6 +98,17 @@ let pipeline "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" + let dockerSpec = + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = Artifacts.Type.Daemon + , network = network_name + , deb_codename = debVersion + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + } + in Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = [ S.everything ] @@ -155,20 +166,7 @@ let pipeline , key = "publish-hardfork-deb-pkg" , target = Size.Small } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - , deps = - [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Daemon - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" - } + , DockerImage.generateStep dockerSpec , Command.build Command.Config::{ , commands = @@ -182,10 +180,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -202,10 +197,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -214,28 +206,20 @@ let pipeline DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Archive + , service = Artifacts.Type.Archive , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_codename = debVersion , deb_profile = profile , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" } , DockerImage.generateStep DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Rosetta + , service = Artifacts.Type.Rosetta , network = network_name , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , deb_codename = debVersion } ] } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index e4f15a74db3..176e8380ae8 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -42,7 +42,7 @@ let MinaBuildSpec = , artifacts : List Artifacts.Type , debVersion : DebianVersions.DebVersion , profile : Profiles.Type - , networks : List Network.Type + , network : Network.Type , buildFlags : BuildFlags.Type , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type @@ -55,7 +55,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Devnet ] + , network = Network.Type.Devnet , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -63,6 +63,30 @@ let MinaBuildSpec = } } +let labelSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion} ${Network.capitalName + spec.network} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let nameSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion}${Network.capitalName + spec.network}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let keySuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}" + let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -79,22 +103,18 @@ let build_artifacts , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - , Network.foldMinaBuildMainnetEnv spec.networks + , Network.buildMainnetEnv spec.network ] # BuildFlags.buildEnvs spec.buildFlags ) "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts - spec.networks}" + spec.network}" # [ Cmd.run "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName spec.debVersion}" ] - , label = - "Build Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile} ${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , label = "Debian: Build ${labelSuffix spec}" , key = "build-deb-pkg" , target = Size.XLarge , retries = @@ -115,124 +135,97 @@ let docker_step DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags step_dep_name in merge { Daemon = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Daemon - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Daemon + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , LogProc = [] : List DockerImage.ReleaseSpec.Type + , Toolchain = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Toolchain + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] + , ItnOrchestrator = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.ItnOrchestrator + , network = Network.lowerName Network.Type.Devnet + , deb_repo = DebianRepo.Type.Local + } + ] + , Leaderboard = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Leaderboard + } + ] , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = Artifacts.dockerName Artifacts.Type.BatchTxn + , service = Artifacts.Type.BatchTxn , network = Network.lowerName Network.Type.Devnet - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "batch-txn-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = Artifacts.dockerName Artifacts.Type.Archive - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.Archive + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Rosetta = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Rosetta - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "rosetta-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Rosetta + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , deb_repo = DebianRepo.Type.Local + } + ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.ZkappTestTransaction + , service = Artifacts.Type.ZkappTestTransaction , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" + , deb_codename = spec.debVersion } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.FunctionalTestSuite + , service = Artifacts.Type.FunctionalTestSuite , network = Network.lowerName Network.Type.Devnet - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , deb_codename = spec.debVersion , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , step_key = - "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] } @@ -282,15 +275,13 @@ let publish_to_debian_repo = , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" ] "./buildkite/scripts/debian/publish.sh" - , label = - "Publish Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile}" + , label = "Debian: Publish ${labelSuffix spec}" , key = "publish-deb-pkg" , depends_on = DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags "build" @@ -305,11 +296,7 @@ let pipelineBuilder , spec = JobSpec::{ , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , name = "${spec.prefix}${nameSuffix spec}" , tags = spec.tags , mode = spec.mode } @@ -341,4 +328,6 @@ let pipeline in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline , MinaBuildSpec = MinaBuildSpec + , labelSuffix = labelSuffix + , keySuffix = keySuffix } diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 0019779bbba..2ac01eb006a 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -40,7 +40,7 @@ let PromoteDebianSpec = , version = "\\\\\$MINA_DEB_VERSION" , new_version = "\\\\\$MINA_DEB_VERSION" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index cbc60627dc8..7e56225d5ef 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -41,7 +41,7 @@ let PromoteDockerSpec = , new_tags = [] : List Text , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 6a742c7e7fd..d93f70a2677 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (None Network.Type) + Network.Type.Devnet "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index 5269fda5619..a9ad935999e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -2,10 +2,6 @@ let Prelude = ../External/Prelude.dhall let P = Prelude -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Text/concatMap = P.Text.concatMap let Cmd = ../Lib/Cmds.dhall @@ -17,16 +13,11 @@ let Artifacts = ../Constants/Artifacts.dhall let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn - : List Text - -> Text - -> Artifacts.Type - -> Optional Network.Type - -> Text - -> Cmd.Type + : List Text -> Text -> Artifacts.Type -> Network.Type -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Artifacts.Type) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(innerScript : Text) -> let port = "5432" @@ -40,15 +31,6 @@ let runInDockerWithPostgresConn let dbName = "archive" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - let pg_conn = "postgres://${user}:${password}@localhost:${port}/${dbName}" @@ -67,7 +49,7 @@ let runInDockerWithPostgresConn let outerDir : Text - = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH${networkOrDefault}" + = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" let minaDockerTag : Text diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index b41f52cf28c..673b14c3808 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -18,6 +18,9 @@ let Artifact | Rosetta | ZkappTestTransaction | FunctionalTestSuite + | Toolchain + | ItnOrchestrator + | Leaderboard > let AllButTests = @@ -33,7 +36,13 @@ let AllButTests = let Main = [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] -let All = AllButTests # [ Artifact.FunctionalTestSuite ] +let All = + AllButTests + # [ Artifact.FunctionalTestSuite + , Artifact.Toolchain + , Artifact.ItnOrchestrator + , Artifact.Leaderboard + ] let capitalName = \(artifact : Artifact) @@ -46,6 +55,9 @@ let capitalName = , Rosetta = "Rosetta" , ZkappTestTransaction = "ZkappTestTransaction" , FunctionalTestSuite = "FunctionalTestSuite" + , Toolchain = "Toolchain" + , ItnOrchestrator = "ItnOrchestrator" + , Leaderboard = "Leaderboard" } artifact @@ -60,6 +72,9 @@ let lowerName = , Rosetta = "rosetta" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "toolchain" + , ItnOrchestrator = "itnOrchestrator" + , Leaderboard = "leaderboard" } artifact @@ -74,6 +89,9 @@ let dockerName = , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" , FunctionalTestSuite = "mina-test-suite" + , Toolchain = "mina-toolchain" + , ItnOrchestrator = "itn-orchestrator" + , Leaderboard = "leaderboard" } artifact @@ -89,38 +107,32 @@ let toDebianName = , Rosetta = "rosetta_${Network.lowerName network}" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "" + , ItnOrchestrator = "" + , Leaderboard = "" } artifact let toDebianNames = \(artifacts : List Artifact) - -> \(networks : List Network.Type) + -> \(network : Network.Type) -> let list_of_list_of_debians = Prelude.List.map Artifact (List Text) ( \(a : Artifact) -> merge - { Daemon = - Prelude.List.map - Network.Type - Text - (\(n : Network.Type) -> toDebianName a n) - networks + { Daemon = [ toDebianName a network ] , Archive = [ "archive" ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] - , Rosetta = - Prelude.List.map - Network.Type - Text - ( \(n : Network.Type) - -> "rosetta_${Network.lowerName n}" - ) - networks + , Rosetta = [ toDebianName a network ] , ZkappTestTransaction = [ "zkapp_test_transaction" ] , FunctionalTestSuite = [ "functional_test_suite" ] + , Toolchain = [] : List Text + , ItnOrchestrator = [] : List Text + , Leaderboard = [] : List Text } a ) @@ -165,6 +177,9 @@ let dockerTag = "${version_and_codename}-${Network.lowerName network}" , ZkappTestTransaction = "${version_and_codename}" , FunctionalTestSuite = "${version_and_codename}" + , Toolchain = "${version_and_codename}" + , ItnOrchestrator = "${version_and_codename}" + , Leaderboard = "${version_and_codename}" } artifact diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index f3042f5f067..ba4bb08311b 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,6 +4,8 @@ let Optional/default = Prelude.Optional.default let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall @@ -33,6 +35,7 @@ let lowerName = let dependsOnStep = \(prefix : Optional Text) -> \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) @@ -40,47 +43,28 @@ let dependsOnStep = let prefix = Optional/default Text "MinaArtifact" prefix + let name = + "${prefix}${capitalName + debVersion}${Network.capitalName + network}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + in merge - { Bookworm = - [ { name = - "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Bullseye = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Jammy = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Focal = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] + { Bookworm = [ { name = name, key = "${step}-deb-pkg" } ] + , Bullseye = [ { name = name, key = "${step}-deb-pkg" } ] + , Jammy = [ { name = name, key = "${step}-deb-pkg" } ] + , Focal = [ { name = name, key = "${step}-deb-pkg" } ] } debVersion let dependsOn = \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> dependsOnStep (None Text) debVersion + network profile BuildFlags.Type.None "build" diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index a09718f19e7..926a077b77d 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,9 +1,3 @@ -let Prelude = ../External/Prelude.dhall - -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Profiles = ./Profiles.dhall let Artifacts = ./Artifacts.dhall @@ -36,43 +30,47 @@ let lowerName = let dependsOn = \(docker : Docker) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> let profileSuffix = Profiles.toSuffixUppercase profile + -> let network = "${Network.capitalName network}" + + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let prefix = "MinaArtifact" let suffix = "docker-image" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - - let key = - "${Artifacts.lowerName - binary}${networkOrDefault}-${lowerName docker}-${suffix}" + let key = "${Artifacts.lowerName binary}-${suffix}" in merge { Bookworm = - [ { name = "${prefix}${profileSuffix}", key = key } ] + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" + , key = key + } + ] , Bullseye = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Jammy = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] , Focal = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 92462c87ae3..86094577017 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -4,23 +4,18 @@ let List/any = Prelude.List.any let Network : Type - = < Devnet | Mainnet | Berkeley > + = < Devnet | Mainnet > let capitalName = \(network : Network) - -> merge - { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } - network + -> merge { Devnet = "Devnet", Mainnet = "Mainnet" } network let lowerName = \(network : Network) - -> merge - { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } - network + -> merge { Devnet = "devnet", Mainnet = "mainnet" } network let requiresMainnetBuild = - \(network : Network) - -> merge { Devnet = True, Mainnet = True, Berkeley = False } network + \(network : Network) -> merge { Devnet = False, Mainnet = True } network let foldMinaBuildMainnetEnv = \(networks : List Network) @@ -30,9 +25,28 @@ let foldMinaBuildMainnetEnv = else "MINA_BUILD_MAINNET=false" +let buildMainnetEnv = + \(network : Network) + -> if requiresMainnetBuild network + + then "MINA_BUILD_MAINNET=true" + + else "MINA_BUILD_MAINNET=false" + +let foldNames = + \(networks : List Network) + -> Prelude.List.fold + Network + networks + Text + (\(x : Network) -> \(y : Text) -> "${capitalName x}" ++ y) + "" + in { Type = Network , capitalName = capitalName , lowerName = lowerName , requiresMainnetBuild = requiresMainnetBuild , foldMinaBuildMainnetEnv = foldMinaBuildMainnetEnv + , buildMainnetEnv = buildMainnetEnv + , foldNames = foldNames } diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall deleted file mode 100644 index 8e9d8271d88..00000000000 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let DebianRepo = ../../Constants/DebianRepo.dhall - -let Network = ../../Constants/Network.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "itn-orchestrator" - , step_key = "itn-orchestrator-docker-image" - , network = "${Network.lowerName Network.Type.Devnet}" - , deb_repo = DebianRepo.Type.Local - , deps = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") - , S.strictlyStart (S.contains "src/app/itn_orchestrator") - ] - , path = "Release" - , name = "ItnOrchestratorArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = [ DockerImage.generateStep spec ] - } diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall deleted file mode 100644 index 2482ee5b0d3..00000000000 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ /dev/null @@ -1,54 +0,0 @@ -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Command = ../../Command/Base.dhall - -let Size = ../../Command/Size.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "leaderboard" - , step_key = "leaderboard-docker-image" - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/LeaderboardArtifact") - , S.strictlyStart (S.contains "frontend/leaderboard") - ] - , path = "Release" - , name = "LeaderboardArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run - "echo export MINA_VERSION=\$(cat frontend/leaderboard/package.json | jq '.version') > LEADERBOARD_DEPLOY_ENV" - ] - , label = "Setup Leaderboard docker image deploy environment" - , key = "setup-deploy-env" - , target = Size.Small - , artifact_paths = - [ S.contains "frontend/leaderboard/package.json" ] - } - , DockerImage.generateStep spec - ] - } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall similarity index 88% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index aa4037d8626..18633217d3c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -21,13 +21,12 @@ in Pipeline.build , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ] - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall similarity index 89% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 82d9d49a532..76829c42836 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -24,13 +24,12 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction ] , debVersion = DebianVersions.DebVersion.Focal - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 86e483e1e31..357da774d05 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -27,10 +31,9 @@ in Pipeline.build , steps = [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "bullseye" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Bullseye , no_cache = True - , step_key = "toolchain-bullseye-docker-image" } in DockerImage.generateStep toolchainBullseyeSpec diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall index ab0030f99f4..2ffe0472892 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -25,14 +29,13 @@ in Pipeline.build , tags = [ PipelineTag.Type.Toolchain ] } , steps = - [ let toolchainSpec = + [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "focal" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Focal , no_cache = True - , step_key = "toolchain-focal-docker-image" } - in DockerImage.generateStep toolchainSpec + in DockerImage.generateStep toolchainBullseyeSpec ] } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index dee29949cd5..bef4ebb1e47 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -12,9 +12,12 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall deleted file mode 100644 index 312e63f9435..00000000000 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ /dev/null @@ -1,51 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let B = ../../External/Buildkite.dhall - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let Artifacts = ../../Constants/Artifacts.dhall - -let Network = ../../Constants/Network.dhall - -let Dockers = ../../Constants/DockerVersions.dhall - -let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - (Some Network.Type.Berkeley) - Profiles.Type.Standard - Artifacts.Type.Daemon - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" - , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" - ] - , path = "Test" - , name = "ConnectToBerkeley" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - } - , steps = - [ ConnectToTestnet.step - dependsOn - "berkeley" - "40s" - "2m" - (B/SoftFail.Boolean True) - ] - } diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 97f05e0e19f..526ee022143 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -21,6 +21,7 @@ let Network = ../../Constants/Network.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 8660fdfa790..8bcecf6c27f 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -14,11 +14,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-devnet-bullseye-docker-image" - } - ] + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Lightnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 0e3d665e2ce..24ce7acf7e1 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -19,7 +19,7 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index a336603b2a1..02ed47087de 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -26,6 +26,8 @@ let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,12 +57,13 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Berkeley) + Network.Type.Devnet "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] @@ -71,7 +74,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index b1229798a96..ebfea0150c1 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -26,6 +26,8 @@ let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,7 +57,8 @@ in Pipeline.build , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-full.sh" ] @@ -66,7 +69,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index 0523f1e1e74..accae521318 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -16,11 +16,14 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall +let Network = ../../Constants/Network.dhall + let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 74959dd325a..e5bbae08182 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -14,6 +14,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,9 +23,11 @@ let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Lightnet # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index f08d92218a8..545e4590570 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Devnet) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Rosetta # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index d75e12b2a35..85a8b614ee2 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Devnet) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index cb1301cc01d..83bdca8bbbb 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -16,7 +16,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Standard let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> Command.Type diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 20b7bca95af..07cf02af46a 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -18,7 +18,7 @@ function export_base_image () { function export_version () { case "${SERVICE}" in - mina-daemon|mina-batch-txn) export VERSION="${VERSION}-${NETWORK##*=}" ;; + mina-daemon|mina-batch-txn|mina-rosetta) export VERSION="${VERSION}-${NETWORK##*=}" ;; *) ;; esac } From 9dfd385b0b5c7303b4dbea75a6f3511b413c8e90 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 5 Sep 2024 00:14:00 +0200 Subject: [PATCH 041/178] Remove berkeley codename and fix step dependencies --- buildkite/src/Command/DockerImage.dhall | 60 ++++-- .../Command/HardforkPackageGeneration.dhall | 52 ++--- buildkite/src/Command/MinaArtifact.dhall | 186 ++++++++++-------- .../src/Command/Promotion/PromoteDebian.dhall | 2 +- .../src/Command/Promotion/PromoteDocker.dhall | 2 +- buildkite/src/Command/ReplayerTest.dhall | 1 + buildkite/src/Command/RunWithPostgres.dhall | 5 +- buildkite/src/Constants/Artifacts.dhall | 47 +++-- buildkite/src/Constants/DebianVersions.dhall | 46 ++--- buildkite/src/Constants/DockerVersions.dhall | 35 +++- buildkite/src/Constants/Network.dhall | 32 ++- .../Release/ItnOrchestratorArtifact.dhall | 48 ----- .../Jobs/Release/LeaderboardArtifact.dhall | 54 ----- ...dhall => MinaArtifactBullseyeDevnet.dhall} | 0 ...aArtifactBullseyeDevnetInstrumented.dhall} | 0 ... MinaArtifactBullseyeDevnetLightnet.dhall} | 0 ...hall => MinaArtifactBullseyeMainnet.dhall} | 3 +- ...al.dhall => MinaArtifactFocalDevnet.dhall} | 0 ...l.dhall => MinaArtifactFocalMainnet.dhall} | 3 +- ...inaArtifactOnlyDebianBullseyeDevnet.dhall} | 0 .../MinaToolchainArtifactBullseye.dhall | 9 +- .../Release/MinaToolchainArtifactFocal.dhall | 13 +- .../src/Jobs/Test/CheckGraphQLSchema.dhall | 3 + buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 1 + .../src/Jobs/Test/DevelopCompatibility.dhall | 14 +- buildkite/src/Jobs/Test/ReplayerTest.dhall | 10 +- .../Jobs/Test/RosettaIntegrationTests.dhall | 15 +- .../Test/RosettaIntegrationTestsLong.dhall | 22 ++- .../src/Jobs/Test/RunSnarkProfiler.dhall | 3 + buildkite/src/Jobs/Test/SingleNodeTest.dhall | 4 + .../Jobs/Test/TestnetIntegrationTests.dhall | 13 +- .../Test/TestnetIntegrationTestsLong.dhall | 13 +- buildkite/src/Jobs/Test/VersionLint.dhall | 12 +- scripts/docker/helper.sh | 2 +- 34 files changed, 379 insertions(+), 331 deletions(-) delete mode 100644 buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall delete mode 100644 buildkite/src/Jobs/Release/LeaderboardArtifact.dhall rename buildkite/src/Jobs/Release/{MinaArtifactBullseye.dhall => MinaArtifactBullseyeDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeInstrumented.dhall => MinaArtifactBullseyeDevnetInstrumented.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeLightnet.dhall => MinaArtifactBullseyeDevnetLightnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactMainnetBullseye.dhall => MinaArtifactBullseyeMainnet.dhall} (88%) rename buildkite/src/Jobs/Release/{MinaArtifactFocal.dhall => MinaArtifactFocalDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactMainnetFocal.dhall => MinaArtifactFocalMainnet.dhall} (89%) rename buildkite/src/Jobs/Release/{MinaArtifactOnlyDebianBullseye.dhall => MinaArtifactOnlyDebianBullseyeDevnet.dhall} (100%) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index cc5131772c4..d2030dc8d32 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -18,57 +18,82 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianVersions = ../Constants/DebianVersions.dhall + +let Network = ../Constants/Network.dhall + +let Artifacts = ../Constants/Artifacts.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type , network : Text - , service : Text + , service : Artifacts.Type , version : Text , branch : Text , repo : Text , no_cache : Bool - , deb_codename : Text + , deb_codename : DebianVersions.DebVersion , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , step_key : Text + , step_key_suffix : Text , if : Optional B/If } , default = { deps = [] : List Command.TaggedKey.Type - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" - , service = "\\\${MINA_SERVICE}" + , service = Artifacts.Type.Daemon , branch = "\\\${BUILDKITE_BRANCH}" , repo = "\\\${BUILDKITE_REPO}" - , deb_codename = "bullseye" + , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test , no_cache = False - , step_key = "daemon-standard-docker-image" + , step_key_suffix = "-docker-image" , if = None B/If } } +let stepKey = + \(spec : ReleaseSpec.Type) + -> "${Artifacts.lowerName + spec.service}${Profiles.toLabelSegment + spec.deb_profile}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" + +let stepLabel = + \(spec : ReleaseSpec.Type) + -> "Docker: ${Artifacts.capitalName + spec.service} ${spec.network} ${DebianVersions.capitalName + spec.deb_codename} ${Profiles.toSuffixUppercase + spec.deb_profile} ${BuildFlags.toSuffixUppercase + spec.build_flags}" + let generateStep = \(spec : ReleaseSpec.Type) - -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + -> let exportMinaDebCmd = + "export MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.deb_codename}" let maybeCacheOption = if spec.no_cache then "--no-cache" else "" let buildDockerCmd = "./scripts/docker/build.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" + ++ " --network ${spec.network}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" ++ " --branch ${spec.branch}" ++ " ${maybeCacheOption} " - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" @@ -79,10 +104,11 @@ let generateStep = let releaseDockerCmd = "./scripts/docker/release.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName @@ -119,12 +145,16 @@ let generateStep = in Command.build Command.Config::{ , commands = commands - , label = "Docker: ${spec.step_key}" - , key = spec.step_key + , label = "${stepLabel spec}" + , key = "${stepKey spec}" , target = Size.XLarge , docker_login = Some DockerLogin::{=} , depends_on = spec.deps , if = spec.if } -in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } +in { generateStep = generateStep + , ReleaseSpec = ReleaseSpec + , stepKey = stepKey + , stepLabel = stepLabel + } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 7353a8e9bb3..765b80e5996 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -96,6 +96,17 @@ let pipeline "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" + let dockerSpec = + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = Artifacts.Type.Daemon + , network = network_name + , deb_codename = debVersion + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + } + in Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = [ S.everything ] @@ -153,20 +164,7 @@ let pipeline , key = "publish-hardfork-deb-pkg" , target = Size.Small } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - , deps = - [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = "mina-daemon" - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" - } + , DockerImage.generateStep dockerSpec , Command.build Command.Config::{ , commands = @@ -180,10 +178,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -200,10 +195,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -212,28 +204,20 @@ let pipeline DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = "mina-archive" + , service = Artifacts.Type.Archive , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_codename = debVersion , deb_profile = profile , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" } , DockerImage.generateStep DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = "mina-rosetta" + , service = Artifacts.Type.Rosetta , network = network_name , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , deb_codename = debVersion } ] } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 4c6310998d1..d7049b27a87 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -42,7 +42,7 @@ let MinaBuildSpec = , artifacts : List Artifacts.Type , debVersion : DebianVersions.DebVersion , profile : Profiles.Type - , networks : List Network.Type + , network : Network.Type , buildFlags : BuildFlags.Type , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type @@ -55,7 +55,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Devnet ] + , network = Network.Type.Devnet , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -63,6 +63,30 @@ let MinaBuildSpec = } } +let labelSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion} ${Network.capitalName + spec.network} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let nameSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion}${Network.capitalName + spec.network}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let keySuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}" + let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -79,22 +103,18 @@ let build_artifacts , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - , Network.foldMinaBuildMainnetEnv spec.networks + , Network.buildMainnetEnv spec.network ] # BuildFlags.buildEnvs spec.buildFlags ) "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts - spec.networks}" + spec.network}" # [ Cmd.run "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName spec.debVersion}" ] - , label = - "Build Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile} ${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , label = "Debian: Build ${labelSuffix spec}" , key = "build-deb-pkg" , target = Size.XLarge , retries = @@ -145,115 +165,97 @@ let docker_step DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags step_dep_name in merge { Daemon = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = "mina-daemon" - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Daemon + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , LogProc = [] : List DockerImage.ReleaseSpec.Type + , Toolchain = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Toolchain + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] + , ItnOrchestrator = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.ItnOrchestrator + , network = Network.lowerName Network.Type.Devnet + , deb_repo = DebianRepo.Type.Local + } + ] + , Leaderboard = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Leaderboard + } + ] , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-batch-txn" - , network = "devnet" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.BatchTxn + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "batch-txn-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-archive" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.Archive + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Rosetta = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-rosetta" - , network = "devnet" - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local + , service = Artifacts.Type.Rosetta + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" + , deb_repo = DebianRepo.Type.Local } ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-zkapp-test-transaction" + , service = Artifacts.Type.ZkappTestTransaction , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + , deb_codename = spec.debVersion } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-test-suite" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.FunctionalTestSuite + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , step_key = - "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" - , network = "devnet" } ] } @@ -288,6 +290,34 @@ let docker_commands ) flattened_docker_steps +let publish_to_debian_repo = + \(spec : MinaBuildSpec.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" + ] + "./buildkite/scripts/debian/publish.sh" + , label = "Debian: Publish ${labelSuffix spec}" + , key = "publish-deb-pkg" + , depends_on = + DebianVersions.dependsOnStep + (Some spec.prefix) + spec.debVersion + spec.network + spec.profile + spec.buildFlags + "build" + , target = Size.Small + } + let pipelineBuilder : MinaBuildSpec.Type -> List Command.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) @@ -296,11 +326,7 @@ let pipelineBuilder , spec = JobSpec::{ , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , name = "${spec.prefix}${nameSuffix spec}" , tags = spec.tags , mode = spec.mode } @@ -332,4 +358,6 @@ let pipeline in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline , MinaBuildSpec = MinaBuildSpec + , labelSuffix = labelSuffix + , keySuffix = keySuffix } diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 0019779bbba..2ac01eb006a 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -40,7 +40,7 @@ let PromoteDebianSpec = , version = "\\\\\$MINA_DEB_VERSION" , new_version = "\\\\\$MINA_DEB_VERSION" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index cbc60627dc8..7e56225d5ef 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -41,7 +41,7 @@ let PromoteDockerSpec = , new_tags = [] : List Text , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 02c02978836..9141b49b978 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -15,6 +15,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive + Network.Type.Devnet "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index f2346e65a87..a9ad935999e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -10,11 +10,14 @@ let ContainerImages = ../Constants/ContainerImages.dhall let Artifacts = ../Constants/Artifacts.dhall +let Network = ../Constants/Network.dhall + let runInDockerWithPostgresConn - : List Text -> Text -> Artifacts.Type -> Text -> Cmd.Type + : List Text -> Text -> Artifacts.Type -> Network.Type -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Artifacts.Type) + -> \(network : Network.Type) -> \(innerScript : Text) -> let port = "5432" diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index 8397d0b27a5..797a7edafb4 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -18,6 +18,9 @@ let Artifact | Rosetta | ZkappTestTransaction | FunctionalTestSuite + | Toolchain + | ItnOrchestrator + | Leaderboard > let AllButTests = @@ -33,7 +36,13 @@ let AllButTests = let Main = [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] -let All = AllButTests # [ Artifact.FunctionalTestSuite ] +let All = + AllButTests + # [ Artifact.FunctionalTestSuite + , Artifact.Toolchain + , Artifact.ItnOrchestrator + , Artifact.Leaderboard + ] let capitalName = \(artifact : Artifact) @@ -46,6 +55,9 @@ let capitalName = , Rosetta = "Rosetta" , ZkappTestTransaction = "ZkappTestTransaction" , FunctionalTestSuite = "FunctionalTestSuite" + , Toolchain = "Toolchain" + , ItnOrchestrator = "ItnOrchestrator" + , Leaderboard = "Leaderboard" } artifact @@ -60,6 +72,9 @@ let lowerName = , Rosetta = "rosetta" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "toolchain" + , ItnOrchestrator = "itnOrchestrator" + , Leaderboard = "leaderboard" } artifact @@ -74,6 +89,9 @@ let dockerName = , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" , FunctionalTestSuite = "mina-test-suite" + , Toolchain = "mina-toolchain" + , ItnOrchestrator = "itn-orchestrator" + , Leaderboard = "leaderboard" } artifact @@ -89,38 +107,32 @@ let toDebianName = , Rosetta = "rosetta_${Network.lowerName network}" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "" + , ItnOrchestrator = "" + , Leaderboard = "" } artifact let toDebianNames = \(artifacts : List Artifact) - -> \(networks : List Network.Type) + -> \(network : Network.Type) -> let list_of_list_of_debians = Prelude.List.map Artifact (List Text) ( \(a : Artifact) -> merge - { Daemon = - Prelude.List.map - Network.Type - Text - (\(n : Network.Type) -> toDebianName a n) - networks + { Daemon = [ toDebianName a network ] , Archive = [ "archive" ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] - , Rosetta = - Prelude.List.map - Network.Type - Text - ( \(n : Network.Type) - -> "rosetta_${Network.lowerName n}" - ) - networks + , Rosetta = [ toDebianName a network ] , ZkappTestTransaction = [ "zkapp_test_transaction" ] , FunctionalTestSuite = [ "functional_test_suite" ] + , Toolchain = [] : List Text + , ItnOrchestrator = [] : List Text + , Leaderboard = [] : List Text } a ) @@ -164,6 +176,9 @@ let dockerTag = , Rosetta = "${version_and_codename}" , ZkappTestTransaction = "${version_and_codename}" , FunctionalTestSuite = "${version_and_codename}" + , Toolchain = "${version_and_codename}" + , ItnOrchestrator = "${version_and_codename}" + , Leaderboard = "${version_and_codename}" } artifact diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index f3042f5f067..ba4bb08311b 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,6 +4,8 @@ let Optional/default = Prelude.Optional.default let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall @@ -33,6 +35,7 @@ let lowerName = let dependsOnStep = \(prefix : Optional Text) -> \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) @@ -40,47 +43,28 @@ let dependsOnStep = let prefix = Optional/default Text "MinaArtifact" prefix + let name = + "${prefix}${capitalName + debVersion}${Network.capitalName + network}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + in merge - { Bookworm = - [ { name = - "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Bullseye = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Jammy = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Focal = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] + { Bookworm = [ { name = name, key = "${step}-deb-pkg" } ] + , Bullseye = [ { name = name, key = "${step}-deb-pkg" } ] + , Jammy = [ { name = name, key = "${step}-deb-pkg" } ] + , Focal = [ { name = name, key = "${step}-deb-pkg" } ] } debVersion let dependsOn = \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> dependsOnStep (None Text) debVersion + network profile BuildFlags.Type.None "build" diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index fe094476b0f..595da9737e3 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -26,33 +26,48 @@ let lowerName = let dependsOn = \(docker : Docker) + -> \(network : Network.Type) -> \(profile : Profiles.Type) - -> \(binary : Text) - -> let profileSuffix = Profiles.toSuffixUppercase profile + -> \(binary : Artifacts.Type) + -> let network = "${Network.capitalName network}" + + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let prefix = "MinaArtifact" let suffix = "docker-image" + let key = "${Artifacts.lowerName binary}-${suffix}" + in merge { Bookworm = - [ { name = "${prefix}${profileSuffix}" - , key = "${binary}-${lowerName docker}-${suffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" + , key = key } ] , Bullseye = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" - , key = "${binary}-${lowerName docker}-${suffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" + , key = key } ] , Jammy = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" - , key = "${binary}-${lowerName docker}-${suffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" + , key = key } ] , Focal = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" - , key = "${binary}-${lowerName docker}-${suffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" + , key = key } ] } diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 92462c87ae3..86094577017 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -4,23 +4,18 @@ let List/any = Prelude.List.any let Network : Type - = < Devnet | Mainnet | Berkeley > + = < Devnet | Mainnet > let capitalName = \(network : Network) - -> merge - { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } - network + -> merge { Devnet = "Devnet", Mainnet = "Mainnet" } network let lowerName = \(network : Network) - -> merge - { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } - network + -> merge { Devnet = "devnet", Mainnet = "mainnet" } network let requiresMainnetBuild = - \(network : Network) - -> merge { Devnet = True, Mainnet = True, Berkeley = False } network + \(network : Network) -> merge { Devnet = False, Mainnet = True } network let foldMinaBuildMainnetEnv = \(networks : List Network) @@ -30,9 +25,28 @@ let foldMinaBuildMainnetEnv = else "MINA_BUILD_MAINNET=false" +let buildMainnetEnv = + \(network : Network) + -> if requiresMainnetBuild network + + then "MINA_BUILD_MAINNET=true" + + else "MINA_BUILD_MAINNET=false" + +let foldNames = + \(networks : List Network) + -> Prelude.List.fold + Network + networks + Text + (\(x : Network) -> \(y : Text) -> "${capitalName x}" ++ y) + "" + in { Type = Network , capitalName = capitalName , lowerName = lowerName , requiresMainnetBuild = requiresMainnetBuild , foldMinaBuildMainnetEnv = foldMinaBuildMainnetEnv + , buildMainnetEnv = buildMainnetEnv + , foldNames = foldNames } diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall deleted file mode 100644 index 8e9d8271d88..00000000000 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let DebianRepo = ../../Constants/DebianRepo.dhall - -let Network = ../../Constants/Network.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "itn-orchestrator" - , step_key = "itn-orchestrator-docker-image" - , network = "${Network.lowerName Network.Type.Devnet}" - , deb_repo = DebianRepo.Type.Local - , deps = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") - , S.strictlyStart (S.contains "src/app/itn_orchestrator") - ] - , path = "Release" - , name = "ItnOrchestratorArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = [ DockerImage.generateStep spec ] - } diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall deleted file mode 100644 index 2482ee5b0d3..00000000000 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ /dev/null @@ -1,54 +0,0 @@ -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Command = ../../Command/Base.dhall - -let Size = ../../Command/Size.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "leaderboard" - , step_key = "leaderboard-docker-image" - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/LeaderboardArtifact") - , S.strictlyStart (S.contains "frontend/leaderboard") - ] - , path = "Release" - , name = "LeaderboardArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run - "echo export MINA_VERSION=\$(cat frontend/leaderboard/package.json | jq '.version') > LEADERBOARD_DEPLOY_ENV" - ] - , label = "Setup Leaderboard docker image deploy environment" - , key = "setup-deploy-env" - , target = Size.Small - , artifact_paths = - [ S.contains "frontend/leaderboard/package.json" ] - } - , DockerImage.generateStep spec - ] - } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall similarity index 88% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index aa4037d8626..18633217d3c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -21,13 +21,12 @@ in Pipeline.build , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ] - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall similarity index 89% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 82d9d49a532..76829c42836 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -24,13 +24,12 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction ] , debVersion = DebianVersions.DebVersion.Focal - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 86e483e1e31..357da774d05 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -27,10 +31,9 @@ in Pipeline.build , steps = [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "bullseye" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Bullseye , no_cache = True - , step_key = "toolchain-bullseye-docker-image" } in DockerImage.generateStep toolchainBullseyeSpec diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall index ab0030f99f4..2ffe0472892 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -25,14 +29,13 @@ in Pipeline.build , tags = [ PipelineTag.Type.Toolchain ] } , steps = - [ let toolchainSpec = + [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "focal" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Focal , no_cache = True - , step_key = "toolchain-focal-docker-image" } - in DockerImage.generateStep toolchainSpec + in DockerImage.generateStep toolchainBullseyeSpec ] } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index dee29949cd5..bef4ebb1e47 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -12,9 +12,12 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 9f744d1f382..d06aae00efc 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -19,6 +19,7 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 99132224a5b..8bcecf6c27f 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -14,11 +14,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Lightnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 1d538bebaed..24ce7acf7e1 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -12,8 +12,16 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let dependsOn = - Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" + Dockers.dependsOn + Dockers.Type.Bullseye + Network.Type.Devnet + Profiles.Type.Standard + Artifacts.Type.Archive in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 067a536251b..02ed47087de 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -18,12 +18,16 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall -let Artifacts = ../../Constants/Artifacts.dhall - let RunWithPostgres = ../../Command/RunWithPostgres.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -53,11 +57,13 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta + Network.Type.Devnet "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] @@ -68,8 +74,9 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye + network Profiles.Type.Standard - "rosetta" + Artifacts.Type.Rosetta } ] } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index eaee44624b3..ebfea0150c1 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -16,8 +16,18 @@ let Command = ../../Command/Base.dhall let Size = ../../Command/Size.dhall +let Profiles = ../../Constants/Profiles.dhall + +let Dockers = ../../Constants/DockerVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -47,7 +57,8 @@ in Pipeline.build , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-full.sh" ] @@ -56,10 +67,11 @@ in Pipeline.build , soft_fail = Some (B/SoftFail.Boolean True) , target = Size.Small , depends_on = - [ { name = "MinaArtifactBullseye" - , key = "rosetta-bullseye-docker-image" - } - ] + Dockers.dependsOn + Dockers.Type.Bullseye + network + Profiles.Type.Standard + Artifacts.Type.Rosetta } ] } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index 0523f1e1e74..accae521318 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -16,11 +16,14 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall +let Network = ../../Constants/Network.dhall + let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 74959dd325a..e5bbae08182 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -14,6 +14,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,9 +23,11 @@ let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Lightnet # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 0d7099909ad..545e4590570 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -14,12 +14,21 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye + Network.Type.Devnet + Profiles.Type.Standard + Artifacts.Type.Rosetta + # Dockers.dependsOn + Dockers.Type.Bullseye + Network.Type.Devnet Profiles.Type.Standard - "daemon-devnet" - # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" + Artifacts.Type.Archive in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 55df0195024..85a8b614ee2 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -14,12 +14,21 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye + Network.Type.Devnet + Profiles.Type.Standard + Artifacts.Type.Daemon + # Dockers.dependsOn + Dockers.Type.Bullseye + Network.Type.Devnet Profiles.Type.Standard - "daemon-devnet" - # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" + Artifacts.Type.Archive in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index cb1301cc01d..83bdca8bbbb 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -16,7 +16,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Standard let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> Command.Type diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 20b7bca95af..07cf02af46a 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -18,7 +18,7 @@ function export_base_image () { function export_version () { case "${SERVICE}" in - mina-daemon|mina-batch-txn) export VERSION="${VERSION}-${NETWORK##*=}" ;; + mina-daemon|mina-batch-txn|mina-rosetta) export VERSION="${VERSION}-${NETWORK##*=}" ;; *) ;; esac } From 62423466de3a9335a98c22b126a4282e7696de0b Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 5 Sep 2024 00:18:30 +0200 Subject: [PATCH 042/178] Port develop changes --- .../Command/HardforkPackageGeneration.dhall | 2 ++ buildkite/src/Command/MinaArtifact.dhall | 30 ------------------- buildkite/src/Command/ReplayerTest.dhall | 2 ++ buildkite/src/Constants/DockerVersions.dhall | 4 +++ buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 2 ++ 5 files changed, 10 insertions(+), 30 deletions(-) diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 765b80e5996..9e9af392e52 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -36,6 +36,8 @@ let Toolchain = ../Constants/Toolchain.dhall let Network = ../Constants/Network.dhall +let Artifacts = ../Constants/Artifacts.dhall + let Spec = { Type = { codename : DebianVersions.DebVersion diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index d7049b27a87..176e8380ae8 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -125,36 +125,6 @@ let build_artifacts ] } -let publish_to_debian_repo = - \(spec : MinaBuildSpec.Type) - -> Command.build - Command.Config::{ - , commands = - Toolchain.select - spec.toolchainSelectMode - spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] - "./buildkite/scripts/debian/publish.sh" - , label = - "Publish Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile}" - , key = "publish-deb-pkg" - , depends_on = - DebianVersions.dependsOnStep - (Some spec.prefix) - spec.debVersion - spec.profile - spec.buildFlags - "build" - , target = Size.Small - } - let docker_step : Artifacts.Type -> MinaBuildSpec.Type -> List DockerImage.ReleaseSpec.Type = \(artifact : Artifacts.Type) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 9141b49b978..791b5aac246 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -1,5 +1,7 @@ let Artifacts = ../Constants/Artifacts.dhall +let Network = ../Constants/Network.dhall + let Command = ./Base.dhall let Size = ./Size.dhall diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 595da9737e3..71168d53614 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,5 +1,9 @@ let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + +let Artifacts = ./Artifacts.dhall + let Docker : Type = < Bookworm | Bullseye | Jammy | Focal > diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index d06aae00efc..50d78117bfc 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -14,6 +14,8 @@ let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let DebianVersions = ../../Constants/DebianVersions.dhall let dependsOn = From 402c5cff856ff742ce39bc9f2e0802605c4e60f1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 5 Sep 2024 08:31:14 +0200 Subject: [PATCH 043/178] lint dhall --- buildkite/src/Command/HardforkPackageGeneration.dhall | 2 -- buildkite/src/Command/ReplayerTest.dhall | 2 -- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 2 -- 3 files changed, 6 deletions(-) diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 2b7d7398c96..9e9af392e52 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -30,8 +30,6 @@ let DebianVersions = ../Constants/DebianVersions.dhall let DebianRepo = ../Constants/DebianRepo.dhall -let Artifacts = ../Constants/Artifacts.dhall - let Profiles = ../Constants/Profiles.dhall let Toolchain = ../Constants/Toolchain.dhall diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index c9b2b7e1587..d93f70a2677 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -1,7 +1,5 @@ let Artifacts = ../Constants/Artifacts.dhall -let Network = ../Constants/Network.dhall - let Command = ./Base.dhall let Size = ./Size.dhall diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index dd8a50d4733..526ee022143 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -14,8 +14,6 @@ let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall let Profiles = ../../Constants/Profiles.dhall -let Network = ../../Constants/Network.dhall - let DebianVersions = ../../Constants/DebianVersions.dhall let Network = ../../Constants/Network.dhall From fdbd3b43f63a0f20a4d2d51f81decba65152b380 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 5 Sep 2024 12:56:27 +0200 Subject: [PATCH 044/178] fix compilation --- src/app/replayer/replayer.ml | 3 ++- src/lib/cli_lib/flag.ml | 5 +++++ src/lib/cli_lib/flag.mli | 2 ++ .../logger/file_system/logger_file_system.ml | 18 ++++++++++++++++-- .../logger/file_system/logger_file_system.mli | 2 ++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/app/replayer/replayer.ml b/src/app/replayer/replayer.ml index 5ecb12b6294..a11615e0aca 100644 --- a/src/app/replayer/replayer.ml +++ b/src/app/replayer/replayer.ml @@ -639,7 +639,8 @@ let main ~input_file ~output_file_opt ~archive_uri ~continue_on_error Option.iter log_filename ~f:(fun log_filename -> Logger.Consumer_registry.register ~id:"default" ~processor:(Logger.Processor.raw ~log_level:file_log_level ()) - ~transport:(Logger_file_system.evergrowing ~log_filename) ) ; + ~transport:(Logger_file_system.evergrowing ~log_filename) + () ) ; let logger = Logger.create () in let json = Yojson.Safe.from_file input_file in let input = diff --git a/src/lib/cli_lib/flag.ml b/src/lib/cli_lib/flag.ml index 65deba405a3..c96200feb9e 100644 --- a/src/lib/cli_lib/flag.ml +++ b/src/lib/cli_lib/flag.ml @@ -334,6 +334,11 @@ module Log = struct %d)" Default.file_log_rotations ) (optional_with_default Default.file_log_rotations int) + + let file = + let open Command.Param in + flag "--log-file" ~aliases:[ "log-file" ] + ~doc:"FILE Set log file (stores JSON)" (optional string) end type signed_command_common = diff --git a/src/lib/cli_lib/flag.mli b/src/lib/cli_lib/flag.mli index 70b2c532156..69ff38f86f3 100644 --- a/src/lib/cli_lib/flag.mli +++ b/src/lib/cli_lib/flag.mli @@ -74,6 +74,8 @@ module Log : sig val file_log_level : Logger.Level.t Command.Param.t + val file : string option Command.Param.t + val file_log_rotations : int Command.Param.t end diff --git a/src/lib/logger/file_system/logger_file_system.ml b/src/lib/logger/file_system/logger_file_system.ml index 327be0ca470..5b7fed7f9b2 100644 --- a/src/lib/logger/file_system/logger_file_system.ml +++ b/src/lib/logger/file_system/logger_file_system.ml @@ -1,10 +1,10 @@ open Core +let log_perm = 0o644 + module Dumb_logrotate = struct open Core.Unix - let log_perm = 0o644 - type t = { directory : string ; log_filename : string @@ -68,6 +68,20 @@ let dumb_logrotate ~directory ~log_filename ~max_size ~num_rotate = (module Dumb_logrotate) (Dumb_logrotate.create ~directory ~log_filename ~max_size ~num_rotate) +let evergrowing ~log_filename = + let open Unix in + Logger.Transport.create + ( module struct + type t = File_descr.t + + let transport t str = + let str = str ^ "\n" in + let len = String.length str in + if write t ~buf:(Bytes.of_string str) ~len <> len then + printf "unexpected error writing to log" + end ) + (openfile ~perm:log_perm ~mode:[ O_RDWR; O_APPEND; O_CREAT ] log_filename) + let time_pretty_to_string timestamp = Time.format timestamp "%Y-%m-%d %H:%M:%S UTC" ~zone:Time.Zone.utc diff --git a/src/lib/logger/file_system/logger_file_system.mli b/src/lib/logger/file_system/logger_file_system.mli index c450e816104..3868bffc8c9 100644 --- a/src/lib/logger/file_system/logger_file_system.mli +++ b/src/lib/logger/file_system/logger_file_system.mli @@ -11,6 +11,8 @@ val dumb_logrotate : -> num_rotate:int -> Logger.Transport.t +val evergrowing : log_filename:string -> Logger.Transport.t + (** Pretty printer for time, in "%Y-%m-%d %H:%M:%S UTC" format. On linking this library, this is used to override the JS-safe From 4e1fe257ea7fb88a97821183e78c0fa5960e08be Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 9 Sep 2024 23:02:33 +0200 Subject: [PATCH 045/178] fix dhall --- buildkite/src/Command/PatchArchiveTest.dhall | 3 +++ buildkite/src/Jobs/Test/PatchArchiveTest.dhall | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index 55d01eab194..e94cb9c2adf 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -4,6 +4,8 @@ let Command = ./Base.dhall let Size = ./Size.dhall +let Network = ../Constants/Network.dhall + let RunWithPostgres = ./RunWithPostgres.dhall in { step = @@ -17,6 +19,7 @@ in { step = ] "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.FunctionalTestSuite + (Some Network.Type.Devnet) "./scripts/patch-archive-test.sh" ] , label = "Archive: Patch Archive test" diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index 8fc433daa57..e6b468d843b 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -10,13 +10,18 @@ let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + let Dockers = ../../Constants/DockerVersions.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye + (None Network.Type) Profiles.Type.Standard - "test-suite" + Artifacts.Type.FunctionalTestSuite in Pipeline.build Pipeline.Config::{ From f57ddbfbf5e73daf4ee2f5b8957f5b5950d3da4a Mon Sep 17 00:00:00 2001 From: SanabriaRusso Date: Tue, 10 Sep 2024 10:59:09 +0200 Subject: [PATCH 046/178] Adjusting Dune's simultaneous compilations to new Infra --- buildkite/scripts/unit-test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildkite/scripts/unit-test.sh b/buildkite/scripts/unit-test.sh index de885193086..b38ad7d94a8 100755 --- a/buildkite/scripts/unit-test.sh +++ b/buildkite/scripts/unit-test.sh @@ -21,10 +21,10 @@ export LIBP2P_NIXLESS=1 PATH=/usr/lib/go/bin:$PATH GO=/usr/lib/go/bin/go time make build echo "--- Build all targets" -dune build "${path}" --profile="${profile}" -j16 +dune build "${path}" --profile="${profile}" -j8 echo "--- Check for changes to verification keys" -time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j16 +time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j8 # Turn on the proof-cache assertion, so that CI will fail if the proofs need to # be updated. @@ -35,8 +35,8 @@ export ERROR_ON_PROOF=true # skip running all of the tests that have already succeeded, since dune will # only retry those tests that failed. echo "--- Run unit tests" -time dune runtest "${path}" --profile="${profile}" -j16 || \ +time dune runtest "${path}" --profile="${profile}" -j8 || \ (./scripts/link-coredumps.sh && \ echo "--- Retrying failed unit tests" && \ - time dune runtest "${path}" --profile="${profile}" -j16 || \ + time dune runtest "${path}" --profile="${profile}" -j8 || \ (./scripts/link-coredumps.sh && false)) From 80e61228b73e88d7728503c1020aafbbb04530f1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 10 Sep 2024 13:21:09 +0200 Subject: [PATCH 047/178] remove --no-merges from uploading type shape to gs --- buildkite/src/Jobs/Test/VersionLint.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index 8d693ab291c..971dd12d44e 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -45,7 +45,7 @@ let buildTestCmd ([] : List Text) "buildkite/scripts/dump-mina-type-shapes.sh" # [ Cmd.run - "gsutil cp \$(git log -n 1 --format=%h --abbrev=7 --no-merges)-type_shape.txt \$MINA_TYPE_SHAPE gs://mina-type-shapes" + "gsutil cp \$(git log -n 1 --format=%h --abbrev=7)-type_shape.txt \$MINA_TYPE_SHAPE gs://mina-type-shapes" ] # RunInToolchain.runInToolchain ([] : List Text) From 26bba68f2bcd094579e00d0cbbae998b9c6dc18b Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 12 Sep 2024 21:44:55 +0200 Subject: [PATCH 048/178] fix dhall --- buildkite/src/Command/PatchArchiveTest.dhall | 2 +- buildkite/src/Command/ReplayerTest.dhall | 2 +- buildkite/src/Jobs/Test/PatchArchiveTest.dhall | 2 +- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index 4f4ce6a4c2c..e07671823e6 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -19,7 +19,7 @@ in { step = ] "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.FunctionalTestSuite - (Some Network.Type.Devnet) + Network.Type.Devnet "./scripts/patch-archive-test.sh" ] , label = "Archive: Patch Archive test" diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 5832d3e0348..d93f70a2677 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (Some Network.Type.Devnet) + Network.Type.Devnet "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index f9eca18c7d5..8bbf02c72ef 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -19,7 +19,7 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.FunctionalTestSuite diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 21bfdcb7498..02ed47087de 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -57,7 +57,7 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Devnet) + Network.Type.Devnet "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ From 1d95f9bdf92bb5721aaa1e29f2fe991f295da0b3 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 12 Sep 2024 21:51:45 +0200 Subject: [PATCH 049/178] correctly build patch_archive_test --- src/dune-project | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dune-project b/src/dune-project index 91a68888957..3c4d5463c4c 100644 --- a/src/dune-project +++ b/src/dune-project @@ -135,6 +135,7 @@ (package (name parallel)) (package (name parallel_scan)) (package (name participating_state)) +(package (name patch_archive_test)) (package (name pasta_bindings)) (package (name perf_histograms)) (package (name pickles_base)) From a6136d95c48a23628613f4b4abdd3efbe5e74dd8 Mon Sep 17 00:00:00 2001 From: svv232 Date: Thu, 12 Sep 2024 23:13:51 -0400 Subject: [PATCH 050/178] overriding faulty compile time config in devnet vs itn test net --- src/lib/integration_test_lib/test_config.ml | 2 +- src/lib/integration_test_local_engine/docker_node_config.ml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 810a99e11b4..00a80aacd34 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -92,7 +92,7 @@ let proof_config_default : Runtime_config.Proof_keys.t = ; transaction_capacity = None ; coinbase_amount = None ; supercharged_coinbase_factor = None - ; account_creation_fee = None + ; account_creation_fee = Some Currency.Fee.one ; fork = None } diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 16bba170998..f207055960d 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -105,6 +105,7 @@ module Base_node_config = struct mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" fi done + rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" |} From 4969625dbb779a2eb105356125ba4f92f541cbc6 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 03:20:04 -0400 Subject: [PATCH 051/178] fixing more integration tests: --- src/lib/integration_test_lib/test_config.ml | 2 +- src/lib/integration_test_local_engine/mina_docker.ml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 00a80aacd34..fedae678eac 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -89,7 +89,7 @@ let proof_config_default : Runtime_config.Proof_keys.t = ; ledger_depth = None ; work_delay = None ; block_window_duration_ms = Some 120000 - ; transaction_capacity = None + ; transaction_capacity = Some (Log_2 3) ; coinbase_amount = None ; supercharged_coinbase_factor = None ; account_creation_fee = Some Currency.Fee.one diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index ffa7a10ca81..448efd107dd 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -158,16 +158,16 @@ module Network_config = struct Some { txpool_max_size = Some txpool_max_size ; peer_list_url = None - ; zkapp_proof_update_cost = None - ; zkapp_signed_single_update_cost = None - ; zkapp_signed_pair_update_cost = None - ; zkapp_transaction_cost_limit = None + ; zkapp_proof_update_cost = (Some 1.) + ; zkapp_signed_single_update_cost = (Some 1.) + ; zkapp_signed_pair_update_cost = (Some 1.) + ; zkapp_transaction_cost_limit = (Some 1000.) ; max_event_elements = None ; max_action_elements = None - ; zkapp_cmd_limit_hardcap = None + ; zkapp_cmd_limit_hardcap = (Some 1000) ; slot_tx_end ; slot_chain_end - ; minimum_user_command_fee = None + ; minimum_user_command_fee = Some (Currency.Fee.of_string "0.001") ; network_id } ; genesis = From 1d2ad87d8c1f4e020349d3e4445d3b4ee95c4954 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 03:25:11 -0400 Subject: [PATCH 052/178] adding formatting --- src/lib/integration_test_local_engine/mina_docker.ml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 448efd107dd..8028742d20f 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -158,13 +158,13 @@ module Network_config = struct Some { txpool_max_size = Some txpool_max_size ; peer_list_url = None - ; zkapp_proof_update_cost = (Some 1.) - ; zkapp_signed_single_update_cost = (Some 1.) - ; zkapp_signed_pair_update_cost = (Some 1.) - ; zkapp_transaction_cost_limit = (Some 1000.) + ; zkapp_proof_update_cost = Some 1. + ; zkapp_signed_single_update_cost = Some 1. + ; zkapp_signed_pair_update_cost = Some 1. + ; zkapp_transaction_cost_limit = Some 1000. ; max_event_elements = None ; max_action_elements = None - ; zkapp_cmd_limit_hardcap = (Some 1000) + ; zkapp_cmd_limit_hardcap = Some 1000 ; slot_tx_end ; slot_chain_end ; minimum_user_command_fee = Some (Currency.Fee.of_string "0.001") From c13d24f67f8b5e6a8008edd0d6a1f903ad95569c Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 13:05:10 -0400 Subject: [PATCH 053/178] matching parameters with dev profile --- src/lib/integration_test_lib/test_config.ml | 6 +++--- src/lib/integration_test_local_engine/mina_docker.ml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index fedae678eac..2e697fa0ae5 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -86,12 +86,12 @@ type t = let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full ; sub_windows_per_window = None - ; ledger_depth = None + ; ledger_depth = Some 10 ; work_delay = None ; block_window_duration_ms = Some 120000 ; transaction_capacity = Some (Log_2 3) - ; coinbase_amount = None - ; supercharged_coinbase_factor = None + ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 20) + ; supercharged_coinbase_factor = Some 1 ; account_creation_fee = Some Currency.Fee.one ; fork = None } diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 8028742d20f..e25b558901b 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -164,10 +164,10 @@ module Network_config = struct ; zkapp_transaction_cost_limit = Some 1000. ; max_event_elements = None ; max_action_elements = None - ; zkapp_cmd_limit_hardcap = Some 1000 + ; zkapp_cmd_limit_hardcap = None ; slot_tx_end ; slot_chain_end - ; minimum_user_command_fee = Some (Currency.Fee.of_string "0.001") + ; minimum_user_command_fee = Some (Currency.Fee.of_string "2") ; network_id } ; genesis = From f6581f868221c8d28c30031c28ef1017ff7033c1 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 14:24:52 -0400 Subject: [PATCH 054/178] tweaking parameters --- src/lib/integration_test_lib/test_config.ml | 4 ++-- src/lib/integration_test_local_engine/mina_docker.ml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 2e697fa0ae5..452f5d1e0c2 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -92,7 +92,7 @@ let proof_config_default : Runtime_config.Proof_keys.t = ; transaction_capacity = Some (Log_2 3) ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 20) ; supercharged_coinbase_factor = Some 1 - ; account_creation_fee = Some Currency.Fee.one + ; account_creation_fee = (Some (Currency.Fee.of_mina_string_exn ".001")) ; fork = None } @@ -126,7 +126,7 @@ let default = ; k = 20 ; slots_per_epoch = 3 * 8 * 20 ; slots_per_sub_window = 2 - ; grace_period_slots = 140 + ; grace_period_slots = 180 ; delta = 0 ; txpool_max_size = 3000 ; slot_tx_end = None diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index e25b558901b..ca5c29f467e 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -164,7 +164,7 @@ module Network_config = struct ; zkapp_transaction_cost_limit = Some 1000. ; max_event_elements = None ; max_action_elements = None - ; zkapp_cmd_limit_hardcap = None + ; zkapp_cmd_limit_hardcap = Some 0 ; slot_tx_end ; slot_chain_end ; minimum_user_command_fee = Some (Currency.Fee.of_string "2") From 18279a0e9f0d08be91e444f2af433657607cf995 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 14:47:04 -0400 Subject: [PATCH 055/178] tweaking grace period --- src/lib/integration_test_lib/test_config.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 452f5d1e0c2..fc997aa5a3c 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -126,7 +126,7 @@ let default = ; k = 20 ; slots_per_epoch = 3 * 8 * 20 ; slots_per_sub_window = 2 - ; grace_period_slots = 180 + ; grace_period_slots = 140 ; delta = 0 ; txpool_max_size = 3000 ; slot_tx_end = None From c793e80ea15b8724b4b12cdf7564fb7f615f0676 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 16:12:37 -0400 Subject: [PATCH 056/178] matching berk itn runtime config --- src/lib/integration_test_lib/test_config.ml | 12 ++++++------ .../integration_test_local_engine/mina_docker.ml | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index fc997aa5a3c..0758a27893e 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -85,14 +85,14 @@ type t = let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full - ; sub_windows_per_window = None - ; ledger_depth = Some 10 - ; work_delay = None + ; sub_windows_per_window = Some 11 + ; ledger_depth = Some 35 + ; work_delay = Some 2 ; block_window_duration_ms = Some 120000 - ; transaction_capacity = Some (Log_2 3) - ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 20) + ; transaction_capacity = Some (Log_2 7) + ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 720) ; supercharged_coinbase_factor = Some 1 - ; account_creation_fee = (Some (Currency.Fee.of_mina_string_exn ".001")) + ; account_creation_fee = Some (Currency.Fee.of_mina_string_exn "1") ; fork = None } diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index ca5c29f467e..a58dfd17c9b 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -158,16 +158,16 @@ module Network_config = struct Some { txpool_max_size = Some txpool_max_size ; peer_list_url = None - ; zkapp_proof_update_cost = Some 1. - ; zkapp_signed_single_update_cost = Some 1. - ; zkapp_signed_pair_update_cost = Some 1. - ; zkapp_transaction_cost_limit = Some 1000. - ; max_event_elements = None - ; max_action_elements = None - ; zkapp_cmd_limit_hardcap = Some 0 + ; zkapp_proof_update_cost = Some 10.26 + ; zkapp_signed_single_update_cost = Some 9.140000000000001 + ; zkapp_signed_pair_update_cost = Some 10.08 + ; zkapp_transaction_cost_limit = Some 69.45 + ; max_event_elements = Some 100 + ; max_action_elements = Some 100 + ; zkapp_cmd_limit_hardcap = Some 128 ; slot_tx_end ; slot_chain_end - ; minimum_user_command_fee = Some (Currency.Fee.of_string "2") + ; minimum_user_command_fee = Some (Currency.Fee.of_string ".001") ; network_id } ; genesis = From 77f5a19ca53069a4009c60834b9ff4b3dffb90ac Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 16:57:05 -0400 Subject: [PATCH 057/178] nano mina conversion --- src/lib/integration_test_local_engine/mina_docker.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index a58dfd17c9b..e2f9628a5fb 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -167,7 +167,7 @@ module Network_config = struct ; zkapp_cmd_limit_hardcap = Some 128 ; slot_tx_end ; slot_chain_end - ; minimum_user_command_fee = Some (Currency.Fee.of_string ".001") + ; minimum_user_command_fee = Some (Currency.Fee.of_string "1000000") ; network_id } ; genesis = From a3e12197a584f9e91604eab318690a7f5779783e Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 20:31:36 -0400 Subject: [PATCH 058/178] fixing invalid file read on boot --- src/lib/integration_test_local_engine/docker_node_config.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index f207055960d..959bbd8cd27 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -101,7 +101,7 @@ module Base_node_config = struct ^ container_keys_path ^ {|/*; do # Exclude specific keys (e.g., libp2p keys) - if [[ $(basename "$key_file") != "libp2p_key" ]]; then + if [[ "$base_name" != "libp2p_key" && "$base_name" != *.peerid ]]; then mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" fi done From 2ce325a265e7b6c95b861cf64f7d783214d25054 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 22:05:18 -0400 Subject: [PATCH 059/178] fixing faulty condition --- src/lib/integration_test_local_engine/docker_node_config.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 959bbd8cd27..9e276e1b5b2 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -101,7 +101,7 @@ module Base_node_config = struct ^ container_keys_path ^ {|/*; do # Exclude specific keys (e.g., libp2p keys) - if [[ "$base_name" != "libp2p_key" && "$base_name" != *.peerid ]]; then + if [[ "$base_name" != *key && "$base_name" != *.peerid ]]; then mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" fi done From 3824b4bd06a2afff13f4216780bdb8f772a4b749 Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 22:12:07 -0400 Subject: [PATCH 060/178] rm account update loop --- .../docker_node_config.ml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 9e276e1b5b2..9d82b56e223 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -93,18 +93,7 @@ module Base_node_config = struct if [ ! -f "$LIBP2P_KEY_PATH" ]; then mina libp2p generate-keypair --privkey-path $LIBP2P_KEY_PATH fi - /bin/chmod -R 700 |} - ^ container_keys_path ^ {|/ - # Import any compatible keys in |} - ^ container_keys_path ^ {|/*, excluding certain keys - for key_file in |} - ^ container_keys_path - ^ {|/*; do - # Exclude specific keys (e.g., libp2p keys) - if [[ "$base_name" != *key && "$base_name" != *.peerid ]]; then - mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" - fi - done + /bin/chmod -R 700 |} ^ {|/ rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" From f6efe735bbedb7b447fb41fbff94ca6223f9aefd Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 22:18:39 -0400 Subject: [PATCH 061/178] syntax --- src/lib/integration_test_local_engine/docker_node_config.ml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 9d82b56e223..0aa3f79d33f 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -93,7 +93,10 @@ module Base_node_config = struct if [ ! -f "$LIBP2P_KEY_PATH" ]; then mina libp2p generate-keypair --privkey-path $LIBP2P_KEY_PATH fi - /bin/chmod -R 700 |} ^ {|/ + /bin/chmod -R 700 |} + ^ container_keys_path ^ {|/ + # Import any compatible keys in |} + ^ container_keys_path ^ {|/*, excluding certain keys rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" From 2c4eb9320aaba832d51b17df0c573cc674c74c9a Mon Sep 17 00:00:00 2001 From: svv232 Date: Fri, 13 Sep 2024 22:25:16 -0400 Subject: [PATCH 062/178] revert --- .../integration_test_local_engine/docker_node_config.ml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 0aa3f79d33f..f207055960d 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -97,6 +97,14 @@ module Base_node_config = struct ^ container_keys_path ^ {|/ # Import any compatible keys in |} ^ container_keys_path ^ {|/*, excluding certain keys + for key_file in |} + ^ container_keys_path + ^ {|/*; do + # Exclude specific keys (e.g., libp2p keys) + if [[ $(basename "$key_file") != "libp2p_key" ]]; then + mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" + fi + done rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" From e43c573987f284b6768d8ceda9be4ab33c35a52b Mon Sep 17 00:00:00 2001 From: svv232 Date: Sat, 14 Sep 2024 01:27:38 -0400 Subject: [PATCH 063/178] adding lightnet case to docker tagging --- scripts/docker/helper.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 07cf02af46a..312b79657ec 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -43,6 +43,18 @@ function export_suffixes () { ;; esac ;; + lightnet) + case "${DEB_BUILD_FLAGS}" in + *instrumented) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet-instrumented" + export BUILD_FLAG_SUFFIX="lightnet-instrumented" + ;; + *) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet" + export BUILD_FLAG_SUFFIX="-lightnet" + ;; + esac + ;; *) case "${DEB_BUILD_FLAGS}" in *instrumented) From cf93a4c96bb4229a40fce3e15c8bc4ccaa87f355 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 14 Sep 2024 19:55:26 +0200 Subject: [PATCH 064/178] fix dhall --- buildkite/src/Command/ReplayerTest.dhall | 2 +- buildkite/src/Command/RunWithPostgres.dhall | 9 +++++++-- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index d93f70a2677..5832d3e0348 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - Network.Type.Devnet + (Some Network.Type.Devnet) "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index 0de46554094..b6279112a23 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -17,11 +17,16 @@ let Artifacts = ../Constants/Artifacts.dhall let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn - : List Text -> Text -> Artifacts.Type -> Network.Type -> Text -> Cmd.Type + : List Text + -> Text + -> Artifacts.Type + -> Optional Network.Type + -> Text + -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Artifacts.Type) - -> \(network : Network.Type) + -> \(network : Optional Network.Type) -> \(innerScript : Text) -> let port = "5432" diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 2a3773837a1..701efb55eb9 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -57,7 +57,7 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - Network.Type.Devnet + (Some Network.Type.Devnet) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ From 05ae5a1e1ec36aa1fd43f2735bccd55dd94a03f5 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 11:22:19 +0200 Subject: [PATCH 065/178] pass buildkite branch env to build debian script since buildkite is checking out commits in detached head --- buildkite/scripts/build-release.sh | 2 +- scripts/debian/build.sh | 10 +++++++++- scripts/export-git-env-vars.sh | 11 ++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index 8b3f3034ce6..ab5bee131be 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -14,7 +14,7 @@ echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" echo "--- Prepare debian packages" -./scripts/debian/build.sh $@ +./scripts/debian/build.sh -b "$BUILDKIE_BRANCH" $@ echo "--- Git diff after build is complete:" git diff --exit-code -- . \ No newline at end of file diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index c3e55f3e49d..9939d617a84 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -4,8 +4,16 @@ set -eox pipefail +# In case of running this script on detached head, script has difficulties in finding out +# what is the current +while [[ "$#" -gt 0 ]]; do case $1 in + -b|--branch-name) BRANCH_NAME_OPT="-b $2"; shift;; + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + + SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -source ${SCRIPTPATH}/../export-git-env-vars.sh +source ${SCRIPTPATH}/../export-git-env-vars.sh "$BRANCH_NAME_OPT" echo "after export" diff --git a/scripts/export-git-env-vars.sh b/scripts/export-git-env-vars.sh index c1d08262488..6958131f39d 100755 --- a/scripts/export-git-env-vars.sh +++ b/scripts/export-git-env-vars.sh @@ -1,8 +1,15 @@ #!/bin/bash set -euo pipefail - set -x + +# In case of running this script on detached head, script has difficulties in finding out +# what is the current +while [[ "$#" -gt 0 ]]; do case $1 in + -b|--branch-name) BRANCH_NAME="$2"; shift;; + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + echo "Exporting Git Variables: " git fetch @@ -19,7 +26,9 @@ function find_most_recent_numeric_tag() { export GITHASH=$(git rev-parse --short=7 HEAD) export THIS_COMMIT_TAG=$(git tag --points-at HEAD) + export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +[[ -n "$BRANCH_NAME" ]] && export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') export GITTAG=$(find_most_recent_numeric_tag HEAD) From 6c0552c1216fdc803b2ac46a4020e51bc873300c Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 11:39:13 +0200 Subject: [PATCH 066/178] fix usage of BRNACH_NAME --- buildkite/scripts/build-release.sh | 2 +- scripts/debian/build.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index ab5bee131be..fdd6440cd5c 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -14,7 +14,7 @@ echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" echo "--- Prepare debian packages" -./scripts/debian/build.sh -b "$BUILDKIE_BRANCH" $@ +BRANCH_NAME="$BUILDKIE_BRANCH" ./scripts/debian/build.sh $@ echo "--- Git diff after build is complete:" git diff --exit-code -- . \ No newline at end of file diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index 9939d617a84..2d61daba6bc 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -6,11 +6,7 @@ set -eox pipefail # In case of running this script on detached head, script has difficulties in finding out # what is the current -while [[ "$#" -gt 0 ]]; do case $1 in - -b|--branch-name) BRANCH_NAME_OPT="-b $2"; shift;; - *) echo "Unknown parameter passed: $1"; exit 1;; -esac; shift; done - +[[ -n "$BRANCH_NAME" ]] && export BRANCH_NAME_OPT="-b $BRANCH_NAME" SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" source ${SCRIPTPATH}/../export-git-env-vars.sh "$BRANCH_NAME_OPT" From 841b27bde6dff62b2296c449c91437c075bc0245 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 11:51:30 +0200 Subject: [PATCH 067/178] fix usage of export-git-env-vars --- scripts/debian/build.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index 2d61daba6bc..e750ca1a03b 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -4,12 +4,15 @@ set -eox pipefail -# In case of running this script on detached head, script has difficulties in finding out -# what is the current -[[ -n "$BRANCH_NAME" ]] && export BRANCH_NAME_OPT="-b $BRANCH_NAME" - SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -source ${SCRIPTPATH}/../export-git-env-vars.sh "$BRANCH_NAME_OPT" + +# In case of running this script on detached head, script has difficulties in finding out +# what is the current branch. +if [[ -n "$BRANCH_NAME" ]]; then + source ${SCRIPTPATH}/../export-git-env-vars.sh -b $BRANCH_NAME +else + source ${SCRIPTPATH}/../export-git-env-vars.sh +fi echo "after export" From dcb19dfdb8d92670f15766e3eb5359e66a7fa08e Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Mon, 16 Sep 2024 13:10:43 +0200 Subject: [PATCH 068/178] Update scripts/export-git-env-vars.sh Co-authored-by: George Agapov --- scripts/export-git-env-vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/export-git-env-vars.sh b/scripts/export-git-env-vars.sh index 6958131f39d..a12642a6fa4 100755 --- a/scripts/export-git-env-vars.sh +++ b/scripts/export-git-env-vars.sh @@ -28,7 +28,7 @@ export GITHASH=$(git rev-parse --short=7 HEAD) export THIS_COMMIT_TAG=$(git tag --points-at HEAD) export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) -[[ -n "$BRANCH_NAME" ]] && export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') +[[ -n "$BRANCH_NAME" ]] && export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) export GITTAG=$(find_most_recent_numeric_tag HEAD) From 66ccf9f518432a665787e7744b52372268f6fa54 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 13:15:28 +0200 Subject: [PATCH 069/178] fix export-git-env-vars script --- buildkite/scripts/export-git-env-vars.sh | 6 +++--- scripts/export-git-env-vars.sh | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index 9b78aa7123d..dfe4e0dd9fd 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -15,7 +15,6 @@ function find_most_recent_numeric_tag() { } export GITHASH=$(git rev-parse --short=7 HEAD) -export GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) export THIS_COMMIT_TAG=$(git tag --points-at HEAD) export PROJECT="mina" @@ -26,8 +25,9 @@ export BUILD_URL=${BUILDKITE_BUILD_URL} set -u export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye} -[[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') - +[[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) + + export RELEASE=unstable if [ "${BUILDKITE_REPO}" != "${MINA_REPO}" ]; then diff --git a/scripts/export-git-env-vars.sh b/scripts/export-git-env-vars.sh index a12642a6fa4..373e2c57d5a 100755 --- a/scripts/export-git-env-vars.sh +++ b/scripts/export-git-env-vars.sh @@ -5,11 +5,6 @@ set -x # In case of running this script on detached head, script has difficulties in finding out # what is the current -while [[ "$#" -gt 0 ]]; do case $1 in - -b|--branch-name) BRANCH_NAME="$2"; shift;; - *) echo "Unknown parameter passed: $1"; exit 1;; -esac; shift; done - echo "Exporting Git Variables: " git fetch @@ -27,7 +22,6 @@ function find_most_recent_numeric_tag() { export GITHASH=$(git rev-parse --short=7 HEAD) export THIS_COMMIT_TAG=$(git tag --points-at HEAD) -export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) [[ -n "$BRANCH_NAME" ]] && export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) export GITTAG=$(find_most_recent_numeric_tag HEAD) From d5ca8fe0592d9f43072fdeb0c2ad74de9d1d1de9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 20:24:12 +0200 Subject: [PATCH 070/178] fix rosetta tests setup --- buildkite/src/Command/RunWithPostgres.dhall | 9 --------- .../src/Jobs/Test/RosettaIntegrationTests.dhall | 4 ++-- .../src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 4 ++-- scripts/docker/helper.sh | 12 ++++++++++++ 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index b6279112a23..f5781fef62f 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -64,15 +64,6 @@ let runInDockerWithPostgresConn : Text = "\\\$MINA_DOCKER_TAG" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 701efb55eb9..21bfdcb7498 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -18,11 +18,11 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let RunWithPostgres = ../../Command/RunWithPostgres.dhall let Network = ../../Constants/Network.dhall -let RunWithPostgres = ../../Command/RunWithPostgres.dhall +let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 441945d5b6e..ec77336a12c 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -20,10 +20,10 @@ let Network = ../../Constants/Network.dhall let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall - let Dockers = ../../Constants/DockerVersions.dhall +let Artifacts = ../../Constants/Artifacts.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type let network = Network.Type.Devnet diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 07cf02af46a..312b79657ec 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -43,6 +43,18 @@ function export_suffixes () { ;; esac ;; + lightnet) + case "${DEB_BUILD_FLAGS}" in + *instrumented) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet-instrumented" + export BUILD_FLAG_SUFFIX="lightnet-instrumented" + ;; + *) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet" + export BUILD_FLAG_SUFFIX="-lightnet" + ;; + esac + ;; *) case "${DEB_BUILD_FLAGS}" in *instrumented) From 941f6f51f09f7277080deb9f3ec2132a4ffd85ef Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 20:30:46 +0200 Subject: [PATCH 071/178] fix RunWithPostgres --- buildkite/src/Command/RunWithPostgres.dhall | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index f5781fef62f..b6279112a23 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -64,6 +64,15 @@ let runInDockerWithPostgresConn : Text = "\\\$MINA_DOCKER_TAG" + let maybeNetwork = + Optional/map + Network.Type + Text + (\(network : Network.Type) -> "-${Network.lowerName network}") + network + + let networkOrDefault = Optional/default Text "" maybeNetwork + in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" From dde87774dc4fd96af497dd3211231510fc028e33 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 20:35:15 +0200 Subject: [PATCH 072/178] remove more berkeley leftovers --- buildkite/src/Jobs/Lint/Merge.dhall | 15 +-------------- scripts/debian/aptly.sh | 2 +- scripts/debian/build.sh | 1 - scripts/debian/builder-helpers.sh | 19 ++----------------- src/app/rosetta/README.md | 3 +-- src/app/rosetta/scripts/berkeley.env | 6 +++--- 6 files changed, 8 insertions(+), 38 deletions(-) diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 57c2ecc0099..8b7ddad80a2 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -73,20 +73,7 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } - , Command.build - Command.Config::{ - , commands = - [ Cmd.run "scripts/merged-to-proof-systems.sh berkeley" ] - , label = - "[proof-systems] Check merges cleanly into proof-systems berkeley branch" - , key = "merged-to-proof-systems-berkeley" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Small - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } - } - , Command.build + Command.build Command.Config::{ , commands = [ Cmd.run "scripts/merged-to-proof-systems.sh develop" ] diff --git a/scripts/debian/aptly.sh b/scripts/debian/aptly.sh index 8eae74c3544..c9051b6a5fc 100755 --- a/scripts/debian/aptly.sh +++ b/scripts/debian/aptly.sh @@ -68,7 +68,7 @@ function start_help(){ echo "" echo "Parameters:" echo "" - echo " -b, --background The Docker name (mina-berkeley, mina-archive etc.)" + echo " -b, --background The Docker name (mina, mina-archive etc.)" echo " -c, --codename The Codename for debian repository" echo " -d, --debians The Debian(s) to be available in aptly. Supports regular expression" echo " -m, --component The Component for debian repository. For example: unstable" diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index e750ca1a03b..2c8dc064846 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -25,7 +25,6 @@ if [ $# -eq 0 ] build_keypair_deb build_archive_deb build_batch_txn_deb - build_daemon_berkeley_deb build_daemon_mainnet_deb build_daemon_devnet_deb build_test_executive_deb diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 0cfd2d219c7..f4676e198b6 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -137,9 +137,9 @@ copy_common_daemon_configs() { echo "------------------------------------------------------------" echo "copy_common_daemon_configs inputs:" - echo "Network Name: ${1} (like mainnet, devnet, berkeley)" + echo "Network Name: ${1} (like mainnet, devnet)" echo "Signature Type: ${2} (mainnet or testnet)" - echo "Seed List URL path: ${3} (like seed-lists/berkeley_seeds.txt)" + echo "Seed List URL path: ${3} (like seed-lists/devnet_seeds.txt)" # Copy shared binaries cp ../src/app/libp2p_helper/result/bin/libp2p_helper "${BUILDDIR}/usr/local/bin/coda-libp2p_helper" @@ -161,7 +161,6 @@ copy_common_daemon_configs() { # Include all useful genesis ledgers cp ../genesis_ledgers/mainnet.json "${BUILDDIR}/var/lib/coda/mainnet.json" cp ../genesis_ledgers/devnet.json "${BUILDDIR}/var/lib/coda/devnet.json" - cp ../genesis_ledgers/berkeley.json "${BUILDDIR}/var/lib/coda/berkeley.json" # Set the default configuration based on Network name ($1) cp ../genesis_ledgers/${1}.json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" cp ../scripts/hardfork/create_runtime_config.sh "${BUILDDIR}/usr/local/bin/mina-hf-create-runtime-config" @@ -291,20 +290,6 @@ build_rosetta_devnet_deb() { build_deb mina-rosetta-devnet } -##################################### ROSETTA BERKELEY PACKAGE ####################################### -build_rosetta_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building rosetta berkeley deb" - - create_control_file mina-rosetta-berkeley "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" - - copy_common_rosetta_configs "testnet" - - build_deb mina-rosetta-berkeley -} - - ##################################### MAINNET PACKAGE ####################################### build_daemon_mainnet_deb() { diff --git a/src/app/rosetta/README.md b/src/app/rosetta/README.md index b8fdf6fbc83..d99a25923e2 100644 --- a/src/app/rosetta/README.md +++ b/src/app/rosetta/README.md @@ -11,10 +11,9 @@ recommended 16GB) and then run the following: ```bash cat dockerfiles/Dockerfile-mina-rosetta \ | docker build -t mina-rosetta:v2.0.0 \ - --build-arg "network=berkeley" \ + --build-arg "network=devnet" \ --build-arg "image=debian:bullseye" \ --build-arg "deb_codename=bullseye" \ - --build-arg "deb_release=berkeley" \ --build-arg "deb_version=2.0.0berkeley-rc1-1551e2f" - ``` diff --git a/src/app/rosetta/scripts/berkeley.env b/src/app/rosetta/scripts/berkeley.env index d64c3057ffe..6f7d55beaa8 100644 --- a/src/app/rosetta/scripts/berkeley.env +++ b/src/app/rosetta/scripts/berkeley.env @@ -1,5 +1,5 @@ -MINA_NETWORK=berkeley -PEER_LIST_URL=https://bootnodes.minaprotocol.com/networks/berkeley.txt +MINA_NETWORK=devnet +PEER_LIST_URL=https://bootnodes.minaprotocol.com/networks/devnet.txt MINA_ARCHIVE_DUMP_URL=https://storage.googleapis.com/mina-archive-dumps -MINA_GENESIS_LEDGER_URL=https://673156464838-mina-genesis-ledgers.s3.us-west-2.amazonaws.com/berkeley/genesis_ledger.json +MINA_GENESIS_LEDGER_URL=https://673156464838-mina-genesis-ledgers.s3.us-west-2.amazonaws.com/devnet/genesis_ledger.json BLOCKS_BUCKET=https://storage.googleapis.com/mina_network_block_data From 95dd42dcfe2e9dd4b84ea9e59cd5ae6a7a7d31e1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 20:36:04 +0200 Subject: [PATCH 073/178] remove berkeley.env config file in rosetta app --- src/app/rosetta/scripts/berkeley.env | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/app/rosetta/scripts/berkeley.env diff --git a/src/app/rosetta/scripts/berkeley.env b/src/app/rosetta/scripts/berkeley.env deleted file mode 100644 index 6f7d55beaa8..00000000000 --- a/src/app/rosetta/scripts/berkeley.env +++ /dev/null @@ -1,5 +0,0 @@ -MINA_NETWORK=devnet -PEER_LIST_URL=https://bootnodes.minaprotocol.com/networks/devnet.txt -MINA_ARCHIVE_DUMP_URL=https://storage.googleapis.com/mina-archive-dumps -MINA_GENESIS_LEDGER_URL=https://673156464838-mina-genesis-ledgers.s3.us-west-2.amazonaws.com/devnet/genesis_ledger.json -BLOCKS_BUCKET=https://storage.googleapis.com/mina_network_block_data From d490208c4d5b4df69ecfd223717c32c087ae20ba Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 16 Sep 2024 20:54:54 +0200 Subject: [PATCH 074/178] fix comma --- buildkite/src/Jobs/Lint/Merge.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 8b7ddad80a2..9d7bc0acc73 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -73,7 +73,7 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } - Command.build + , Command.build Command.Config::{ , commands = [ Cmd.run "scripts/merged-to-proof-systems.sh develop" ] From 3704e57af3cdb5c4fe2438913364dd5bfa33492f Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 08:49:04 +0200 Subject: [PATCH 075/178] remove suffix from archive docker --- buildkite/src/Command/ReplayerTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 5832d3e0348..6e71e618f9d 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (Some Network.Type.Devnet) + (None Network.Type.Devnet) "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" From fe353a8abfbb0c354067cc32ba229a6dfe8d46cb Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 08:59:07 +0200 Subject: [PATCH 076/178] fix debian branch name logic --- buildkite/scripts/export-git-env-vars.sh | 8 ++++++-- scripts/export-git-env-vars.sh | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index dfe4e0dd9fd..08f601d5b17 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -24,8 +24,12 @@ export BUILD_NUM=${BUILDKITE_BUILD_NUM} export BUILD_URL=${BUILDKITE_BUILD_URL} set -u -export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye} -[[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +if [[ -n "$BUILDKITE_BRANCH" ]]; then + export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') +else + export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +fi + export RELEASE=unstable diff --git a/scripts/export-git-env-vars.sh b/scripts/export-git-env-vars.sh index 373e2c57d5a..c9f09e101e5 100755 --- a/scripts/export-git-env-vars.sh +++ b/scripts/export-git-env-vars.sh @@ -22,7 +22,11 @@ function find_most_recent_numeric_tag() { export GITHASH=$(git rev-parse --short=7 HEAD) export THIS_COMMIT_TAG=$(git tag --points-at HEAD) -[[ -n "$BRANCH_NAME" ]] && export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +if [[ -n "$BRANCH_NAME" ]]; then + export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') +else + export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +fi export GITTAG=$(find_most_recent_numeric_tag HEAD) From 75329567754f3f18690d04c71211c878fa98b238 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 09:09:03 +0200 Subject: [PATCH 077/178] fix dhall syntax --- buildkite/src/Command/ReplayerTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 6e71e618f9d..6a742c7e7fd 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (None Network.Type.Devnet) + (None Network.Type) "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" From b326ecdbaeb9a373c7033f4d58ba20f14c6c01e5 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 10:28:05 +0200 Subject: [PATCH 078/178] correct BatchTxn deps and revert removal of MINA_DEB_CODENAME --- buildkite/scripts/export-git-env-vars.sh | 1 + buildkite/src/Command/MinaArtifact.dhall | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index 08f601d5b17..cdbcf369532 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -24,6 +24,7 @@ export BUILD_NUM=${BUILDKITE_BUILD_NUM} export BUILD_URL=${BUILDKITE_BUILD_URL} set -u +export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye} if [[ -n "$BUILDKITE_BRANCH" ]]; then export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') else diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 176e8380ae8..1af0da3f865 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -180,7 +180,7 @@ let docker_step [ DockerImage.ReleaseSpec::{ , deps = deps , service = Artifacts.Type.BatchTxn - , network = Network.lowerName Network.Type.Devnet + , network = Network.lowerName spec.network , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags From eb9b3b185e721539cd9176758cef4aa76f37db68 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 12:44:00 +0200 Subject: [PATCH 079/178] rm /etc/apt/sources.list.d/hashicorp.list before creating it once again --- buildkite/scripts/run-test-executive-local.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 087ad7cc1e2..2d99b95c39c 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -37,6 +37,8 @@ fi # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive +rm /etc/apt/sources.list.d/hashicorp.list + apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl From 3322ebbe7d163e90dcce9715b552db867081f537 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 19:03:40 +0200 Subject: [PATCH 080/178] do not bother checking if hashicorp.list file exists --- buildkite/scripts/run-test-executive-local.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 2d99b95c39c..8512b93df0f 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -37,7 +37,7 @@ fi # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive -rm /etc/apt/sources.list.d/hashicorp.list +rm -f /etc/apt/sources.list.d/hashicorp.list apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl From 35f969654752b528506c4e5cd36d912e1f52d96a Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Tue, 17 Sep 2024 22:11:45 +0200 Subject: [PATCH 081/178] Update src/test/archive/patch_archive_test/patch_archive_test.ml Co-authored-by: George Agapov --- src/test/archive/patch_archive_test/patch_archive_test.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/archive/patch_archive_test/patch_archive_test.ml b/src/test/archive/patch_archive_test/patch_archive_test.ml index ea61373c72b..e9e86b39ee3 100644 --- a/src/test/archive/patch_archive_test/patch_archive_test.ml +++ b/src/test/archive/patch_archive_test/patch_archive_test.ml @@ -77,7 +77,7 @@ let main ~db_uri ~network_data_folder () = let n = List.range 0 missing_blocks_count - |> List.map ~f:(fun _ -> Random.int (List.length extensional_files)) + List.init missing_blocks_count ~f:(fun _ -> Random.int (List.length extensional_files)) in let unpatched_extensional_files = From c73ada9bbdccfe95f7a448c75019cef613367103 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 22:25:50 +0200 Subject: [PATCH 082/178] remove unused module --- src/test/archive/patch_archive_test/sql.ml | 29 ---------------------- 1 file changed, 29 deletions(-) delete mode 100644 src/test/archive/patch_archive_test/sql.ml diff --git a/src/test/archive/patch_archive_test/sql.ml b/src/test/archive/patch_archive_test/sql.ml deleted file mode 100644 index b8ce7abf09a..00000000000 --- a/src/test/archive/patch_archive_test/sql.ml +++ /dev/null @@ -1,29 +0,0 @@ -module Block = struct - let run_state_hash (module Conn : Caqti_async.CONNECTION) = - Conn.collect_list - (Caqti_request.collect Caqti_type.unit Caqti_type.string - {sql| SELECT state_hash from blocks - |sql} ) - () - - let run (module Conn : Caqti_async.CONNECTION) ~state_hash = - Conn.find - (Caqti_request.find Caqti_type.string Caqti_type.int - {sql| SELECT id from blocks where state_hash = ? - |sql} ) - state_hash - - let run_unset_parent (module Conn : Caqti_async.CONNECTION) id = - Conn.exec - (Caqti_request.exec Caqti_type.int - {sql| UPDATE blocks SET parent_hash = NULL where id = ? - |sql} ) - id - - let run_delete (module Conn : Caqti_async.CONNECTION) ~state_hash = - Conn.exec - (Caqti_request.exec Caqti_type.string - {sql| DELETE FROM blocks where state_hash = ? CASCADE - |sql} ) - state_hash -end From 617d6271c19dc5a6daaeabe4214abaa41778ce9a Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 22:26:09 +0200 Subject: [PATCH 083/178] use List.init instead of List.range and List.map --- src/test/archive/patch_archive_test/patch_archive_test.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/archive/patch_archive_test/patch_archive_test.ml b/src/test/archive/patch_archive_test/patch_archive_test.ml index e9e86b39ee3..8d0b2f91a67 100644 --- a/src/test/archive/patch_archive_test/patch_archive_test.ml +++ b/src/test/archive/patch_archive_test/patch_archive_test.ml @@ -76,8 +76,8 @@ let main ~db_uri ~network_data_folder () = in let n = - List.range 0 missing_blocks_count - List.init missing_blocks_count ~f:(fun _ -> Random.int (List.length extensional_files)) + List.init missing_blocks_count ~f:(fun _ -> + Random.int (List.length extensional_files) ) in let unpatched_extensional_files = From afe43a73c855ced2144d47b0caaa210962fc0949 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 18 Sep 2024 10:15:28 +0200 Subject: [PATCH 084/178] Dhall lints --- buildkite/src/Command/RunWithPostgres.dhall | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index af229d5999b..c953576bd10 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -4,10 +4,6 @@ let P = Prelude let Text/concatMap = P.Text.concatMap -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Cmd = ../Lib/Cmds.dhall let ContainerImages = ../Constants/ContainerImages.dhall @@ -64,15 +60,6 @@ let runInDockerWithPostgresConn : Text = "\\\$MINA_DOCKER_TAG" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" From ead07b9a942b96abd98b7b17c108343f77bda177 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 18 Sep 2024 21:25:02 +0200 Subject: [PATCH 085/178] Revert "Auxiliary commit to revert individual files from b756da7c140ac1d17157af029b3a5baa52c9a1be" This reverts commit 091ddca79ae8e939d4065d9f195bd075898704e5, reversing changes made to b5d254684aa48de0e89a4d917a1c687f79ebf198. --- src/dune-project | 1 + src/lib/cli_lib/dune | 2 +- src/lib/mina_compile_config/dune | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dune-project b/src/dune-project index 91a68888957..3c4d5463c4c 100644 --- a/src/dune-project +++ b/src/dune-project @@ -135,6 +135,7 @@ (package (name parallel)) (package (name parallel_scan)) (package (name participating_state)) +(package (name patch_archive_test)) (package (name pasta_bindings)) (package (name perf_histograms)) (package (name pickles_base)) diff --git a/src/lib/cli_lib/dune b/src/lib/cli_lib/dune index dc0f9c72a9d..05b36e7b400 100644 --- a/src/lib/cli_lib/dune +++ b/src/lib/cli_lib/dune @@ -45,7 +45,7 @@ kimchi_pasta.basic ppx_version.runtime gossip_net - mina_runtime_config) + runtime_config) (preprocess (pps ppx_version ppx_jane ppx_deriving_yojson ppx_deriving.make)) (instrumentation (backend bisect_ppx)) diff --git a/src/lib/mina_compile_config/dune b/src/lib/mina_compile_config/dune index fac058923b4..181360fbaf2 100644 --- a/src/lib/mina_compile_config/dune +++ b/src/lib/mina_compile_config/dune @@ -1,6 +1,6 @@ (library (name mina_compile_config) (public_name mina_compile_config) - (libraries mina_node_config mina_node_config.for_unit_tests core_kernel currency) + (libraries mina_node_config node_config_for_unit_tests core_kernel currency) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_version ppx_base ppx_deriving_yojson))) From 3a04375821f379fdda4eaec7a58868f4c5472e77 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Sep 2024 09:47:58 +0200 Subject: [PATCH 086/178] pass BRANCH_NAME to export-git-env-vars script properly --- scripts/debian/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index e750ca1a03b..1df887ce9d9 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -9,7 +9,7 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # In case of running this script on detached head, script has difficulties in finding out # what is the current branch. if [[ -n "$BRANCH_NAME" ]]; then - source ${SCRIPTPATH}/../export-git-env-vars.sh -b $BRANCH_NAME + BRANCH_NAME="$BRANCH_NAME" source ${SCRIPTPATH}/../export-git-env-vars.sh else source ${SCRIPTPATH}/../export-git-env-vars.sh fi From b806cb93dc9465e656f4281e35f6680f346e662f Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Sep 2024 16:10:00 +0200 Subject: [PATCH 087/178] fix step name for functional test suite --- buildkite/src/Command/MinaArtifact.dhall | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 5b01e4e34f3..57da96fd009 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -250,10 +250,11 @@ let docker_step , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile , step_key = - "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" + "${Artifacts.lowerName + Artifacts.Type.FunctionalTestSuite}-${DebianVersions.lowerName + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" , network = "berkeley" } ] From 421e95f39e21f8775d5d5b7135b5368df6eefba0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Sep 2024 16:28:41 +0200 Subject: [PATCH 088/178] Update main Readme --- README.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b7c73df3815..3e2459e5daa 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ -### Build status + -| Develop | Compatible | Master | -| ------- | ---------- | ---------- | -| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) +

Mina

-
+
- - Mina logo - + ![GitHub stars](https://img.shields.io/github/stars/minaprotocol/mina)  ![GitHub forks](https://img.shields.io/github/forks/minaprotocol/mina) -
+![GitHub contributors](https://img.shields.io/github/contributors/minaprotocol/mina)  ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/minaprotocol/mina)  ![GitHub last commit](https://img.shields.io/github/last-commit/minaprotocol/mina) + +| Develop[^1] | Compatible[^2] | Master[^3] | +| ------- | ---------- | ---------- | +| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) + +
Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy! @@ -62,3 +64,7 @@ The [Node Developers](https://docs.minaprotocol.com/node-developers) docs contai [Apache 2.0](LICENSE) Commits older than 2018-10-03 do not have a [LICENSE](LICENSE) file or this notice, but are distributed under the same terms. + +[^1]: Develop is a mainline branch containig code that may be not compatible with current mainnet and may require major upgrade (hardfork). +[^2]: Compatible is a mainline branch containig code which does not need hardfork in order to apply it to mainnet. +[^3]: Branch which contains current mainnet code. \ No newline at end of file From 6a37077ff0a8e51a4a68c0388e844479ad9ebf65 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 20 Sep 2024 09:14:31 +0200 Subject: [PATCH 089/178] fix PatchArchiveTest usage of network arg --- buildkite/src/Command/PatchArchiveTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index ef170ca524a..aff17f8519b 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -19,7 +19,7 @@ in { step = ] "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.FunctionalTestSuite - Network.Type.Devnet + (None Network.Type) "./scripts/patch-archive-test.sh" ] , label = "Archive: Patch Archive test" From 11fbd9c3e9488e761d5cd2dc6509bae1e4818c9a Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 20 Sep 2024 09:25:49 +0200 Subject: [PATCH 090/178] revert docker path + replace TAG with NEW_VERSION --- buildkite/scripts/run_promote_build_job.sh | 2 +- buildkite/src/Command/Promotion/PromoteDebian.dhall | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 68e827f9392..c7d02b1a5fa 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -112,4 +112,4 @@ if [[ "${REMOVE_PROFILE_FROM_NAME}" -eq 0 ]]; then else REMOVE_PROFILE_FROM_NAME="True" fi -echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted +echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${NEW_VERSION}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 2ac01eb006a..cc31e92dcbe 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -76,10 +76,10 @@ let promoteDebianStep = , "AWS_SECRET_ACCESS_KEY" , "FROM_VERSION_MANUAL" ] - ". ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/promote-deb.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName - spec.codename} --from-component ${DebianChannel.lowerName - spec.from_channel} --to-component ${DebianChannel.lowerName - spec.to_channel} ${new_name}" + ". ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/debian/promote.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName + spec.codename} --from-component ${DebianChannel.lowerName + spec.from_channel} --to-component ${DebianChannel.lowerName + spec.to_channel} ${new_name}" , label = "Debian: ${spec.step_key}" , key = spec.step_key , target = Size.Small From 72a4b0e790d0b04b7ceda029ad67ab5412725350 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 20 Sep 2024 09:45:24 +0200 Subject: [PATCH 091/178] Fix archive proof mismatch between compiled nad runtime config --- src/app/archive/lib/processor.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml index db9a4af803a..e8452d413da 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -4737,7 +4737,8 @@ let add_genesis_accounts ~logger ~(runtime_config_opt : Runtime_config.t option) | Some runtime_config -> ( let%bind precomputed_values = match%map - Genesis_ledger_helper.init_from_config_file ~logger ~proof_level:None + Genesis_ledger_helper.init_from_config_file ~logger + ~proof_level:Genesis_constants.Compiled.proof_level ~genesis_constants ~constraint_constants runtime_config ~cli_proof_level:None with From 8d72e9c1bc05d6880bfca14ba0052e52f4297d1f Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 23 Sep 2024 18:19:27 +0200 Subject: [PATCH 092/178] fix mina-functional-test usage in PathArchiveTest.dhall --- buildkite/src/Command/PatchArchiveTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index e94cb9c2adf..aff17f8519b 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -19,7 +19,7 @@ in { step = ] "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.FunctionalTestSuite - (Some Network.Type.Devnet) + (None Network.Type) "./scripts/patch-archive-test.sh" ] , label = "Archive: Patch Archive test" From 20b35ab1c96ef7106fd8cd0e466f009106897bb7 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 23 Sep 2024 19:21:14 +0200 Subject: [PATCH 093/178] allow errors just for docker search --- buildkite/scripts/check-compatibility.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index 43f5dd4073b..c9bf16868d4 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -23,16 +23,19 @@ function download-docker { function try_docker_shas { DOCKER_SHAS=$1 GOT_DOCKER=0 - for sha in $DOCKER_SHAS; do - download-docker $sha - if [ $? -eq 0 ] ; then - GOT_DOCKER=1 - image_tag $sha - break - else - echo "No docker available for SHA=$sha" - fi + + set +e + download-docker $sha + set -e + + if [ $? -eq 0 ] ; then + GOT_DOCKER=1 + image_tag $sha + break + else + echo "No docker available for SHA=$sha" + fi done } From afab80926ffbd23507d4b67710f46128a5141a51 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 23 Sep 2024 19:23:07 +0200 Subject: [PATCH 094/178] fix compatibility test by allowing error just for searching dockers --- buildkite/scripts/check-compatibility.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index 9b39c5f34f0..d76edb192ff 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -22,16 +22,16 @@ function download-docker { function try_docker_shas { DOCKER_SHAS=$1 GOT_DOCKER=0 - + set -x for sha in $DOCKER_SHAS; do - download-docker $sha - if [ $? -eq 0 ] ; then - GOT_DOCKER=1 - image_tag $sha - break - else - echo "No docker available for SHA=$sha" - fi + download-docker $sha + if [ $? -eq 0 ] ; then + GOT_DOCKER=1 + image_tag $sha + break + else + echo "No docker available for SHA=$sha" + fi done } From a02176075c5ddf8e5b91d8a8263700bf9e20d015 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 23 Sep 2024 20:25:19 +0200 Subject: [PATCH 095/178] fix correct return handling for download-docker command --- buildkite/scripts/check-compatibility.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index da5fe15d1af..241de3c15d6 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -26,7 +26,6 @@ function try_docker_shas { set +e download-docker $sha - set -e if [ $? -eq 0 ] ; then GOT_DOCKER=1 @@ -35,7 +34,13 @@ function try_docker_shas { else echo "No docker available for SHA=$sha" fi + set -e done + + if [[ $GOT_DOCKER == 0 ]]; then + echo "docker cannot be found for given shas: $DOCKER_SHAS" + exit 1 + fi } function image_id { From 0ffa50c91089370da2d562375ecab854d8047b0d Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 23 Sep 2024 23:54:51 +0200 Subject: [PATCH 096/178] Make network optional for RunWithPostgres --- buildkite/src/Command/ReplayerTest.dhall | 2 +- buildkite/src/Command/RunWithPostgres.dhall | 24 ++++++++++++++++--- .../Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index d93f70a2677..5832d3e0348 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - Network.Type.Devnet + (Some Network.Type.Devnet) "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index a9ad935999e..1c10cae985e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -6,6 +6,10 @@ let Text/concatMap = P.Text.concatMap let Cmd = ../Lib/Cmds.dhall +let Optional/map = Prelude.Optional.map + +let Optional/default = Prelude.Optional.default + let ContainerImages = ../Constants/ContainerImages.dhall let Artifacts = ../Constants/Artifacts.dhall @@ -13,11 +17,16 @@ let Artifacts = ../Constants/Artifacts.dhall let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn - : List Text -> Text -> Artifacts.Type -> Network.Type -> Text -> Cmd.Type + : List Text + -> Text + -> Artifacts.Type + -> Optional Network.Type + -> Text + -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Artifacts.Type) - -> \(network : Network.Type) + -> \(network : Optional Network.Type) -> \(innerScript : Text) -> let port = "5432" @@ -55,6 +64,15 @@ let runInDockerWithPostgresConn : Text = "\\\$MINA_DOCKER_TAG" + let maybeNetwork = + Optional/map + Network.Type + Text + (\(network : Network.Type) -> "-${Network.lowerName network}") + network + + let networkOrDefault = Optional/default Text "" maybeNetwork + in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" @@ -62,7 +80,7 @@ let runInDockerWithPostgresConn , "sleep 5" , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}" , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName - docker}:${minaDockerTag} ${innerScript}" + docker}:${minaDockerTag}${networkOrDefault} ${innerScript}" ] in { runInDockerWithPostgresConn = runInDockerWithPostgresConn } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 02ed47087de..21bfdcb7498 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -57,7 +57,7 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - Network.Type.Devnet + (Some Network.Type.Devnet) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ From 8556500d421aecf00ade64612412476c9d36d04d Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 24 Sep 2024 08:57:01 +0200 Subject: [PATCH 097/178] fix ReplayerTest.dhall --- buildkite/src/Command/ReplayerTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 5832d3e0348..6a742c7e7fd 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -17,7 +17,7 @@ in { step = ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Archive - (Some Network.Type.Devnet) + (None Network.Type) "./buildkite/scripts/replayer-test.sh" ] , label = "Archive: Replayer test" From b4568751ac928a9846afab0e137794dfb4e82528 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 18:48:06 +0200 Subject: [PATCH 098/178] Scripts: make shellcheck happy on aptly.sh --- scripts/debian/aptly.sh | 53 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/scripts/debian/aptly.sh b/scripts/debian/aptly.sh index c9051b6a5fc..408bda8f105 100755 --- a/scripts/debian/aptly.sh +++ b/scripts/debian/aptly.sh @@ -12,7 +12,6 @@ CLEAR='\033[0m' RED='\033[0;31m' # global variables -declare CLI_VERSION='1.0.0'; declare CLI_NAME='aptly.sh'; declare PS4='debug($LINENO) ${FUNCNAME[0]:+${FUNCNAME[0]}}(): '; @@ -34,28 +33,28 @@ function start_aptly() { local __background=$3 local __clean=$4 local __component=$5 - local __repo="$__distribution-$__component" + local __repo="${__distribution}"-"${__component}" local __port=$6 - if [ $__clean = 1 ]; then + if [ "${__clean}" = 1 ]; then rm -rf ~/.aptly fi - - aptly repo create -component $__component -distribution $__distribution $__repo - aptly repo add $__repo $__debs + aptly repo create -component "${__component}" -distribution "${__distribution}" "${__repo}" - aptly snapshot create $__component from repo $__repo + aptly repo add "${__repo}" "${__debs}" - aptly publish snapshot -distribution=$__distribution -skip-signing $__component + aptly snapshot create "${__component}" from repo "${__repo}" - if [ $__background = 1 ]; then - aptly serve -listen localhost:$__port & - else - aptly serve -listen localhost:$__port + aptly publish snapshot -distribution="${__distribution}" -skip-signing "${__component}" + + if [ "${__background}" = 1 ]; then + aptly serve -listen localhost:"${__port}" & + else + aptly serve -listen localhost:"${__port}" fi - + } @@ -91,7 +90,7 @@ function start(){ local __clean=0 local __component="unstable" local __port=$PORT - + while [ ${#} -gt 0 ]; do error_message="Error: a value is needed for '$1'"; @@ -99,7 +98,7 @@ function start(){ -h | --help ) start_help; ;; - -b | --background ) + -b | --background ) __background=1 shift; ;; @@ -130,13 +129,13 @@ function start(){ ;; esac done - - start_aptly $__distribution \ - $__debs \ - $__background \ - $__clean \ - $__component \ - $__port + + start_aptly "${__distribution}" \ + "${__debs}" \ + "${__background}" \ + "${__clean}" \ + "${__component}" \ + "${__port}" } @@ -156,9 +155,9 @@ function stop_help(){ } function stop(){ - + local __clean=0 - + while [ ${#} -gt 0 ]; do case $1 in -h | --help ) @@ -175,9 +174,9 @@ function stop(){ ;; esac done - + pkill aptly - if [ $__clean = 1 ]; then + if [ "${__clean}" = 1 ]; then rm -rf ~/.aptly fi } @@ -215,4 +214,4 @@ function main(){ esac } -main "$@"; \ No newline at end of file +main "$@"; From efc6c777d6bfc19308b46d357ac56c734d1a2ad6 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 18:51:25 +0200 Subject: [PATCH 099/178] Scripts: make shellcheck partially happy Still getting the following warnings/errors: ``` In scripts/debian/build.sh line 12: BRANCH_NAME="${BRANCH_NAME}" source "${SCRIPTPATH}"/../export-git-env-vars.sh ^-- SC1091 (info): Not following: ./../export-git-env-vars.sh was not specified as input (see shellcheck -x). In scripts/debian/build.sh line 14: source "${SCRIPTPATH}"/../export-git-env-vars.sh ^-- SC1091 (info): Not following: ./../export-git-env-vars.sh was not specified as input (see shellcheck -x). In scripts/debian/build.sh line 19: source "${SCRIPTPATH}"/builder-helpers.sh ^-- SC1091 (info): Not following: ./builder-helpers.sh was not specified as input (see shellcheck -x). ``` --- scripts/debian/build.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index f29127611c0..610e2e41b5a 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -8,16 +8,16 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # In case of running this script on detached head, script has difficulties in finding out # what is the current branch. -if [[ -n "$BRANCH_NAME" ]]; then - BRANCH_NAME="$BRANCH_NAME" source ${SCRIPTPATH}/../export-git-env-vars.sh +if [[ -n "${BRANCH_NAME}" ]]; then + BRANCH_NAME="${BRANCH_NAME}" source "${SCRIPTPATH}"/../export-git-env-vars.sh else - source ${SCRIPTPATH}/../export-git-env-vars.sh -fi + source "${SCRIPTPATH}"/../export-git-env-vars.sh +fi echo "after export" -source ${SCRIPTPATH}/builder-helpers.sh - +source "${SCRIPTPATH}"/builder-helpers.sh + if [ $# -eq 0 ] then echo "No arguments supplied. Building all known debian packages" @@ -31,15 +31,15 @@ if [ $# -eq 0 ] build_functional_test_suite_deb build_zkapp_test_transaction_deb - else + else for i in "$@"; do - if [[ $(type -t "build_${i}_deb") == function ]] - then + if [[ $(type -t "build_${i}_deb") == function ]] + then echo "Building $i debian package" "build_${i}_deb" - else + else echo "invalid debian package name '$i'" exit 1 fi done -fi \ No newline at end of file +fi From 13c5dbcf826e5d65575b7a1423836935b6d521d8 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 18:59:56 +0200 Subject: [PATCH 100/178] Scripts: make shellcheck happy on builder-helpers.sh --- scripts/debian/builder-helpers.sh | 43 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index e145d4af6b8..96539eb5b9d 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -63,7 +63,7 @@ if [[ -v DUNE_INSTRUMENT_WITH ]]; then INSTRUMENTED_SUFFIX=instrumented MINA_DEB_NAME="${MINA_DEB_NAME}-${INSTRUMENTED_SUFFIX}" DEB_SUFFIX="${DEB_SUFFIX}-${INSTRUMENTED_SUFFIX}" -fi +fi BUILDDIR="deb_build" @@ -123,9 +123,9 @@ build_deb() { # Build the package echo "------------------------------------------------------------" - fakeroot dpkg-deb --build "${BUILDDIR}" ${1}_${MINA_DEB_VERSION}.deb + fakeroot dpkg-deb --build "${BUILDDIR}" "${1}"_"${MINA_DEB_VERSION}".deb echo "build_deb outputs:" - ls -lh ${1}_*.deb + ls -lh "${1}"_*.deb echo "deleting BUILDDIR ${BUILDDIR}" rm -rf "${BUILDDIR}" @@ -148,7 +148,7 @@ copy_common_daemon_configs() { cp ./default/src/app/validate_keypair/validate_keypair.exe "${BUILDDIR}/usr/local/bin/mina-validate-keypair" # Copy signature-based Binaries (based on signature type $2 passed into the function) - cp ./default/src/app/cli/src/mina_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina" + cp ./default/src/app/cli/src/mina_"${2}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina" # Copy over Build Configs (based on $2) mkdir -p "${BUILDDIR}/etc/coda/build_config" @@ -162,7 +162,7 @@ copy_common_daemon_configs() { cp ../genesis_ledgers/mainnet.json "${BUILDDIR}/var/lib/coda/mainnet.json" cp ../genesis_ledgers/devnet.json "${BUILDDIR}/var/lib/coda/devnet.json" # Set the default configuration based on Network name ($1) - cp ../genesis_ledgers/${1}.json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" + cp ../genesis_ledgers/"${1}".json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" cp ../scripts/hardfork/create_runtime_config.sh "${BUILDDIR}/usr/local/bin/mina-hf-create-runtime-config" cp ../scripts/mina-verify-packaged-fork-config "${BUILDDIR}/usr/local/bin/mina-verify-packaged-fork-config" # Update the mina.service with a new default PEERS_URL based on Seed List URL $3 @@ -240,7 +240,7 @@ build_functional_test_suite_deb() { mkdir -p "${BUILDDIR}/etc/mina/test/archive" - cp -r ../src/test/archive/* ${BUILDDIR}/etc/mina/test/archive/ + cp -r ../src/test/archive/* "${BUILDDIR}"/etc/mina/test/archive/ # Binaries cp ./default/src/test/command_line_tests/command_line_tests.exe "${BUILDDIR}/usr/local/bin/mina-command-line-tests" @@ -252,11 +252,11 @@ build_functional_test_suite_deb() { ##################################### END TEST SUITE PACKAGE ####################################### function copy_common_rosetta_configs () { - - # Copy rosetta-based Binaries - cp ./default/src/app/rosetta/rosetta_${1}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" - cp ./default/src/app/rosetta/ocaml-signer/signer_${1}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" - + + # Copy rosetta-based Binaries + cp ./default/src/app/rosetta/rosetta_"${1}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" + cp ./default/src/app/rosetta/ocaml-signer/signer_"${1}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" + mkdir -p "${BUILDDIR}/etc/mina/rosetta" mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" mkdir -p "${BUILDDIR}/etc/mina/rosetta/scripts" @@ -266,38 +266,38 @@ function copy_common_rosetta_configs () { cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ./default/src/app/rosetta/indexer_test/indexer_test.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-indexer-test" - + } ##################################### ROSETTA MAINNET PACKAGE ####################################### build_rosetta_mainnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building mainnet rosetta deb" create_control_file mina-rosetta-mainnet "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" copy_common_rosetta_configs "mainnet" - + build_deb mina-rosetta-mainnet } ##################################### ROSETTA MAINNET PACKAGE ####################################### build_rosetta_devnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building devnet rosetta deb" create_control_file mina-rosetta-devnet "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" copy_common_rosetta_configs "testnet" - + build_deb mina-rosetta-devnet } ##################################### MAINNET PACKAGE ####################################### build_daemon_mainnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building mainnet deb without keys:" @@ -311,7 +311,7 @@ build_daemon_mainnet_deb() { ##################################### DEVNET PACKAGE ####################################### build_daemon_devnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building testnet signatures deb without keys:" @@ -337,12 +337,13 @@ build_archive_deb () { cp ./default/src/app/archive/archive.exe "${BUILDDIR}/usr/local/bin/mina-archive" cp ./default/src/app/archive_blocks/archive_blocks.exe "${BUILDDIR}/usr/local/bin/mina-archive-blocks" cp ./default/src/app/extract_blocks/extract_blocks.exe "${BUILDDIR}/usr/local/bin/mina-extract-blocks" - + mkdir -p "${BUILDDIR}/etc/mina/archive" cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/usr/local/bin/mina-missing-blocks-guardian" + cp ./default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe "${BUILDDIR}/usr/local/bin/mina-missing-blocks-auditor" cp ./default/src/app/replayer/replayer.exe "${BUILDDIR}/usr/local/bin/mina-replayer" - + cp ../src/app/archive/create_schema.sql "${BUILDDIR}/etc/mina/archive" cp ../src/app/archive/drop_tables.sql "${BUILDDIR}/etc/mina/archive" @@ -360,6 +361,6 @@ build_zkapp_test_transaction_deb () { # Binaries cp ./default/src/app/zkapp_test_transaction/zkapp_test_transaction.exe "${BUILDDIR}/usr/local/bin/mina-zkapp-test-transaction" - build_deb mina-zkapp-test-transaction + build_deb mina-zkapp-test-transaction } ##################################### END ZKAPP TEST TXN PACKAGE ####################################### From d56b853b18bac78f6615bd02fa2c81110ddf0530 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 19:00:09 +0200 Subject: [PATCH 101/178] Scripts: make shellcheck a bit happier on publish.sh I am still getting: ``` In scripts/debian/publish.sh line 60: ) && break || scripts/debian/clear-s3-lockfile.sh; done ^-- SC2015 (info): Note that A && B || C is not if-then-else. C may run when A is true. In scripts/debian/publish.sh line 80: for _i in {1..10}; do (verify_o1test_repo_has_package "${deb}") && break || sleep 60; done ^-- SC2015 (info): Note that A && B || C is not if-then-else. C may run when A is true. For more information: https://www.shellcheck.net/wiki/SC2015 -- Note that A && B || C is not if-t... ``` --- scripts/debian/publish.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index f20d9b53d33..0d0e3336fcf 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -52,7 +52,7 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}" # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. -for i in {1..10}; do ( +for _i in {1..10}; do ( ${DEBS3_UPLOAD} \ --component "${DEB_RELEASE}" \ --codename "${DEB_CODENAME}" \ @@ -62,23 +62,21 @@ for i in {1..10}; do ( # Verify integrity of debs on remote repo function verify_o1test_repo_has_package { sudo apt-get update - ${DEBS3_SHOW} ${1} ${DEB_VERSION} $ARCH -c $DEB_CODENAME -m $DEB_RELEASE + ${DEBS3_SHOW} "${1}" "${DEB_VERSION}" ${ARCH} -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" return $? } for deb in $DEB_NAMES do - echo "Adding packages.o1test.net $DEB_CODENAME $DEB_RELEASE" - sudo echo "deb [trusted=yes] http://packages.o1test.net $DEB_CODENAME $DEB_RELEASE" | sudo tee /etc/apt/sources.list.d/mina.list + echo "Adding packages.o1test.net ${DEB_CODENAME} ${DEB_RELEASE}" + sudo echo "deb [trusted=yes] http://packages.o1test.net ${DEB_CODENAME} ${DEB_RELEASE}" | sudo tee /etc/apt/sources.list.d/mina.list DEBS3_SHOW="deb-s3 show $BUCKET_ARG $S3_REGION_ARG" - deb_split=(${deb//_/ }) + deb_split=("${deb//_/ }") deb="${deb_split[0]}" - deb=$(basename $deb) - - for i in {1..10}; do (verify_o1test_repo_has_package $deb) && break || sleep 60; done - -done + deb=$(basename "${deb}") + for _i in {1..10}; do (verify_o1test_repo_has_package "${deb}") && break || sleep 60; done +done From e863832555479f6a77f8be462495c656fac902f7 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 19:02:12 +0200 Subject: [PATCH 102/178] Scripts: make shellcheck a bit happier on reversion.sh I am still getting the following message: ``` In scripts/debian/reversion.sh line 63: source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version "${NEW_VERSION}" --codename "${CODENAME}" --release "${NEW_RELEASE}" ^-----------------------^ SC1091 (info): Not following: scripts/debian/publish.sh was not specified as input (see shellcheck -x). For more information: https://www.shellcheck.net/wiki/SC1091 -- Not following: scripts/debian/pub... ``` --- scripts/debian/reversion.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/debian/reversion.sh b/scripts/debian/reversion.sh index 0aa2c16faf3..b17bbbab95b 100755 --- a/scripts/debian/reversion.sh +++ b/scripts/debian/reversion.sh @@ -49,8 +49,8 @@ if [[ -z "$SUITE" ]]; then NEW_SUITE=$SUITE; fi; function rebuild_deb() { rm -f "${DEB}_${VERSION}.deb" rm -rf "${NEW_NAME}_${NEW_VERSION}" - - wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb + + wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/"${CODENAME}"/m/mi/"${DEB}"_"${VERSION}".deb dpkg-deb -R "${DEB}_${VERSION}.deb" "${NEW_NAME}_${NEW_VERSION}" sed -i 's/Version: '"${VERSION}"'/Version: '"${NEW_VERSION}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" sed -i 's/Package: '"${DEB}"'/Package: '"${NEW_NAME}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" @@ -60,4 +60,4 @@ function rebuild_deb() { rebuild_deb -source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} +source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version "${NEW_VERSION}" --codename "${CODENAME}" --release "${NEW_RELEASE}" From c338c2650c13495ee801302d0451c50391c81b5b Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 19:03:01 +0200 Subject: [PATCH 103/178] Scripts: make shellcheck a bit happier on verify.sh I am still getting: ``` In scripts/debian/verify.sh line 47: && echo 'OK: ALL WORKED FINE!' || (echo 'KO: ERROR!!!' && exit 1) ^-- SC2015 (info): Note that A && B || C is not if-then-else. C may run when A is true. For more information: https://www.shellcheck.net/wiki/SC2015 -- Note that A && B || C is not if-t... ``` --- scripts/debian/verify.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index c61d3636a7e..25877aa3f67 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -13,7 +13,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done -if [ -z $PACKAGE ]; then +if [ -z "${PACKAGE}" ]; then echo "No package defined. exiting.."; exit 1; fi From 2d5b026912b574c95853f04c960169cdcee616a9 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 14:47:07 +0200 Subject: [PATCH 104/178] reformat: ignore opam-switches --- src/app/reformat/reformat.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/reformat/reformat.ml b/src/app/reformat/reformat.ml index 2d37b135169..0ea294650e1 100644 --- a/src/app/reformat/reformat.ml +++ b/src/app/reformat/reformat.ml @@ -18,6 +18,7 @@ let dirs_trustlist = ; "proof-systems" ; "snarky" ; "_opam" + ; "opam_switches" ; ".direnv" ] let rec fold_over_files ~path ~process_path ~init ~f = From 66d3ab7df02131e228fa52ea2bad2e3fec85ac66 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sun, 29 Sep 2024 21:55:21 +0200 Subject: [PATCH 105/178] fix typo when passing by BUILDKITE_BRANCH --- buildkite/scripts/build-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index fdd6440cd5c..dbf3bb490bc 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -14,7 +14,7 @@ echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" echo "--- Prepare debian packages" -BRANCH_NAME="$BUILDKIE_BRANCH" ./scripts/debian/build.sh $@ +BRANCH_NAME="$BUILDKITE_BRANCH" ./scripts/debian/build.sh $@ echo "--- Git diff after build is complete:" git diff --exit-code -- . \ No newline at end of file From ffa2ff3970ecfe3888aa66678a9ed4b89b5f2760 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 30 Sep 2024 09:57:39 +0200 Subject: [PATCH 106/178] Remove Buster support --- README-ci-failures.md | 6 +-- automation/services/watchdog/Dockerfile | 2 +- buildkite/scripts/run_promote_build_job.sh | 6 +-- .../scripts/run_verify_promoted_build_job.sh | 4 +- buildkite/src/Command/MinaArtifact.dhall | 2 + buildkite/src/Command/RunInToolchain.dhall | 10 ----- buildkite/src/Constants/ContainerImages.dhall | 2 - buildkite/src/Constants/DebianVersions.dhall | 35 ++++++++-------- buildkite/src/Constants/DockerVersions.dhall | 8 +--- buildkite/src/Constants/Toolchain.dhall | 2 - .../Entrypoints/GenerateHardforkPackage.dhall | 2 +- .../src/Jobs/Release/MinaArtifactBuster.dhall | 22 ---------- .../Release/MinaArtifactMainnetBuster.dhall | 35 ---------------- .../Release/MinaToolchainArtifactBuster.dhall | 41 ------------------- buildkite/src/README.md | 12 +++--- dockerfiles/Dockerfile-mina-daemon | 8 ---- dockerfiles/Dockerfile-mina-rosetta | 10 +---- dockerfiles/Dockerfile-mina-test-suite | 8 ---- dockerfiles/stages/1-build-deps | 2 +- dockerfiles/stages/2-opam-deps | 2 + scripts/deb-builder-helpers.sh | 4 -- scripts/debian/verify.sh | 1 - scripts/release-docker.sh | 4 +- 23 files changed, 44 insertions(+), 184 deletions(-) delete mode 100644 buildkite/src/Jobs/Release/MinaArtifactBuster.dhall delete mode 100644 buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall delete mode 100644 buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall diff --git a/README-ci-failures.md b/README-ci-failures.md index cfd8f4bc81d..595b083f84b 100644 --- a/README-ci-failures.md +++ b/README-ci-failures.md @@ -29,8 +29,8 @@ details and links to the failures. ## CI environment mismatch The CI runs its jobs in multiple Docker images. The images that it is using are -specified in `buildkite/src/Constants/COntainerImages.dhall`: the CI uses all -Debian images prefixed by `minaToolchainBuster`. +specified in `buildkite/src/Constants/ContainerImages.dhall`: the CI uses all +Debian images prefixed by `minaToolchainBullseye`. These images are generated by the CI itself, in particular based on the content of the `dockerfiles` directory and the `opam.export` file (which describes versions of @@ -46,7 +46,7 @@ In this case, you should: [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain) where they have been automatically uploaded. Look for your branch name and commit hash in the second column. You should find several images suffixed - with `-buster`, `-bullseye`, `-stretch` and `-focal` (Debian versions). + with `-bullseye` or `-focal` (Debian versions). 3. For each such image, retrieve its full name and hash by hovering its link and clicking the `Copy full image name` tooltip that appears (or retrieve it on the image's page). diff --git a/automation/services/watchdog/Dockerfile b/automation/services/watchdog/Dockerfile index 43e6e00f825..05e207350f6 100644 --- a/automation/services/watchdog/Dockerfile +++ b/automation/services/watchdog/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.16-slim-buster +FROM python:3.9.16-slim-bullseye ARG GCLOUDSDK_DOWNLOAD_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-296.0.1-linux-x86_64.tar.gz" ARG WATCHMAN_DEB_URL="http://ftp.us.debian.org/debian/pool/main/w/watchman/watchman_4.9.0-5+b1_amd64.deb" diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 26feee3e23a..191c24e197b 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -18,7 +18,7 @@ # - "NETWORK=Devnet" # - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" # - "NEW_VERSION=3.0.0fake-ddb6fc4" -# - "CODENAMES=Focal,Buster,Bullseye" +# - "CODENAMES=Focal,Bullseye" # - "FROM_CHANNEL=Unstable" # - "TO_CHANNEL=Experimental" # image: codaprotocol/ci-toolchain-base:v3 @@ -41,7 +41,7 @@ function usage() { fi echo " DEBIANS The comma delimitered debian names. For example: 'Daemon,Archive' " echo " DOCKERS The comma delimitered docker names. For example: 'Daemon,Archive' " - echo " CODENAMES The Debian codenames (Bullseye, Buster etc.)" + echo " CODENAMES The Debian codenames (Bullseye, Focal etc.)" echo " FROM_VERSION The Source Docker or Debian version " echo " NEW_VERSION The new Debian version or new Docker tag" echo " REMOVE_PROFILE_FROM_NAME Should we remove profile suffix from debian name" @@ -112,4 +112,4 @@ if [[ "${REMOVE_PROFILE_FROM_NAME}" -eq 0 ]]; then else REMOVE_PROFILE_FROM_NAME="True" fi -echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted +echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${NEW_VERSION}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index ed093b1a44b..da2a3789d3f 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -18,7 +18,7 @@ # - "NETWORK=Devnet" # - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" # - "NEW_VERSION=3.0.0fake-ddb6fc4" -# - "CODENAMES=Focal,Buster,Bullseye" +# - "CODENAMES=Focal,Bullseye" # - "FROM_CHANNEL=Unstable" # - "TO_CHANNEL=Experimental" # image: codaprotocol/ci-toolchain-base:v3 @@ -41,7 +41,7 @@ function usage() { fi echo " DEBIANS The comma delimitered debian names. For example: 'Daemon,Archive' " echo " DOCKERS The comma delimitered docker names. For example: 'Daemon,Archive' " - echo " CODENAMES The Debian codenames (Bullseye, Buster etc.)" + echo " CODENAMES The Debian codenames (Bullseye, Focal etc.)" echo " NEW_VERSION The new Debian version or new Docker tag" echo " REMOVE_PROFILE_FROM_NAME Should we remove profile suffix from debian name" echo " PROFILE The Docker and Debian profile (Standard, Lightnet)" diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 5e6968df95f..b810cf02b8b 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -117,6 +117,7 @@ let docker_step let deps = DebianVersions.dependsOnStep + (None Text) debVersion profile buildFlags @@ -308,6 +309,7 @@ let publish_to_debian_repo = , key = "publish-deb-pkg" , depends_on = DebianVersions.dependsOnStep + (Some spec.prefix) spec.debVersion spec.profile spec.buildFlags diff --git a/buildkite/src/Command/RunInToolchain.dhall b/buildkite/src/Command/RunInToolchain.dhall index 6d4616b605d..ecc05ca00f1 100644 --- a/buildkite/src/Command/RunInToolchain.dhall +++ b/buildkite/src/Command/RunInToolchain.dhall @@ -33,15 +33,6 @@ let runInToolchainBullseye environment innerScript -let runInToolchainBuster - : List Text -> Text -> List Cmd.Type - = \(environment : List Text) - -> \(innerScript : Text) - -> runInToolchainImage - ContainerImages.minaToolchainBuster - environment - innerScript - let runInToolchain : List Text -> Text -> List Cmd.Type = \(environment : List Text) @@ -55,5 +46,4 @@ in { runInToolchain = runInToolchain , runInToolchainImage = runInToolchainImage , runInToolchainBookworm = runInToolchainBookworm , runInToolchainBullseye = runInToolchainBullseye - , runInToolchainBuster = runInToolchainBuster } diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index ff3cd4d0d58..d6a2e64be80 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -3,8 +3,6 @@ -- NOTE: minaToolchainBullseye is also used for building Ubuntu Focal packages in CI -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI { toolchainBase = "codaprotocol/ci-toolchain-base:v3" -, minaToolchainBuster = - "gcr.io/o1labs-192920/mina-toolchain@sha256:d27c15e3143a99b86155ba57696020c00e2a296b388499b3e6fb364478ddda3a" , minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f" , minaToolchainBookworm = diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 98cd6882689..f3042f5f067 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -1,17 +1,20 @@ +let Prelude = ../External/Prelude.dhall + +let Optional/default = Prelude.Optional.default + let Profiles = ./Profiles.dhall let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall -let DebVersion = < Bookworm | Bullseye | Buster | Jammy | Focal > +let DebVersion = < Bookworm | Bullseye | Jammy | Focal > let capitalName = \(debVersion : DebVersion) -> merge { Bookworm = "Bookworm" , Bullseye = "Bullseye" - , Buster = "Buster" , Jammy = "Jammy" , Focal = "Focal" } @@ -22,20 +25,20 @@ let lowerName = -> merge { Bookworm = "bookworm" , Bullseye = "bullseye" - , Buster = "buster" , Jammy = "jammy" , Focal = "focal" } debVersion let dependsOnStep = - \(debVersion : DebVersion) + \(prefix : Optional Text) + -> \(debVersion : DebVersion) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) -> let profileSuffix = Profiles.toSuffixUppercase profile - let prefix = "MinaArtifact" + let prefix = Optional/default Text "MinaArtifact" prefix in merge { Bookworm = @@ -53,14 +56,6 @@ let dependsOnStep = , key = "${step}-deb-pkg" } ] - , Buster = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] , Jammy = [ { name = "${prefix}${capitalName @@ -83,7 +78,12 @@ let dependsOnStep = let dependsOn = \(debVersion : DebVersion) -> \(profile : Profiles.Type) - -> dependsOnStep debVersion profile BuildFlags.Type.None "build" + -> dependsOnStep + (None Text) + debVersion + profile + BuildFlags.Type.None + "build" let minimalDirtyWhen = [ S.exactly "buildkite/src/Constants/DebianVersions" "dhall" @@ -92,8 +92,10 @@ let minimalDirtyWhen = , S.exactly "buildkite/src/Command/MinaArtifact" "dhall" , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") , S.strictlyStart (S.contains "dockerfiles/stages") - , S.exactly "scripts/rebuild-deb" "sh" - , S.exactly "scripts/release-docker" "sh" + , S.exactly "scripts/debian/build" "sh" + , S.exactly "scripts/debian/builder-helpers" "sh" + , S.exactly "scripts/docker/release" "sh" + , S.exactly "scripts/docker/build" "sh" , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" @@ -122,7 +124,6 @@ let dirtyWhen = -> merge { Bookworm = minimalDirtyWhen , Bullseye = bullseyeDirtyWhen - , Buster = minimalDirtyWhen , Jammy = minimalDirtyWhen , Focal = minimalDirtyWhen } diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 3079bab2fd5..7db6179132c 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -12,14 +12,13 @@ let Network = ./Network.dhall let Docker : Type - = < Bookworm | Bullseye | Buster | Jammy | Focal > + = < Bookworm | Bullseye | Jammy | Focal > let capitalName = \(docker : Docker) -> merge { Bookworm = "Bookworm" , Bullseye = "Bullseye" - , Buster = "Buster" , Jammy = "Jammy" , Focal = "Focal" } @@ -30,7 +29,6 @@ let lowerName = -> merge { Bookworm = "bookworm" , Bullseye = "bullseye" - , Buster = "buster" , Jammy = "jammy" , Focal = "focal" } @@ -62,13 +60,11 @@ let dependsOn = in merge { Bookworm = - [ { name = "${prefix}${profileSuffix}", key = key } ] - , Bullseye = [ { name = "${prefix}${capitalName docker}${profileSuffix}" , key = key } ] - , Buster = + , Bullseye = [ { name = "${prefix}${capitalName docker}${profileSuffix}" , key = key } diff --git a/buildkite/src/Constants/Toolchain.dhall b/buildkite/src/Constants/Toolchain.dhall index b70181349bf..21cbde37fe5 100644 --- a/buildkite/src/Constants/Toolchain.dhall +++ b/buildkite/src/Constants/Toolchain.dhall @@ -13,7 +13,6 @@ let runner = -> merge { Bookworm = RunInToolchain.runInToolchainBookworm , Bullseye = RunInToolchain.runInToolchainBullseye - , Buster = RunInToolchain.runInToolchainBuster , Jammy = RunInToolchain.runInToolchainBookworm , Focal = RunInToolchain.runInToolchainBullseye } @@ -34,7 +33,6 @@ let image = -> merge { Bookworm = ContainerImages.minaToolchainBookworm , Bullseye = ContainerImages.minaToolchainBullseye - , Buster = ContainerImages.minaToolchainBuster , Jammy = ContainerImages.minaToolchainBookworm , Focal = ContainerImages.minaToolchainBullseye } diff --git a/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall b/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall index 31756138e28..64bfb78f805 100644 --- a/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall +++ b/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall @@ -35,7 +35,7 @@ let generate_hardfork_package = , Cmd.quietly "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{=}).pipeline' | buildkite-agent pipeline upload" , Cmd.quietly - "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{codename = DebianVersions.DebVersion.Buster}).pipeline' | buildkite-agent pipeline upload" + "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{codename = DebianVersions.DebVersion.Bullseye}).pipeline' | buildkite-agent pipeline upload" ] , label = "Generate hardfork package" , key = "generate-hardfork-package" diff --git a/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall deleted file mode 100644 index c45b8c1b993..00000000000 --- a/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall +++ /dev/null @@ -1,22 +0,0 @@ -let ArtifactPipelines = ../../Command/MinaArtifact.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let Artifacts = ../../Constants/Artifacts.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -in Pipeline.build - ( ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - , artifacts = - [ Artifacts.Type.Daemon - , Artifacts.Type.Archive - , Artifacts.Type.BatchTxn - , Artifacts.Type.TestExecutive - , Artifacts.Type.Rosetta - , Artifacts.Type.ZkappTestTransaction - ] - , debVersion = DebianVersions.DebVersion.Buster - } - ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall b/buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall deleted file mode 100644 index 49b2adcc03c..00000000000 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall +++ /dev/null @@ -1,35 +0,0 @@ -let ArtifactPipelines = ../../Command/MinaArtifact.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let Network = ../../Constants/Network.dhall - -let Artifacts = ../../Constants/Artifacts.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineMode = ../../Pipeline/Mode.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -in Pipeline.build - ( ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - , artifacts = - [ Artifacts.Type.Daemon - , Artifacts.Type.Archive - , Artifacts.Type.BatchTxn - , Artifacts.Type.Rosetta - , Artifacts.Type.ZkappTestTransaction - ] - , debVersion = DebianVersions.DebVersion.Buster - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" - } - ) diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall deleted file mode 100644 index 3057316b1cc..00000000000 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall +++ /dev/null @@ -1,41 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let PipelineMode = ../../Pipeline/Mode.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "dockerfiles/stages/1-") - , S.strictlyStart (S.contains "dockerfiles/stages/2-") - , S.strictlyStart (S.contains "dockerfiles/stages/3-") - , S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact") - , S.strictly (S.contains "opam.export") - , S.strictlyEnd (S.contains "rust-toolchain.toml") - ] - , path = "Release" - , name = "MinaToolchainArtifactBuster" - , tags = [ PipelineTag.Type.Toolchain ] - , mode = PipelineMode.Type.Stable - } - , steps = - [ let toolchainBusterSpec = - DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "buster" - , extra_args = "--no-cache" - , step_key = "toolchain-buster-docker-image" - } - - in DockerImage.generateStep toolchainBusterSpec - ] - } diff --git a/buildkite/src/README.md b/buildkite/src/README.md index d9b577e81b5..e8bfe284ddb 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -134,7 +134,7 @@ steps: - "NETWORK=Devnet" - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" - "NEW_VERSION=3.0.0fake-ddb6fc4" - - "CODENAMES=Focal,Buster,Bullseye" + - "CODENAMES=Focal,Bullseye" - "FROM_CHANNEL=Unstable" - "TO_CHANNEL=Experimental" image: codaprotocol/ci-toolchain-base:v3 @@ -146,11 +146,11 @@ Above definition one need to paste into steps edit box for given pipeline and th All list of available parameters: -- DEBIANS - The comma delimitered debian names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/DebianPackage.dhall` files. Only CamelCase format is supported +- DEBIANS - The comma delimited debian names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/DebianPackage.dhall` files. Only CamelCase format is supported -- DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported +- DOCKERS - The comma delimited docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported -- CODENAMES - The Debian codenames `Bullseye,Buster,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported +- CODENAMES - The Debian codenames `Bullseye,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported - FROM_VERSION - The Source Docker or Debian version @@ -187,7 +187,7 @@ Pipeline with create 6 jobs for each Docker and Debian component separately. - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" - "NEW_VERSION=3.0.0-ddb6fc4" - "PUBLISH"=1 - - "CODENAMES=Focal,Buster,Bullseye" + - "CODENAMES=Focal,Bullseye" - "FROM_CHANNEL=Unstable" - "TO_CHANNEL=Devnet" @@ -202,6 +202,6 @@ We want only to move dockers from gcr to dockerhub without changing version. Cur - "NETWORK=Devnet" - "FROM_VERSION=3.0.0-dc6bf78" - "NEW_VERSION=3.0.0-dc6bf78" - - "CODENAMES=Focal,Buster,Bullseye" + - "CODENAMES=Focal,Bullseye" - "PUBLISH=1" ``` \ No newline at end of file diff --git a/dockerfiles/Dockerfile-mina-daemon b/dockerfiles/Dockerfile-mina-daemon index 4f718e5ea3f..7d43e27bd0e 100644 --- a/dockerfiles/Dockerfile-mina-daemon +++ b/dockerfiles/Dockerfile-mina-daemon @@ -22,14 +22,6 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. -# Stretch-only Deps: -# libffi6 -# libprocps6 -# libjemalloc1 -# Buster-only Deps: -# libffi7 -# libprocps7 -# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 8c5cd9b82b6..51c90c78c0f 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -22,14 +22,6 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. -# Stretch-only Deps: -# libffi6 -# libprocps6 -# libjemalloc1 -# Buster-only Deps: -# libffi7 -# libprocps7 -# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 @@ -99,7 +91,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres diff --git a/dockerfiles/Dockerfile-mina-test-suite b/dockerfiles/Dockerfile-mina-test-suite index 98ac49ce9cd..1162bfa5e9e 100644 --- a/dockerfiles/Dockerfile-mina-test-suite +++ b/dockerfiles/Dockerfile-mina-test-suite @@ -13,14 +13,6 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. -# Stretch-only Deps: -# libffi6 -# libprocps6 -# libjemalloc1 -# Buster-only Deps: -# libffi7 -# libprocps7 -# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 diff --git a/dockerfiles/stages/1-build-deps b/dockerfiles/stages/1-build-deps index 3c57e9d40fd..3e5348cc6e5 100644 --- a/dockerfiles/stages/1-build-deps +++ b/dockerfiles/stages/1-build-deps @@ -2,7 +2,7 @@ # The "build-deps" stage # - Installs all compilers/interpreters, tools, and OS packages on the given debian or ubuntu image ################################################################################################# -# Supports debian:buster-slim, debian:bullseye-slim, and ubuntu:focal +# Supports debian:bullseye-slim, and ubuntu:focal ARG image=ubuntu:focal FROM ${image} AS build-deps diff --git a/dockerfiles/stages/2-opam-deps b/dockerfiles/stages/2-opam-deps index 24b6ad9f246..09da654bdd2 100644 --- a/dockerfiles/stages/2-opam-deps +++ b/dockerfiles/stages/2-opam-deps @@ -25,6 +25,8 @@ ENV OPAMREUSEBUILDDIR false # Limit logs for opam install errors to 20 lines per error ENV OPAMERRLOGLEN 20 +ENV GIT_LFS_SKIP_SMUDGE 1 + # --- Shallow clone the Mina repo, only focused on the given MINA_BRANCH # git will clone into an empty dir, but this also helps us set the workdir in advance RUN git clone \ diff --git a/scripts/deb-builder-helpers.sh b/scripts/deb-builder-helpers.sh index 666078720b9..10af20e47b1 100755 --- a/scripts/deb-builder-helpers.sh +++ b/scripts/deb-builder-helpers.sh @@ -38,10 +38,6 @@ case "${MINA_DEB_CODENAME}" in DAEMON_DEPS=", libffi7, libjemalloc2, libpq-dev, libprocps8, mina-logproc" ARCHIVE_DEPS="libssl1.1, libgomp1, libpq-dev, libjemalloc2" ;; - buster) - DAEMON_DEPS=", libffi6, libjemalloc2, libpq-dev, libprocps7, mina-logproc" - ARCHIVE_DEPS="libssl1.1, libgomp1, libpq-dev, libjemalloc2" - ;; stretch|bionic) DAEMON_DEPS=", libffi6, libjemalloc1, libpq-dev, libprocps6, mina-logproc" ARCHIVE_DEPS="libssl1.1, libgomp1, libpq-dev, libjemalloc1" diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index 65fce185d12..c61d3636a7e 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -37,7 +37,6 @@ SCRIPT=' set -x \ ' case $CODENAME in - buster) DOCKER_IMAGE="debian:buster" ;; bullseye) DOCKER_IMAGE="debian:bullseye" ;; focal) DOCKER_IMAGE="ubuntu:focal" ;; *) echo "Unknown codename passed: $CODENAME"; exit 1;; diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index c510304d01c..239904f8b09 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -22,7 +22,7 @@ function usage() { echo " -n, --network The network configuration to use (devnet or mainnet). Default=devnet" echo " -b, --branch The branch of the mina repository to use for staged docker builds. Default=compatible" echo " -r, --repo The currently used mina repository" - echo " --deb-codename The debian codename (stretch or buster) to build the docker image from. Default=stretch" + echo " --deb-codename The debian codename (bullseye or focal) to build the docker image from. Default=stretch" echo " --deb-release The debian package release channel to pull from (unstable,alpha,beta,stable). Default=unstable" echo " --deb-version The version string for the debian package to install" echo " --deb-profile The profile string for the debian package to install" @@ -56,7 +56,7 @@ case "${DEB_CODENAME##*=}" in bionic|focal|impish|jammy) IMAGE="ubuntu:${DEB_CODENAME##*=}" ;; - stretch|buster|bullseye|bookworm|sid) + stretch|bullseye|bookworm|sid) IMAGE="debian:${DEB_CODENAME##*=}-slim" ;; esac From fff5487f571dcde33d7b50f7302dd94c63239e31 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Tue, 24 Sep 2024 15:07:01 +0200 Subject: [PATCH 107/178] format: run `dune fmt` in src/app I checked that running `DISABLE_CHECK_OPAM_SWITCH=true make reformat` does not make any addition changes. --- src/app/archive/cli/dune | 39 +-- src/app/archive/dune | 47 ++- src/app/archive/lib/dune | 160 +++++----- src/app/archive_blocks/dune | 49 +-- src/app/batch_txn_tool/dune | 63 ++-- src/app/benchmarks/benchmarks.ml | 4 +- src/app/benchmarks/dune | 15 +- src/app/best_tip_merger/dune | 98 +++--- src/app/cli/src/cli_entrypoint/dune | 155 +++++----- src/app/cli/src/dune | 18 +- src/app/cli/src/init/dune | 279 ++++++++++-------- src/app/delegation_verify/dune | 93 +++--- src/app/disk_caching_stats/dune | 52 +++- src/app/dump_blocks/dump_blocks.ml | 124 ++++---- src/app/dump_blocks/dune | 48 +-- src/app/dump_blocks/encoding.ml | 81 +++-- src/app/extract_blocks/dune | 96 +++--- src/app/generate_keypair/dune | 6 +- src/app/graphql_schema_dump/dune | 17 +- src/app/heap_usage/dune | 109 +++---- src/app/ledger_export_bench/dune | 20 +- src/app/logproc/dune | 35 +-- src/app/logproc/logproc.ml | 56 ++-- src/app/missing_blocks_auditor/dune | 37 +-- src/app/print_blockchain_snark_vk/dune | 16 +- src/app/reformat/dune | 9 +- src/app/reformat/reformat.ml | 19 +- src/app/replayer/dune | 134 +++++---- src/app/rosetta/dune | 53 +++- src/app/rosetta/lib/dune | 3 +- src/app/rosetta/ocaml-signer/dune | 139 +++++---- src/app/runtime_genesis_ledger/dune | 51 ++-- src/app/snark_work_debugger/dune | 65 ++-- src/app/validate_keypair/dune | 23 +- src/app/zkapp_limits/dune | 30 +- src/app/zkapp_test_transaction/dune | 52 ++-- src/app/zkapp_test_transaction/lib/dune | 121 ++++---- src/app/zkapps_examples/add_events/dune | 29 +- src/app/zkapps_examples/calls/dune | 57 ++-- src/app/zkapps_examples/dune | 63 ++-- src/app/zkapps_examples/empty_update/dune | 48 +-- src/app/zkapps_examples/initialize_state/dune | 50 ++-- src/app/zkapps_examples/test/actions/dune | 3 +- src/app/zkapps_examples/test/add_events/dune | 77 ++--- src/app/zkapps_examples/test/calls/dune | 91 +++--- .../zkapps_examples/test/empty_update/dune | 90 +++--- .../test/initialize_state/dune | 91 +++--- .../test/optional_custom_gates/dune | 83 +++--- src/app/zkapps_examples/test/tokens/dune | 93 +++--- src/app/zkapps_examples/tokens/dune | 70 +++-- 50 files changed, 1737 insertions(+), 1524 deletions(-) diff --git a/src/app/archive/cli/dune b/src/app/archive/cli/dune index 69f4c7aa20c..2a815358764 100644 --- a/src/app/archive/cli/dune +++ b/src/app/archive/cli/dune @@ -2,22 +2,23 @@ (name archive_cli) (public_name archive.cli) (libraries - ;;opam libraries - async_kernel - caqti - archive_lib - core - async - caqti-async - core_kernel - async.async_command - ;;local libraries - genesis_constants - mina_runtime_config - cli_lib - logger - block_time - mina_version - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_mina))) + ;;opam libraries + async_kernel + caqti + archive_lib + core + async + caqti-async + core_kernel + async.async_command + ;;local libraries + genesis_constants + mina_runtime_config + cli_lib + logger + block_time + mina_version) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane ppx_mina))) diff --git a/src/app/archive/dune b/src/app/archive/dune index 27c2526bfc1..29b24b628ab 100644 --- a/src/app/archive/dune +++ b/src/app/archive/dune @@ -4,9 +4,18 @@ (public_name archive) (modules archive) (modes native) - (libraries archive_cli async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package archive) @@ -14,9 +23,19 @@ (public_name archive-testnet) (modules archive_testnet_signatures) (modes native) - (libraries archive_cli mina_signature_kind.testnet async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + mina_signature_kind.testnet + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package archive) @@ -24,6 +43,16 @@ (public_name archive-mainnet) (modules archive_mainnet_signatures) (modes native) - (libraries archive_cli mina_signature_kind.mainnet async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + mina_signature_kind.mainnet + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/archive/lib/dune b/src/app/archive/lib/dune index b24273b9de8..3752fa03a60 100644 --- a/src/app/archive/lib/dune +++ b/src/app/archive/lib/dune @@ -2,80 +2,88 @@ (name archive_lib) (public_name archive_lib) (libraries - ;;opam libraries - sexplib0 - async_unix - base64 - async_rpc_kernel - base.caml - base.base_internalhash_types - bin_prot.shape - async_kernel - caqti-driver-postgresql - core - async - core_kernel - caqti - caqti-async - async.async_rpc - integers - ppx_inline_test.config - uri - ;;local libraries - bounded_types - mina_wire_types - kimchi_backend - child_processes - precomputed_values - coda_genesis_ledger - consensus.vrf - mina_runtime_config - sgn - mina_base.util - kimchi_pasta - kimchi_pasta.basic - pickles - random_oracle - pipe_lib - block_time - currency - staged_ledger - data_hash_lib - transition_frontier_base - genesis_ledger_helper - one_or_two - mina_caqti - mina_generators - zkapp_command_builder - mina_base - mina_ledger - mina_block - transition_frontier - logger - mina_metrics - signature_lib - mina_numbers - unsigned_extended - mina_state - pickles_types - pickles.backend - with_hash - mina_base.import - consensus - genesis_constants - genesis_ledger_helper.lib - verifier - quickcheck_lib - snark_params - mina_transaction - protocol_version - staged_ledger_diff - error_json - ppx_deriving_yojson.runtime - ppx_version.runtime - o1trace - ) - (inline_tests (flags -verbose -show-counts)) + ;;opam libraries + sexplib0 + async_unix + base64 + async_rpc_kernel + base.caml + base.base_internalhash_types + bin_prot.shape + async_kernel + caqti-driver-postgresql + core + async + core_kernel + caqti + caqti-async + async.async_rpc + integers + ppx_inline_test.config + uri + ;;local libraries + bounded_types + mina_wire_types + kimchi_backend + child_processes + precomputed_values + coda_genesis_ledger + consensus.vrf + mina_runtime_config + sgn + mina_base.util + kimchi_pasta + kimchi_pasta.basic + pickles + random_oracle + pipe_lib + block_time + currency + staged_ledger + data_hash_lib + transition_frontier_base + genesis_ledger_helper + one_or_two + mina_caqti + mina_generators + zkapp_command_builder + mina_base + mina_ledger + mina_block + transition_frontier + logger + mina_metrics + signature_lib + mina_numbers + unsigned_extended + mina_state + pickles_types + pickles.backend + with_hash + mina_base.import + consensus + genesis_constants + genesis_ledger_helper.lib + verifier + quickcheck_lib + snark_params + mina_transaction + protocol_version + staged_ledger_diff + error_json + ppx_deriving_yojson.runtime + ppx_version.runtime + o1trace) + (inline_tests + (flags -verbose -show-counts)) (modes native) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf ppx_deriving_yojson h_list.ppx))) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_jane + ppx_custom_printf + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/archive_blocks/dune b/src/app/archive_blocks/dune index 48f0db5aabd..152986ce769 100644 --- a/src/app/archive_blocks/dune +++ b/src/app/archive_blocks/dune @@ -3,27 +3,28 @@ (name archive_blocks) (public_name archive_blocks) (libraries - ;; opam libraries - async_unix - core - result - async_kernel - uri - stdio - caqti-driver-postgresql - caqti - async - core_kernel - caqti-async - base - base.caml - async.async_command - ;; local libraries - logger - mina_block - bounded_types - genesis_constants - archive_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + async_unix + core + result + async_kernel + uri + stdio + caqti-driver-postgresql + caqti + async + core_kernel + caqti-async + base + base.caml + async.async_command + ;; local libraries + logger + mina_block + bounded_types + genesis_constants + archive_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) diff --git a/src/app/batch_txn_tool/dune b/src/app/batch_txn_tool/dune index f55ebfdf875..8b957922d9f 100644 --- a/src/app/batch_txn_tool/dune +++ b/src/app/batch_txn_tool/dune @@ -1,37 +1,34 @@ (executable (name batch_txn_tool) (libraries - ;; opam libraries - async.async_command - integers - base - uri - core_kernel - core - async - async_kernel - yojson - async_unix - ;; local libraries - currency - mina_base.import - graphql_lib - signature_lib - mina_base - secrets - mina_numbers - snark_params - unsigned_extended - mina_compile_config - mina_wire_types - integration_test_lib - logger - bounded_types - ) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + async.async_command + integers + base + uri + core_kernel + core + async + async_kernel + yojson + async_unix + ;; local libraries + currency + mina_base.import + graphql_lib + signature_lib + mina_base + secrets + mina_numbers + snark_params + unsigned_extended + mina_compile_config + mina_wire_types + integration_test_lib + logger + bounded_types) + (instrumentation + (backend bisect_ppx)) (preprocessor_deps ../../../graphql_schema.json) - (preprocess (pps - ppx_version - ppx_let - ppx_mina -))) + (preprocess + (pps ppx_version ppx_let ppx_mina))) diff --git a/src/app/benchmarks/benchmarks.ml b/src/app/benchmarks/benchmarks.ml index bc09c3c7b8f..0396bee99c6 100644 --- a/src/app/benchmarks/benchmarks.ml +++ b/src/app/benchmarks/benchmarks.ml @@ -3,11 +3,11 @@ (* you can control which libraries are benchmarked using the environment variable BENCHMARK_LIBRARIES, which is either "all" or a comma-delimited list of libraries; if the variable is not present, or empty, run benchmarks for all libraries - *) +*) open Core_kernel -let available_libraries = ["vrf_lib_tests"; "mina_base"] +let available_libraries = [ "vrf_lib_tests"; "mina_base" ] let run_benchmarks_in_lib libname = printf "Running inline tests in library \"%s\"\n%!" libname ; diff --git a/src/app/benchmarks/dune b/src/app/benchmarks/dune index 23918c24e7c..4052d497da6 100644 --- a/src/app/benchmarks/dune +++ b/src/app/benchmarks/dune @@ -3,8 +3,17 @@ (executable (name benchmarks) (public_name main) - (libraries core_bench.inline_benchmarks vrf_lib_tests mina_base core_kernel core base bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (libraries + core_bench.inline_benchmarks + vrf_lib_tests + mina_base + core_kernel + core + base + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (link_flags -linkall) (modes native)) diff --git a/src/app/best_tip_merger/dune b/src/app/best_tip_merger/dune index 2b8ab0c9e03..93c2ff1087b 100644 --- a/src/app/best_tip_merger/dune +++ b/src/app/best_tip_merger/dune @@ -1,50 +1,50 @@ (executable - (package best_tip_merger) - (name best_tip_merger) - (public_name best_tip_merger) - (modes native) - (libraries - ;; opam libraries - async.async_command - async_kernel - stdio - base.caml - yojson - lib - core_kernel - async - ppx_deriving_yojson.runtime - core - sexplib0 - result - base.base_internalhash_types - async_unix - ;; local libraries - visualization - mina_numbers - mina_state - transition_frontier_extensions - mina_base - logger.file_system - logger - cli_lib - transition_frontier - data_hash_lib - rose_tree - consensus - mina_wire_types - pickles.backend - snark_params - pickles - kimchi_pasta - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_jane - ppx_version - ppx_compare)) - ) + (package best_tip_merger) + (name best_tip_merger) + (public_name best_tip_merger) + (modes native) + (libraries + ;; opam libraries + async.async_command + async_kernel + stdio + base.caml + yojson + lib + core_kernel + async + ppx_deriving_yojson.runtime + core + sexplib0 + result + base.base_internalhash_types + async_unix + ;; local libraries + visualization + mina_numbers + mina_state + transition_frontier_extensions + mina_base + logger.file_system + logger + cli_lib + transition_frontier + data_hash_lib + rose_tree + consensus + mina_wire_types + pickles.backend + snark_params + pickles + kimchi_pasta + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_jane + ppx_version + ppx_compare))) diff --git a/src/app/cli/src/cli_entrypoint/dune b/src/app/cli/src/cli_entrypoint/dune index 9c321cf212c..cefa0dc0d61 100644 --- a/src/app/cli/src/cli_entrypoint/dune +++ b/src/app/cli/src/cli_entrypoint/dune @@ -5,77 +5,84 @@ (public_name cli.mina_cli_entrypoint) (modes native) (libraries - ;;opam libraries - stdio - async_kernel - async_unix - base - core_kernel - core - bin_prot - bin_prot.shape - init - async - core.daemon - base.caml - uri - async.async_command - result - sexplib0 - ;;local libraries - verifier - ledger_proof - snark_worker - mina_block - pipe_lib - gossip_net - data_hash_lib - mina_ledger - precomputed_values - mina_networking - cache_dir - parallel - secrets - logger - mina_lib - currency - signature_lib - mina_base - mina_base.import - error_json - genesis_ledger_helper - consensus - child_processes - node_addrs_and_ports - mina_plugins - logger.file_system - cli_lib - genesis_constants - blake2 - mina_metrics - transaction_witness - snark_work_lib - transaction_snark - mina_compile_config - node_error_service - mina_user_error - file_system - mina_version - mina_net2 - block_time - mina_runtime_config - trust_system - protocol_version - coda_genesis_proof - with_hash - block_producer - prover - blockchain_snark - snarky.backendless - o1trace - ppx_version.runtime - internal_tracing - itn_logger - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_here ppx_let ppx_sexp_conv ppx_deriving_yojson))) + ;;opam libraries + stdio + async_kernel + async_unix + base + core_kernel + core + bin_prot + bin_prot.shape + init + async + core.daemon + base.caml + uri + async.async_command + result + sexplib0 + ;;local libraries + verifier + ledger_proof + snark_worker + mina_block + pipe_lib + gossip_net + data_hash_lib + mina_ledger + precomputed_values + mina_networking + cache_dir + parallel + secrets + logger + mina_lib + currency + signature_lib + mina_base + mina_base.import + error_json + genesis_ledger_helper + consensus + child_processes + node_addrs_and_ports + mina_plugins + logger.file_system + cli_lib + genesis_constants + blake2 + mina_metrics + transaction_witness + snark_work_lib + transaction_snark + mina_compile_config + node_error_service + mina_user_error + file_system + mina_version + mina_net2 + block_time + mina_runtime_config + trust_system + protocol_version + coda_genesis_proof + with_hash + block_producer + prover + blockchain_snark + snarky.backendless + o1trace + ppx_version.runtime + internal_tracing + itn_logger) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_here + ppx_let + ppx_sexp_conv + ppx_deriving_yojson))) diff --git a/src/app/cli/src/dune b/src/app/cli/src/dune index ce6717a5cb4..d88c44698a4 100644 --- a/src/app/cli/src/dune +++ b/src/app/cli/src/dune @@ -7,8 +7,10 @@ (modules mina) (modes native) (libraries mina_cli_entrypoint) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) (executable @@ -18,8 +20,10 @@ (modules mina_testnet_signatures) (modes native) (libraries mina_cli_entrypoint mina_signature_kind.testnet) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) (executable @@ -29,6 +33,8 @@ (modules mina_mainnet_signatures) (modes native) (libraries mina_cli_entrypoint mina_signature_kind.mainnet) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) diff --git a/src/app/cli/src/init/dune b/src/app/cli/src/init/dune index aaa7df52dba..00a4dee6b0a 100644 --- a/src/app/cli/src/init/dune +++ b/src/app/cli/src/init/dune @@ -1,135 +1,158 @@ (rule (targets assets.ml) - (deps (source_tree assets)) - (action (run %{bin:ocaml-crunch} -m plain assets -o assets.ml))) + (deps + (source_tree assets)) + (action + (run %{bin:ocaml-crunch} -m plain assets -o assets.ml))) (library (name init) (public_name init) (libraries - ;;opam libraries - stdio - integers - astring - async_ssl - sexplib0 - async.async_rpc - result - graphql - async_kernel - async_unix - uri - core_kernel - core_kernel.uuid - core - core.uuid - base - async - cohttp - graphql_parser - async.async_command - async_rpc_kernel - base.caml - base.base_internalhash_types - graphql-cohttp - cohttp-async - graphql-async - mirage-crypto-ec - base_quickcheck - ;;local libraries - bounded_types - snark_profiler_lib - archive_lib - mina_wire_types - perf_histograms - one_or_two - block_time - child_processes - mina_net2 - snark_worker - allocation_functor - blockchain_snark - genesis_ledger_helper.lib - transaction_inclusion_status - error_json - network_pool - user_command_input - network_peer - logger - itn_logger - coda_genesis_proof - data_hash_lib - currency - genesis_constants - bash_colors - graphql_lib - o1trace_webkit_event - generated_graphql_queries - itn_crypto - mina_transaction - mina_generators - mina_user_error - mina_ledger - mina_block - mina_lib - mina_base - cli_lib - mina_graphql - mina_state - consensus - mina_version - o1trace - protocol_version - node_error_service - genesis_ledger_helper - signature_lib - mina_base.import - mina_numbers - precomputed_values - with_hash - transaction_snark - transaction_snark_tests - transaction_protocol_state - test_util - file_system - daemon_rpcs - trust_system - secrets - participating_state - staged_ledger - mina_commands - mina_runtime_config - coda_genesis_ledger - mina_networking - sgn - unsigned_extended - mina_metrics - group_map - snark_work_lib - mina_compile_config - mina_transaction_logic - verifier - parallel - random_oracle - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - pickles_types - pickles - pickles.backend - snark_params - mina_signature_kind - string_sign - zkapp_command_builder - internal_tracing - transaction_snark_scan_state - ) - (instrumentation (backend bisect_ppx)) - (preprocessor_deps ../../../../../graphql_schema.json - ../../../../graphql-ppx-config.inc) - (preprocess (pps ppx_deriving_yojson ppx_mina ppx_version - ppx_base ppx_assert ppx_bench ppx_bin_prot ppx_custom_printf - ppx_fields_conv ppx_fixed_literal ppx_here ppx_inline_test ppx_let - ppx_module_timer ppx_optional ppx_pipebang ppx_sexp_message - ppx_sexp_value ppx_string ppx_typerep_conv ppx_variants_conv - graphql_ppx -- %{read-lines:../../../../graphql-ppx-config.inc}))) + ;;opam libraries + stdio + integers + astring + async_ssl + sexplib0 + async.async_rpc + result + graphql + async_kernel + async_unix + uri + core_kernel + core_kernel.uuid + core + core.uuid + base + async + cohttp + graphql_parser + async.async_command + async_rpc_kernel + base.caml + base.base_internalhash_types + graphql-cohttp + cohttp-async + graphql-async + mirage-crypto-ec + base_quickcheck + ;;local libraries + bounded_types + snark_profiler_lib + archive_lib + mina_wire_types + perf_histograms + one_or_two + block_time + child_processes + mina_net2 + snark_worker + allocation_functor + blockchain_snark + genesis_ledger_helper.lib + transaction_inclusion_status + error_json + network_pool + user_command_input + network_peer + logger + itn_logger + coda_genesis_proof + data_hash_lib + currency + genesis_constants + bash_colors + graphql_lib + o1trace_webkit_event + generated_graphql_queries + itn_crypto + mina_transaction + mina_generators + mina_user_error + mina_ledger + mina_block + mina_lib + mina_base + cli_lib + mina_graphql + mina_state + consensus + mina_version + o1trace + protocol_version + node_error_service + genesis_ledger_helper + signature_lib + mina_base.import + mina_numbers + precomputed_values + with_hash + transaction_snark + transaction_snark_tests + transaction_protocol_state + test_util + file_system + daemon_rpcs + trust_system + secrets + participating_state + staged_ledger + mina_commands + mina_runtime_config + coda_genesis_ledger + mina_networking + sgn + unsigned_extended + mina_metrics + group_map + snark_work_lib + mina_compile_config + mina_transaction_logic + verifier + parallel + random_oracle + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + pickles_types + pickles + pickles.backend + snark_params + mina_signature_kind + string_sign + zkapp_command_builder + internal_tracing + transaction_snark_scan_state) + (instrumentation + (backend bisect_ppx)) + (preprocessor_deps + ../../../../../graphql_schema.json + ../../../../graphql-ppx-config.inc) + (preprocess + (pps + ppx_deriving_yojson + ppx_mina + ppx_version + ppx_base + ppx_assert + ppx_bench + ppx_bin_prot + ppx_custom_printf + ppx_fields_conv + ppx_fixed_literal + ppx_here + ppx_inline_test + ppx_let + ppx_module_timer + ppx_optional + ppx_pipebang + ppx_sexp_message + ppx_sexp_value + ppx_string + ppx_typerep_conv + ppx_variants_conv + graphql_ppx + -- + %{read-lines:../../../../graphql-ppx-config.inc}))) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index af2363a72c9..90b52ac1f3a 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,49 +1,50 @@ (executable (name delegation_verify) (libraries - core_kernel - async - async_kernel - async_unix - core - stdio - base - base.caml - ppx_deriving_yojson.runtime - yojson - base64 - integers - async.async_command - sexplib0 - sexplib - hex - ; mina libs - signature_lib - mina_block - transaction_snark - blockchain_snark - mina_base - genesis_constants - uptime_service - currency - ledger_proof - mina_base.import - mina_state - mina_wire_types - consensus - data_hash_lib - mina_numbers - snark_params - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - pickles - pickles_types - pickles.backend - genesis_ledger_helper - mina_runtime_config - precomputed_values - logger - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) + core_kernel + async + async_kernel + async_unix + core + stdio + base + base.caml + ppx_deriving_yojson.runtime + yojson + base64 + integers + async.async_command + sexplib0 + sexplib + hex + ; mina libs + signature_lib + mina_block + transaction_snark + blockchain_snark + mina_base + genesis_constants + uptime_service + currency + ledger_proof + mina_base.import + mina_state + mina_wire_types + consensus + data_hash_lib + mina_numbers + snark_params + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + genesis_ledger_helper + mina_runtime_config + precomputed_values + logger) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/disk_caching_stats/dune b/src/app/disk_caching_stats/dune index 57292875db8..6a8db8e8632 100644 --- a/src/app/disk_caching_stats/dune +++ b/src/app/disk_caching_stats/dune @@ -1,11 +1,45 @@ (executable (name disk_caching_stats) - (libraries pickles pickles_types pickles.backend snark_params crypto_params network_pool mina_base - signature_lib one_or_two currency ledger_proof mina_state mina_base.import mina_wire_types - mina_numbers random_oracle random_oracle_input kimchi_pasta.basic data_hash_lib with_hash - kimchi_pasta mina_ledger transaction_snark_scan_state mina_transaction_logic transaction_snark - sgn snark_profiler_lib genesis_constants digestif bigarray-compat - ; -- - core core_kernel base base.caml bin_prot splittable_random sexplib0) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_jane ppx_version))) + (libraries + pickles + pickles_types + pickles.backend + snark_params + crypto_params + network_pool + mina_base + signature_lib + one_or_two + currency + ledger_proof + mina_state + mina_base.import + mina_wire_types + mina_numbers + random_oracle + random_oracle_input + kimchi_pasta.basic + data_hash_lib + with_hash + kimchi_pasta + mina_ledger + transaction_snark_scan_state + mina_transaction_logic + transaction_snark + sgn + snark_profiler_lib + genesis_constants + digestif + bigarray-compat + ; -- + core + core_kernel + base + base.caml + bin_prot + splittable_random + sexplib0) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_jane ppx_version))) diff --git a/src/app/dump_blocks/dump_blocks.ml b/src/app/dump_blocks/dump_blocks.ml index f1ca08be9d4..8ecc364d923 100644 --- a/src/app/dump_blocks/dump_blocks.ml +++ b/src/app/dump_blocks/dump_blocks.ml @@ -8,10 +8,7 @@ open Full_frontier.For_tests Choice of the actual codec module depends on more than one CLI argument, which is why it must be delayed until all the CLI arguments are parsed. This choice is made by mk_io function below. *) -type 'a io = - { encoding : 'a - ; filename : string - } +type 'a io = { encoding : 'a; filename : string } type 'a codec = (module Encoding.S with type t = 'a) @@ -41,15 +38,20 @@ let encoded_block = Command.Arg_type.create (fun s -> match String.split ~on:':' s with | [ encoding; filename ] -> - { encoding= + { encoding = ( match String.lowercase encoding with - | "sexp" -> Encoding.Sexp - | "json" -> Json - | "bin" | "binary" -> Binary - | _ -> failwith "invalid encoding" ) + | "sexp" -> + Encoding.Sexp + | "json" -> + Json + | "bin" | "binary" -> + Binary + | _ -> + failwith "invalid encoding" ) ; filename } - | _ -> failwith "invalid format" ) + | _ -> + failwith "invalid format" ) let () = let open Core_kernel in @@ -62,18 +64,14 @@ let precomputed_values = Lazy.force Precomputed_values.for_unit_tests let constraint_constants = precomputed_values.constraint_constants -let output_block : type a. a -> a codec io -> unit - = fun content { encoding; filename } -> +let output_block : type a. a -> a codec io -> unit = + fun content { encoding; filename } -> let module Enc : Encoding.S with type t = a = (val encoding) in let channel = - match filename with - | "-" -> Out_channel.stdout - | s -> Out_channel.create s + match filename with "-" -> Out_channel.stdout | s -> Out_channel.create s in - fprintf channel "%s" (Enc.to_string content); - match filename with - | "-" -> () - | _ -> Out_channel.close channel + fprintf channel "%s" (Enc.to_string content) ; + match filename with "-" -> () | _ -> Out_channel.close channel (* This executable outputs random block to stderr in sexp and json The output is useful for src/lib/mina_block tests when the sexp/json representation changes. *) @@ -91,75 +89,67 @@ let f (type a) ?parent (outputs : a codec io list) make_breadcrumb = List.iter outputs ~f:(fun output -> let module Enc = (val output.encoding) in let content = - Enc.of_breadcrumb - ?with_parent_statehash:parent - breadcrumb + Enc.of_breadcrumb ?with_parent_statehash:parent breadcrumb in - eprintf !"Randomly generated block, %s: %s\n" + eprintf + !"Randomly generated block, %s: %s\n" Enc.name (match output.filename with "-" -> "" | s -> s) ; - output_block content output;); + output_block content output ) ; clean_up_persistent_root ~frontier ) let default_outputs = - [ { encoding= Encoding.Sexp; filename= "-" } - ; { encoding= Encoding.Json; filename= "-" } + [ { encoding = Encoding.Sexp; filename = "-" } + ; { encoding = Encoding.Json; filename = "-" } ] let command = Command.basic ~summary:"Transcribe between block formats or generate a random block" (let%map_open.Command outputs = - flag - "-o" - (listed encoded_block) - ~doc:"output :; formats are: sexp / json / bin; a dash (-) denotes stdout (example: -o json:-)" + flag "-o" (listed encoded_block) + ~doc: + "output :; formats are: sexp / json / bin; a dash (-) \ + denotes stdout (example: -o json:-)" and input = - flag - "-i" - (optional encoded_block) - ~doc:"input :; formats are: sexp / json / bin; a dash (-) denotes stdin (example: -i json:-)" + flag "-i" (optional encoded_block) + ~doc: + "input :; formats are: sexp / json / bin; a dash (-) \ + denotes stdin (example: -i json:-)" and full = - flag - "--full" - no_arg - ~doc:"use full blocks, not just precomputed values" + flag "--full" no_arg ~doc:"use full blocks, not just precomputed values" and parent = - flag - "--parent" - (optional string) - ~aliases:[ "--parent-statehash" ] + flag "--parent" (optional string) ~aliases:[ "--parent-statehash" ] ~doc:"statehash of the parent block" in - let outs = match outputs with - | [] -> default_outputs - | _ -> outputs - in + let outs = match outputs with [] -> default_outputs | _ -> outputs in let conf = if full then - Conf ( Option.map input ~f:(mk_io Block) - , List.map ~f:(mk_io Encoding.Block) outs) + Conf + ( Option.map input ~f:(mk_io Block) + , List.map ~f:(mk_io Encoding.Block) outs ) else - Conf ( Option.map input ~f:(mk_io Precomputed) - , List.map ~f:(mk_io Encoding.Precomputed) outs) + Conf + ( Option.map input ~f:(mk_io Precomputed) + , List.map ~f:(mk_io Encoding.Precomputed) outs ) in fun () -> - match conf with - | Conf (None, outs) -> - let verifier = verifier () in - Core_kernel.Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:1 ~f:(f ?parent outs) - | Conf (Some { encoding; filename }, outs)-> - let module Enc = (val encoding) in - let input = - match filename with - | "-" -> In_channel.stdin - | s -> In_channel.create s - in - let contents = - In_channel.input_all input - |> Enc.of_string - in - List.iter outs ~f:(output_block contents)) + match conf with + | Conf (None, outs) -> + let verifier = verifier () in + Core_kernel.Quickcheck.test + (gen_breadcrumb ~verifier ()) + ~trials:1 ~f:(f ?parent outs) + | Conf (Some { encoding; filename }, outs) -> + let module Enc = (val encoding) in + let input = + match filename with + | "-" -> + In_channel.stdin + | s -> + In_channel.create s + in + let contents = In_channel.input_all input |> Enc.of_string in + List.iter outs ~f:(output_block contents) ) let () = Command_unix.run command - diff --git a/src/app/dump_blocks/dune b/src/app/dump_blocks/dune index 1e499797f02..4c403c5a92d 100644 --- a/src/app/dump_blocks/dune +++ b/src/app/dump_blocks/dune @@ -2,27 +2,29 @@ (name dump_blocks) (public_name dump_blocks) (libraries - async - async_kernel - async_unix - base - block_time - bounded_types - core - core_kernel - core_unix.command_unix - logger - mina_base - mina_block - mina_state - mina_wire_types - ppx_inline_test.runner.lib - precomputed_values - sexplib0 - transition_frontier - transition_frontier_base - transition_frontier_full_frontier - yojson) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_custom_printf)) + async + async_kernel + async_unix + base + block_time + bounded_types + core + core_kernel + core_unix.command_unix + logger + mina_base + mina_block + mina_state + mina_wire_types + ppx_inline_test.runner.lib + precomputed_values + sexplib0 + transition_frontier + transition_frontier_base + transition_frontier_full_frontier + yojson) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_custom_printf)) (modes native)) diff --git a/src/app/dump_blocks/encoding.ml b/src/app/dump_blocks/encoding.ml index 1a246a1a2b9..dba55a5b52a 100644 --- a/src/app/dump_blocks/encoding.ml +++ b/src/app/dump_blocks/encoding.ml @@ -12,52 +12,49 @@ let block_of_breadcrumb ?with_parent_statehash breadcrumb = let open Mina_block in let block = Frontier_base.Breadcrumb.block breadcrumb in match with_parent_statehash with - | None -> - block - | Some hash -> - let previous_state_hash = Mina_base.State_hash.of_base58_check_exn hash in - let h = header block in - let state = Header.protocol_state h in - let header = - Header.create - ~protocol_state:{ state with previous_state_hash } - ~protocol_state_proof:(Header.protocol_state_proof h) - ~delta_block_chain_proof:(Header.delta_block_chain_proof h) - ?proposed_protocol_version_opt:(Header.proposed_protocol_version_opt h) - ~current_protocol_version:(Header.current_protocol_version h) - () - in - Mina_block.create ~header ~body:(body block) + | None -> + block + | Some hash -> + let previous_state_hash = Mina_base.State_hash.of_base58_check_exn hash in + let h = header block in + let state = Header.protocol_state h in + let header = + Header.create + ~protocol_state:{ state with previous_state_hash } + ~protocol_state_proof:(Header.protocol_state_proof h) + ~delta_block_chain_proof:(Header.delta_block_chain_proof h) + ?proposed_protocol_version_opt: + (Header.proposed_protocol_version_opt h) + ~current_protocol_version:(Header.current_protocol_version h) + () + in + Mina_block.create ~header ~body:(body block) module type S = sig type t val name : string - val of_breadcrumb : ?with_parent_statehash:string -> Frontier_base.Breadcrumb.t -> t + + val of_breadcrumb : + ?with_parent_statehash:string -> Frontier_base.Breadcrumb.t -> t val to_string : t -> string + val of_string : string -> t end -module Sexp_block : S with type t = Mina_block.t = struct +module Sexp_block : S with type t = Mina_block.t = struct type t = Mina_block.t let name = "sexp" let of_breadcrumb = block_of_breadcrumb + let to_string b = Mina_block.sexp_of_t b |> Sexp.to_string |> append_newline - let to_string b = - Mina_block.sexp_of_t b - |> Sexp.to_string - |> append_newline - - let of_string s = - Sexp.of_string s - |> Mina_block.t_of_sexp + let of_string s = Sexp.of_string s |> Mina_block.t_of_sexp end - module Binary_block : S with type t = Mina_block.t = struct type t = Mina_block.t @@ -84,16 +81,14 @@ let constraint_constants = precomputed_values.constraint_constants let precomputed_of_breadcrumb ?with_parent_statehash breadcrumb = let open Frontier_base in let block = block_of_breadcrumb ?with_parent_statehash breadcrumb in - let staged_ledger = - Transition_frontier.Breadcrumb.staged_ledger breadcrumb - in + let staged_ledger = Transition_frontier.Breadcrumb.staged_ledger breadcrumb in let scheduled_time = Mina_block.(Header.protocol_state @@ header block) |> Mina_state.Protocol_state.blockchain_state |> Mina_state.Blockchain_state.timestamp in - Mina_block.Precomputed.of_block ~logger ~constraint_constants - ~staged_ledger ~scheduled_time + Mina_block.Precomputed.of_block ~logger ~constraint_constants ~staged_ledger + ~scheduled_time (Breadcrumb.block_with_hash breadcrumb) module Sexp_precomputed : S with type t = Mina_block.Precomputed.t = struct @@ -104,16 +99,12 @@ module Sexp_precomputed : S with type t = Mina_block.Precomputed.t = struct let of_breadcrumb = precomputed_of_breadcrumb let to_string b = - Mina_block.Precomputed.sexp_of_t b - |> Sexp.to_string - |> append_newline + Mina_block.Precomputed.sexp_of_t b |> Sexp.to_string |> append_newline - let of_string s = - Sexp.of_string s - |> Mina_block.Precomputed.t_of_sexp + let of_string s = Sexp.of_string s |> Mina_block.Precomputed.t_of_sexp end -module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct +module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct type t = Mina_block.Precomputed.t let name = "json" @@ -122,12 +113,10 @@ module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct let to_string b = Mina_block.Precomputed.to_yojson b - |> Yojson.Safe.to_string - |> append_newline + |> Yojson.Safe.to_string |> append_newline let of_string s = - Yojson.Safe.from_string s - |> Mina_block.Precomputed.of_yojson + Yojson.Safe.from_string s |> Mina_block.Precomputed.of_yojson |> Result.ok_or_failwith end @@ -138,9 +127,7 @@ module Binary_precomputed : S with type t = Mina_block.Precomputed.t = struct let of_breadcrumb = precomputed_of_breadcrumb - let to_string = - Binable.to_string (module Mina_block.Precomputed.Stable.Latest) + let to_string = Binable.to_string (module Mina_block.Precomputed.Stable.Latest) - let of_string = - Binable.of_string (module Mina_block.Precomputed.Stable.Latest) + let of_string = Binable.of_string (module Mina_block.Precomputed.Stable.Latest) end diff --git a/src/app/extract_blocks/dune b/src/app/extract_blocks/dune index a41ea72fc19..1b31d5f0471 100644 --- a/src/app/extract_blocks/dune +++ b/src/app/extract_blocks/dune @@ -3,47 +3,55 @@ (name extract_blocks) (public_name extract_blocks) (libraries - ;; opam libraries - async_unix - core - base - base64 - integers - caqti-async - caqti-driver-postgresql - async - core_kernel - caqti - async_kernel - uri - async.async_command - ;; local libraries - consensus.vrf - mina_wire_types - bounded_types - mina_base - mina_base.import - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - pickles - pickles_types - pickles.backend - archive_lib - data_hash_lib - signature_lib - logger - mina_caqti - block_time - mina_numbers - mina_transaction - protocol_version - snark_params - currency - unsigned_extended - sgn - with_hash - error_json - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv h_list.ppx))) + ;; opam libraries + async_unix + core + base + base64 + integers + caqti-async + caqti-driver-postgresql + async + core_kernel + caqti + async_kernel + uri + async.async_command + ;; local libraries + consensus.vrf + mina_wire_types + bounded_types + mina_base + mina_base.import + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + archive_lib + data_hash_lib + signature_lib + logger + mina_caqti + block_time + mina_numbers + mina_transaction + protocol_version + snark_params + currency + unsigned_extended + sgn + with_hash + error_json) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_version + ppx_mina + ppx_let + ppx_hash + ppx_compare + ppx_sexp_conv + h_list.ppx))) diff --git a/src/app/generate_keypair/dune b/src/app/generate_keypair/dune index 1a987e63659..6ef13957692 100644 --- a/src/app/generate_keypair/dune +++ b/src/app/generate_keypair/dune @@ -4,6 +4,8 @@ (public_name generate_keypair) (modes native) (libraries cli_lib async core_kernel mina_version async_unix bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_sexp_conv)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) diff --git a/src/app/graphql_schema_dump/dune b/src/app/graphql_schema_dump/dune index 7116105997b..6294dcf3ae4 100644 --- a/src/app/graphql_schema_dump/dune +++ b/src/app/graphql_schema_dump/dune @@ -1,5 +1,16 @@ (executable (name graphql_schema_dump) - (libraries graphql_parser mina_version mina_graphql async async_unix graphql-async yojson bounded_types genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + graphql_parser + mina_version + mina_graphql + async + async_unix + graphql-async + yojson + bounded_types + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/heap_usage/dune b/src/app/heap_usage/dune index a74b285b81f..e9b41204cb6 100644 --- a/src/app/heap_usage/dune +++ b/src/app/heap_usage/dune @@ -3,57 +3,58 @@ (name heap_usage) (public_name heap_usage) (libraries - ;; opam libraries - async_unix - core - result - async_kernel - stdio - async - core_kernel - base - base.caml - base.base_internalhash_types - async.async_command - yojson - sexplib0 - ;; local libraries - mina_block ; temp - mina_base - mina_base.import - mina_numbers - currency - bounded_types - data_hash_lib - signature_lib - merkle_ledger - mina_compile_config - mina_ledger - mina_state - mina_transaction_logic - genesis_constants - staged_ledger_diff - parallel_scan - pickles - pickles.backend - pickles_types - transaction_snark - transaction_snark_scan_state - ledger_proof - blake2 - mina_generators - with_hash - random_oracle - snark_params - crypto_params - kimchi_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - zkapp_command_builder - transaction_snark_tests - snark_profiler_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + async_unix + core + result + async_kernel + stdio + async + core_kernel + base + base.caml + base.base_internalhash_types + async.async_command + yojson + sexplib0 + ;; local libraries + mina_block ; temp + mina_base + mina_base.import + mina_numbers + currency + bounded_types + data_hash_lib + signature_lib + merkle_ledger + mina_compile_config + mina_ledger + mina_state + mina_transaction_logic + genesis_constants + staged_ledger_diff + parallel_scan + pickles + pickles.backend + pickles_types + transaction_snark + transaction_snark_scan_state + ledger_proof + blake2 + mina_generators + with_hash + random_oracle + snark_params + crypto_params + kimchi_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + zkapp_command_builder + transaction_snark_tests + snark_profiler_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) diff --git a/src/app/ledger_export_bench/dune b/src/app/ledger_export_bench/dune index 2352ffd4130..6ee4c2a05c7 100644 --- a/src/app/ledger_export_bench/dune +++ b/src/app/ledger_export_bench/dune @@ -1,15 +1,17 @@ (executable (name ledger_export_benchmark) (libraries - base - core - core_bench - core_kernel - yojson - ; mina libraries - mina_runtime_config) + base + core + core_bench + core_kernel + yojson + ; mina libraries + mina_runtime_config) ; the -w list here should be the same as in src/dune (flags -short-paths -g -w @a-4-29-40-41-42-44-45-48-58-59-60) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane)) (modes native)) diff --git a/src/app/logproc/dune b/src/app/logproc/dune index 4d38e6278e4..77ac3b6d14e 100644 --- a/src/app/logproc/dune +++ b/src/app/logproc/dune @@ -3,20 +3,21 @@ (public_name logproc) (modules logproc) (libraries - ;;opam libraries - result - stdio - core - yojson - cmdliner - core_kernel - ppx_deriving_yojson.runtime - ;;local libraries - logger - logproc_lib - bounded_types - bash_colors - interpolator_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_deriving.std))) + ;;opam libraries + result + stdio + core + yojson + cmdliner + core_kernel + ppx_deriving_yojson.runtime + ;;local libraries + logger + logproc_lib + bounded_types + bash_colors + interpolator_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane ppx_deriving.std))) diff --git a/src/app/logproc/logproc.ml b/src/app/logproc/logproc.ml index 8c519b962f1..44a1f7d3fa9 100644 --- a/src/app/logproc/logproc.ml +++ b/src/app/logproc/logproc.ml @@ -81,8 +81,9 @@ let process_line ~timezone ~interpolation_config ~filter line = type parser_state = | Start_of_line - | Processable_line of {line_start_pos: int; previous_elements: string list} - | Unprocessable_line of {line_start_pos: int} + | Processable_line of + { line_start_pos : int; previous_elements : string list } + | Unprocessable_line of { line_start_pos : int } (* Iterates over lines prefixed by `prefix`, invoking `on_hit` for each * line. If a line does not start with `prefix`, `on_miss` is invoked, @@ -105,11 +106,11 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit match state with | Start_of_line -> failwith "cannot extract start of line state" - | Processable_line {line_start_pos; previous_elements} -> + | Processable_line { line_start_pos; previous_elements } -> let this_element = extract_input_string line_start_pos i in let elements = List.rev (this_element :: previous_elements) in on_hit (String.concat elements ~sep:"") - | Unprocessable_line {line_start_pos} -> + | Unprocessable_line { line_start_pos } -> forward_input line_start_pos i ; Out_channel.output_char output_channel '\n' in @@ -117,14 +118,15 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit match state with | Start_of_line -> Start_of_line - | Processable_line {line_start_pos; previous_elements} -> + | Processable_line { line_start_pos; previous_elements } -> let this_element = extract_input_string line_start_pos buf_len in Processable_line - { line_start_pos= 0 - ; previous_elements= this_element :: previous_elements } - | Unprocessable_line {line_start_pos} -> + { line_start_pos = 0 + ; previous_elements = this_element :: previous_elements + } + | Unprocessable_line { line_start_pos } -> forward_input line_start_pos buf_len ; - Unprocessable_line {line_start_pos= 0} + Unprocessable_line { line_start_pos = 0 } in let parse_char ~buf_len i state = if i >= buf_len then `Stop (carryover_state ~buf_len state) @@ -133,10 +135,10 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit | Start_of_line -> let next_state = if Char.equal (Bytes.unsafe_get input_buffer i) prefix then - Processable_line {line_start_pos= i; previous_elements= []} + Processable_line { line_start_pos = i; previous_elements = [] } else ( on_miss () ; - Unprocessable_line {line_start_pos= i} ) + Unprocessable_line { line_start_pos = i } ) in `Continue next_state | Processable_line _ | Unprocessable_line _ -> @@ -209,9 +211,9 @@ let () = Arg.( value & opt - (enum [("hidden", Hidden); ("inline", Inline); ("after", After)]) + (enum [ ("hidden", Hidden); ("inline", Inline); ("after", After) ]) Inline - & info ["i"; "interpolation-mode"] ~docv:"MODE" ~doc) + & info [ "i"; "interpolation-mode" ] ~docv:"MODE" ~doc) in let max_interpolation_length = let doc = @@ -221,16 +223,17 @@ let () = Arg.( value & opt int 25 & info - ["m"; "max-interpolation-length"] + [ "m"; "max-interpolation-length" ] ~docv:"MAX_INTERPOLATION_LENGTH" ~doc) in let pretty_print = let doc = "Pretty print json values." in - Arg.(value & flag & info ["p"; "pretty-print"] ~docv:"PRETTY_PRINT" ~doc) + Arg.( + value & flag & info [ "p"; "pretty-print" ] ~docv:"PRETTY_PRINT" ~doc) in let lift_interpolation_config mode max_interpolation_length pretty_print = let open Interpolator in - {mode; max_interpolation_length; pretty_print} + { mode; max_interpolation_length; pretty_print } in Term.( const lift_interpolation_config @@ -240,23 +243,23 @@ let () = let doc = "Timezone to display timestamps in. Defaults to the system's timezone." in - Arg.(value & opt string "" & info ["z"; "zone"] ~docv:"TIMEZONE" ~doc) + Arg.(value & opt string "" & info [ "z"; "zone" ] ~docv:"TIMEZONE" ~doc) in let filter = let doc = "Filter displayed log lines. The filter language has similar syntax to \ - javascript, with a few notable differences. Similar to \"jq\", doing \ - an anymous access like \"[\"a\"]\" or \".a\" will refer to that key on \ - the javascript object being processed, which in the context of \ - logproc, is the json log entry itself. Basic literals (such as strings \ - and integers) are supported, as is structural equality (\"==\") and \ - boolean operations (\"!\", \"&&\", \"||\"). There is also support for \ - the \"in\" operator for checking existence in arrays (works like \ + javascript, with a few notable differences. Similar to \"jq\", doing an \ + anymous access like \"[\"a\"]\" or \".a\" will refer to that key on the \ + javascript object being processed, which in the context of logproc, is \ + the json log entry itself. Basic literals (such as strings and \ + integers) are supported, as is structural equality (\"==\") and boolean \ + operations (\"!\", \"&&\", \"||\"). There is also support for the \ + \"in\" operator for checking existence in arrays (works like \ javascript, not like jq). Regexes can also be expressed \ (\"/some_regex/\") and can be matched on using a special \"match\" \ operator. See examples for more information." in - Arg.(value & opt string "" & info ["f"; "filter"] ~docv:"FILTER" ~doc) + Arg.(value & opt string "" & info [ "f"; "filter" ] ~docv:"FILTER" ~doc) in let main_term = Term.(const main $ timezone $ interpolation_config $ filter) @@ -289,7 +292,8 @@ let () = ; `I ( "a complex filter:" , {|logproc -f '.message match /broadcast/ && .metadata.peer.host == "182.9.63.3" || .metadata.peer.discover_port == 8302|} - ) ] + ) + ] in Term.info ~version:"0.1" ~doc ~exits:Term.default_exits ~man "logproc" in diff --git a/src/app/missing_blocks_auditor/dune b/src/app/missing_blocks_auditor/dune index 49e9020bc1c..a08801fafa2 100644 --- a/src/app/missing_blocks_auditor/dune +++ b/src/app/missing_blocks_auditor/dune @@ -3,21 +3,22 @@ (name missing_blocks_auditor) (public_name missing_blocks_auditor) (libraries - ;; opam libraries - core - async_kernel - uri - caqti-driver-postgresql - caqti - async - core_kernel - caqti-async - base - async_unix - async.async_command - ;; local libraries - logger - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let))) + ;; opam libraries + core + async_kernel + uri + caqti-driver-postgresql + caqti + async + core_kernel + caqti-async + base + async_unix + async.async_command + ;; local libraries + logger + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_let))) diff --git a/src/app/print_blockchain_snark_vk/dune b/src/app/print_blockchain_snark_vk/dune index 28872e1df1e..ad5287fc4fd 100644 --- a/src/app/print_blockchain_snark_vk/dune +++ b/src/app/print_blockchain_snark_vk/dune @@ -1,19 +1,23 @@ (executable (name print_blockchain_snark_vk) - (libraries - blockchain_snark genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries blockchain_snark genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (rule (deps print_blockchain_snark_vk.exe) (targets blockchain_snark_vk.json.computed) (action - (with-stdout-to %{targets} + (with-stdout-to + %{targets} (run %{deps})))) (rule - (deps (:orig blockchain_snark_vk.json) (:computed blockchain_snark_vk.json.computed)) + (deps + (:orig blockchain_snark_vk.json) + (:computed blockchain_snark_vk.json.computed)) (alias runtest) (action (diff %{orig} %{computed}))) diff --git a/src/app/reformat/dune b/src/app/reformat/dune index e069e49a5da..c0fbc0fa245 100644 --- a/src/app/reformat/dune +++ b/src/app/reformat/dune @@ -1,7 +1,14 @@ (executable (name reformat) (public_name reformat) - (libraries core async async_kernel async_unix core_kernel sexplib0 async.async_command) + (libraries + core + async + async_kernel + async_unix + core_kernel + sexplib0 + async.async_command) (preprocess (pps ppx_jane)) ; the -w list here should be the same as in src/dune diff --git a/src/app/reformat/reformat.ml b/src/app/reformat/reformat.ml index 0ea294650e1..e71dad30c28 100644 --- a/src/app/reformat/reformat.ml +++ b/src/app/reformat/reformat.ml @@ -19,7 +19,8 @@ let dirs_trustlist = ; "snarky" ; "_opam" ; "opam_switches" - ; ".direnv" ] + ; ".direnv" + ] let rec fold_over_files ~path ~process_path ~init ~f = let%bind all = Sys.ls_dir path in @@ -42,11 +43,11 @@ let main dry_run check path = | `Dir -> not (List.exists dirs_trustlist ~f:(fun s -> - String.is_suffix ~suffix:s path )) + String.is_suffix ~suffix:s path ) ) | `File -> (not (List.exists trustlist ~f:(fun s -> - String.is_suffix ~suffix:s path ))) + String.is_suffix ~suffix:s path ) ) ) && ( String.is_suffix ~suffix:".ml" path || String.is_suffix ~suffix:".mli" path ) ) ~f:(fun () file -> @@ -55,7 +56,7 @@ let main dry_run check path = return () in if check then - let prog, args = ("ocamlformat", ["--doc-comments=before"; file]) in + let prog, args = ("ocamlformat", [ "--doc-comments=before"; file ]) in let%bind formatted = Process.run_exn ~prog ~args () in let%bind raw = Reader.file_contents file in if not (String.equal formatted raw) then ( @@ -64,7 +65,7 @@ let main dry_run check path = else return () else let prog, args = - ("ocamlformat", ["--doc-comments=before"; "-i"; file]) + ("ocamlformat", [ "--doc-comments=before"; "-i"; file ]) in if dry_run then dump prog args else @@ -77,16 +78,16 @@ let _cli = let open Command.Let_syntax in Command.async ~summary:"Format all ml and mli files" (let%map_open path = - flag "--path" ~aliases:["path"] ~doc:"Path to traverse" + flag "--path" ~aliases:[ "path" ] ~doc:"Path to traverse" (required string) - and dry_run = flag "--dry-run" ~aliases:["dry-run"] no_arg ~doc:"Dry run" + and dry_run = flag "--dry-run" ~aliases:[ "dry-run" ] no_arg ~doc:"Dry run" and check = - flag "--check" ~aliases:["check"] no_arg + flag "--check" ~aliases:[ "check" ] no_arg ~doc: "Return with an error code if there exists an ml file that was \ formatted improperly" in - fun () -> main dry_run check path) + fun () -> main dry_run check path ) |> Command.run let () = never_returns (Scheduler.go ()) diff --git a/src/app/replayer/dune b/src/app/replayer/dune index 394d304b180..8cd2e071d22 100644 --- a/src/app/replayer/dune +++ b/src/app/replayer/dune @@ -3,66 +3,74 @@ (name replayer) (public_name replayer) (libraries - ;; opam libraries - stdio - sexplib0 - result - async.async_command - async - base.caml - bin_prot.shape - caqti-async - core_kernel - archive_lib - caqti-driver-postgresql - async_kernel - core - yojson - caqti - base - base.base_internalhash_types - uri - integers - async_unix - ;; local libraries - mina_wire_types - with_hash - snark_params - crypto_params - pickles_types - mina_numbers - coda_genesis_ledger - currency - pickles.backend - signature_lib - genesis_ledger_helper.lib - genesis_ledger_helper - mina_state - mina_ledger - mina_base - bounded_types - mina_base.import - mina_transaction - mina_transaction_logic - mina_caqti - genesis_constants - mina_runtime_config - logger - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - kimchi_backend_common - block_time - data_hash_lib - pickles - sgn - unsigned_extended - random_oracle - codable - ppx_version.runtime - cli_lib - mina_version - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx))) + ;; opam libraries + stdio + sexplib0 + result + async.async_command + async + base.caml + bin_prot.shape + caqti-async + core_kernel + archive_lib + caqti-driver-postgresql + async_kernel + core + yojson + caqti + base + base.base_internalhash_types + uri + integers + async_unix + ;; local libraries + mina_wire_types + with_hash + snark_params + crypto_params + pickles_types + mina_numbers + coda_genesis_ledger + currency + pickles.backend + signature_lib + genesis_ledger_helper.lib + genesis_ledger_helper + mina_state + mina_ledger + mina_base + bounded_types + mina_base.import + mina_transaction + mina_transaction_logic + mina_caqti + genesis_constants + mina_runtime_config + logger + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + kimchi_backend_common + block_time + data_hash_lib + pickles + sgn + unsigned_extended + random_oracle + codable + ppx_version.runtime + cli_lib + mina_version) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/rosetta/dune b/src/app/rosetta/dune index 83be7c2bc27..d1120e13d02 100644 --- a/src/app/rosetta/dune +++ b/src/app/rosetta/dune @@ -4,10 +4,19 @@ (public_name rosetta) (modules rosetta) (modes native) - (libraries lib async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package rosetta) @@ -15,10 +24,20 @@ (public_name rosetta-testnet) (modules rosetta_testnet_signatures) (modes native) - (libraries lib mina_signature_kind.testnet async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + mina_signature_kind.testnet + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package rosetta) @@ -26,7 +45,17 @@ (public_name rosetta-mainnet) (modules rosetta_mainnet_signatures) (modes native) - (libraries lib mina_signature_kind.mainnet async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + mina_signature_kind.mainnet + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/rosetta/lib/dune b/src/app/rosetta/lib/dune index 9c6bf25cd02..7515da8d47c 100644 --- a/src/app/rosetta/lib/dune +++ b/src/app/rosetta/lib/dune @@ -51,8 +51,7 @@ hex mina_numbers mina_transaction - mina_version - ) + mina_version) (preprocessor_deps ../../../graphql-ppx-config.inc ../../../../graphql_schema.json) diff --git a/src/app/rosetta/ocaml-signer/dune b/src/app/rosetta/ocaml-signer/dune index f72365e98d9..f4ac9458523 100644 --- a/src/app/rosetta/ocaml-signer/dune +++ b/src/app/rosetta/ocaml-signer/dune @@ -3,40 +3,39 @@ (public_name signer.cli) (modules signer_cli) (libraries - ;;opam libraries - async_kernel - core - core_kernel - async - base - lib - async_unix - async.async_command - ;;local libraries - kimchi_pasta - kimchi_pasta.basic - pickles - rosetta_coding - rosetta_lib - string_sign - signature_lib - mina_base - snark_params - pickles.backend - secrets - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + ;;opam libraries + async_kernel + core + core_kernel + async + base + lib + async_unix + async.async_command + ;;local libraries + kimchi_pasta + kimchi_pasta.basic + pickles + rosetta_coding + rosetta_lib + string_sign + signature_lib + mina_base + snark_params + pickles.backend + secrets) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -45,18 +44,18 @@ (modes native) (modules signer) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -65,18 +64,18 @@ (modes native) (modules signer_testnet_signatures) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -85,15 +84,15 @@ (modes native) (modules signer_mainnet_signatures) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) diff --git a/src/app/runtime_genesis_ledger/dune b/src/app/runtime_genesis_ledger/dune index 0a1191a086d..a6dba29e46b 100644 --- a/src/app/runtime_genesis_ledger/dune +++ b/src/app/runtime_genesis_ledger/dune @@ -2,28 +2,29 @@ (name runtime_genesis_ledger) (public_name runtime_genesis_ledger) (libraries - ;;opam libraries - base - core_kernel - async_kernel - async - core - result - async.async_command - yojson - async_unix - ;;local libraries - genesis_constants - mina_runtime_config - mina_ledger - genesis_ledger_helper - bounded_types - logger - cache_dir - precomputed_values - coda_genesis_ledger - consensus - mina_base - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let ppx_deriving_yojson))) + ;;opam libraries + base + core_kernel + async_kernel + async + core + result + async.async_command + yojson + async_unix + ;;local libraries + genesis_constants + mina_runtime_config + mina_ledger + genesis_ledger_helper + bounded_types + logger + cache_dir + precomputed_values + coda_genesis_ledger + consensus + mina_base) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_let ppx_deriving_yojson))) diff --git a/src/app/snark_work_debugger/dune b/src/app/snark_work_debugger/dune index 0f8b91aaea9..9eda39a4279 100644 --- a/src/app/snark_work_debugger/dune +++ b/src/app/snark_work_debugger/dune @@ -4,33 +4,40 @@ (public_name snark_work_debugger) (modes native) (libraries - ;; opam libraries - base - async - async_kernel - async_unix - unix - core - core_kernel - sexplib0 - ;; local libraries - mina_base - snark_worker - bounded_types - snark_work_lib - transaction_witness - transaction_snark - genesis_constants - logger - error_json - transaction_protocol_state - mina_state - currency - mina_ledger - signature_lib - mina_base.import - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_here ppx_custom_printf ppx_version ppx_mina ppx_let ppx_deriving_yojson ppx_sexp_conv)) + ;; opam libraries + base + async + async_kernel + async_unix + unix + core + core_kernel + sexplib0 + ;; local libraries + mina_base + snark_worker + bounded_types + snark_work_lib + transaction_witness + transaction_snark + genesis_constants + logger + error_json + transaction_protocol_state + mina_state + currency + mina_ledger + signature_lib + mina_base.import) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_here + ppx_custom_printf + ppx_version + ppx_mina + ppx_let + ppx_deriving_yojson + ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) - diff --git a/src/app/validate_keypair/dune b/src/app/validate_keypair/dune index 78e187b0fa9..4346b5856f5 100644 --- a/src/app/validate_keypair/dune +++ b/src/app/validate_keypair/dune @@ -4,15 +4,16 @@ (public_name validate_keypair) (modes native) (libraries - ;; opam libraries - core_kernel - async - async_unix - ;; local libraries - mina_version - bounded_types - cli_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_sexp_conv)) + ;; opam libraries + core_kernel + async + async_unix + ;; local libraries + mina_version + bounded_types + cli_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) diff --git a/src/app/zkapp_limits/dune b/src/app/zkapp_limits/dune index 6cb781b71b2..2c50a3c87ac 100644 --- a/src/app/zkapp_limits/dune +++ b/src/app/zkapp_limits/dune @@ -3,14 +3,22 @@ (name zkapp_limits) (public_name zkapp_limits) (libraries - ;; opam libraries - base - base.caml - core_kernel - ;; local libraries - mina_base - genesis_constants - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_custom_printf ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + base + base.caml + core_kernel + ;; local libraries + mina_base + genesis_constants + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_version + ppx_mina + ppx_custom_printf + ppx_let + ppx_hash + ppx_compare + ppx_sexp_conv))) diff --git a/src/app/zkapp_test_transaction/dune b/src/app/zkapp_test_transaction/dune index ca1edcbdd3c..c81e6d48a82 100644 --- a/src/app/zkapp_test_transaction/dune +++ b/src/app/zkapp_test_transaction/dune @@ -3,24 +3,34 @@ (name zkapp_test_transaction) (public_name zkapp_test_transaction) (libraries - ;;opam libraries - async.async_command - base - core_kernel - core - async_kernel - zkapp_test_transaction_lib - async - async_unix - ;;local libraries - mina_graphql - mina_numbers - currency - bounded_types - cli_lib - mina_base - signature_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_custom_printf ppx_mina ppx_assert ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx))) + ;;opam libraries + async.async_command + base + core_kernel + core + async_kernel + zkapp_test_transaction_lib + async + async_unix + ;;local libraries + mina_graphql + mina_numbers + currency + bounded_types + cli_lib + mina_base + signature_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_custom_printf + ppx_mina + ppx_assert + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/zkapp_test_transaction/lib/dune b/src/app/zkapp_test_transaction/lib/dune index 3b0f5a18ded..81420ca2622 100644 --- a/src/app/zkapp_test_transaction/lib/dune +++ b/src/app/zkapp_test_transaction/lib/dune @@ -2,58 +2,69 @@ (name zkapp_test_transaction_lib) (public_name zkapp_test_transaction_lib) (libraries - ;;opam libraries - async_unix - graphql_parser - splittable_random - result - async - async_kernel - core - core_kernel - base - sexplib0 - base_quickcheck - ppx_inline_test.config - graphql-async - ;;local libraries - pickles - kimchi_pasta - kimchi_pasta.basic - bounded_types - random_oracle - mina_runtime_config - transaction_witness - transaction_protocol_state - coda_genesis_ledger - pickles_types - mina_numbers - random_oracle_input - genesis_ledger_helper - mina_state - mina_generators - mina_base - currency - mina_ledger - mina_graphql - genesis_constants - secrets - transaction_snark - signature_lib - data_hash_lib - consensus - with_hash - genesis_ledger_helper.lib - snark_params - pickles.backend - kimchi_backend - mina_base.import - mina_transaction_logic - mina_transaction - staged_ledger_diff - mina_wire_types - ) - (inline_tests (flags -verbose -show-counts)) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_custom_printf ppx_mina ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx ppx_inline_test))) + ;;opam libraries + async_unix + graphql_parser + splittable_random + result + async + async_kernel + core + core_kernel + base + sexplib0 + base_quickcheck + ppx_inline_test.config + graphql-async + ;;local libraries + pickles + kimchi_pasta + kimchi_pasta.basic + bounded_types + random_oracle + mina_runtime_config + transaction_witness + transaction_protocol_state + coda_genesis_ledger + pickles_types + mina_numbers + random_oracle_input + genesis_ledger_helper + mina_state + mina_generators + mina_base + currency + mina_ledger + mina_graphql + genesis_constants + secrets + transaction_snark + signature_lib + data_hash_lib + consensus + with_hash + genesis_ledger_helper.lib + snark_params + pickles.backend + kimchi_backend + mina_base.import + mina_transaction_logic + mina_transaction + staged_ledger_diff + mina_wire_types) + (inline_tests + (flags -verbose -show-counts)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_custom_printf + ppx_mina + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx + ppx_inline_test))) diff --git a/src/app/zkapps_examples/add_events/dune b/src/app/zkapps_examples/add_events/dune index 618100a98a1..d8b70325b92 100644 --- a/src/app/zkapps_examples/add_events/dune +++ b/src/app/zkapps_examples/add_events/dune @@ -1,19 +1,20 @@ (library (name zkapps_add_events) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - mina_base - pickles - pickles_types - snarky.backendless - snark_params - signature_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + mina_base + pickles + pickles_types + snarky.backendless + snark_params + signature_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess (pps ppx_version))) diff --git a/src/app/zkapps_examples/calls/dune b/src/app/zkapps_examples/calls/dune index eaaa1d87239..0bf875a0810 100644 --- a/src/app/zkapps_examples/calls/dune +++ b/src/app/zkapps_examples/calls/dune @@ -1,33 +1,32 @@ (library (name zkapps_calls) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - pickles - pickles.backend - pickles_types - random_oracle - random_oracle_input - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - with_hash - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + pickles + pickles.backend + pickles_types + random_oracle + random_oracle_input + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + with_hash + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version - h_list.ppx))) + (pps ppx_version h_list.ppx))) diff --git a/src/app/zkapps_examples/dune b/src/app/zkapps_examples/dune index 1db21807142..393112daabf 100644 --- a/src/app/zkapps_examples/dune +++ b/src/app/zkapps_examples/dune @@ -2,36 +2,35 @@ (name zkapps_examples) (public_name zkapps_examples) (libraries - ;; opam libraries - base - async_kernel - core_kernel - ;; local libraries - mina_numbers - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - mina_ledger - pickles - pickles.backend - pickles_types - random_oracle - random_oracle_input - snarky.backendless - snark_params - bounded_types - sgn - signature_lib - tuple_lib - with_hash) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + async_kernel + core_kernel + ;; local libraries + mina_numbers + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + mina_ledger + pickles + pickles.backend + pickles_types + random_oracle + random_oracle_input + snarky.backendless + snark_params + bounded_types + sgn + signature_lib + tuple_lib + with_hash) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_let - ppx_version))) + (pps ppx_let ppx_version))) diff --git a/src/app/zkapps_examples/empty_update/dune b/src/app/zkapps_examples/empty_update/dune index 10790066528..11e0c98a278 100644 --- a/src/app/zkapps_examples/empty_update/dune +++ b/src/app/zkapps_examples/empty_update/dune @@ -1,28 +1,28 @@ (library (name zkapps_empty_update) (libraries - ;; opam libraries - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - pickles - pickles.backend - pickles_types - random_oracle - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + pickles + pickles.backend + pickles_types + random_oracle + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version))) + (pps ppx_version))) diff --git a/src/app/zkapps_examples/initialize_state/dune b/src/app/zkapps_examples/initialize_state/dune index 5dc9bcf172f..63590bd026c 100644 --- a/src/app/zkapps_examples/initialize_state/dune +++ b/src/app/zkapps_examples/initialize_state/dune @@ -1,29 +1,29 @@ (library (name zkapps_initialize_state) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - pickles - pickles.backend - pickles_types - random_oracle - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + pickles + pickles.backend + pickles_types + random_oracle + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version))) + (pps ppx_version))) diff --git a/src/app/zkapps_examples/test/actions/dune b/src/app/zkapps_examples/test/actions/dune index baa85f43623..3c02027061c 100644 --- a/src/app/zkapps_examples/test/actions/dune +++ b/src/app/zkapps_examples/test/actions/dune @@ -40,7 +40,8 @@ zkapps_examples yojson mina_wire_types) - (inline_tests (flags -verbose -show-counts)) + (inline_tests + (flags -verbose -show-counts)) (preprocess (pps ppx_version ppx_jane)) (instrumentation diff --git a/src/app/zkapps_examples/test/add_events/dune b/src/app/zkapps_examples/test/add_events/dune index 583a5c247c7..e86b6cad48e 100644 --- a/src/app/zkapps_examples/test/add_events/dune +++ b/src/app/zkapps_examples/test/add_events/dune @@ -1,41 +1,42 @@ (library (name add_events) (libraries - ;; opam libraries - async - async_kernel - async_unix - core - core_kernel - ppx_inline_test.config - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - merkle_list_verifier - mina_base - mina_base.import - mina_ledger - mina_numbers - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - signature_lib - snark_params - transaction_snark - transaction_snark_tests - with_hash - zkapps_add_events - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + core + core_kernel + ppx_inline_test.config + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + merkle_list_verifier + mina_base + mina_base.import + mina_ledger + mina_numbers + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + signature_lib + snark_params + transaction_snark + transaction_snark_tests + with_hash + zkapps_add_events + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/calls/dune b/src/app/zkapps_examples/test/calls/dune index 25706b15533..d1ed0f79578 100644 --- a/src/app/zkapps_examples/test/calls/dune +++ b/src/app/zkapps_examples/test/calls/dune @@ -1,48 +1,49 @@ (library (name calls) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_calls - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_calls + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/empty_update/dune b/src/app/zkapps_examples/test/empty_update/dune index 534cf6ac7b1..c36a8be7eff 100644 --- a/src/app/zkapps_examples/test/empty_update/dune +++ b/src/app/zkapps_examples/test/empty_update/dune @@ -1,51 +1,51 @@ (executable (name empty_update) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_base - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_empty_update - zkapps_examples - bounded_types - ) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_base + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_empty_update + zkapps_examples + bounded_types) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) (rule (alias runtest) diff --git a/src/app/zkapps_examples/test/initialize_state/dune b/src/app/zkapps_examples/test/initialize_state/dune index eac8a441070..e5693dee970 100644 --- a/src/app/zkapps_examples/test/initialize_state/dune +++ b/src/app/zkapps_examples/test/initialize_state/dune @@ -1,48 +1,49 @@ (library (name initialize_state) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_initialize_state - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_initialize_state + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/optional_custom_gates/dune b/src/app/zkapps_examples/test/optional_custom_gates/dune index ecf9f3905ac..573edf08c77 100644 --- a/src/app/zkapps_examples/test/optional_custom_gates/dune +++ b/src/app/zkapps_examples/test/optional_custom_gates/dune @@ -1,43 +1,44 @@ (library - (name zkapp_optional_custom_gates_tests) + (name zkapp_optional_custom_gates_tests) (libraries - ;; opam libraries - ppx_inline_test.config - async - async_kernel - async_unix - core - core_kernel - result - base - base.caml - sexplib0 - integers - ;; local libraries - mina_base.import - transaction_snark - mina_ledger - mina_base - mina_generators - currency - pickles - pickles_types - mina_numbers - zkapp_command_builder - signature_lib - genesis_constants - sgn - transaction_snark_tests - test_util - mina_transaction_logic - zkapps_examples - pickles_optional_custom_gates_circuits - snark_keys_header - snark_params - data_hash_lib - ) - (library_flags -linkall) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane ppx_mina)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + ppx_inline_test.config + async + async_kernel + async_unix + core + core_kernel + result + base + base.caml + sexplib0 + integers + ;; local libraries + mina_base.import + transaction_snark + mina_ledger + mina_base + mina_generators + currency + pickles + pickles_types + mina_numbers + zkapp_command_builder + signature_lib + genesis_constants + sgn + transaction_snark_tests + test_util + mina_transaction_logic + zkapps_examples + pickles_optional_custom_gates_circuits + snark_keys_header + snark_params + data_hash_lib) + (library_flags -linkall) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane ppx_mina)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/tokens/dune b/src/app/zkapps_examples/test/tokens/dune index 48ecefc7bbe..05877f35fdf 100644 --- a/src/app/zkapps_examples/test/tokens/dune +++ b/src/app/zkapps_examples/test/tokens/dune @@ -1,49 +1,50 @@ (library (name tokens) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_tokens - zkapps_examples - bounded_types - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_tokens + zkapps_examples + bounded_types) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/tokens/dune b/src/app/zkapps_examples/tokens/dune index 1e82e848990..d9f359830bd 100644 --- a/src/app/zkapps_examples/tokens/dune +++ b/src/app/zkapps_examples/tokens/dune @@ -1,40 +1,38 @@ (library (name zkapps_tokens) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core_kernel - ;; local libraries - cache_dir - crypto_params - currency - genesis_constants - kimchi_backend - kimchi_backend_common - kimchi_pasta - mina_base - mina_base.import - pasta_bindings - pickles - pickles.backend - pickles_base - pickles_types - random_oracle - random_oracle_input - sgn - signature_lib - snark_params - snarky.backendless - tuple_lib - with_hash - yojson - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + async + async_kernel + async_unix + base + core_kernel + ;; local libraries + cache_dir + crypto_params + currency + genesis_constants + kimchi_backend + kimchi_backend_common + kimchi_pasta + mina_base + mina_base.import + pasta_bindings + pickles + pickles.backend + pickles_base + pickles_types + random_oracle + random_oracle_input + sgn + signature_lib + snark_params + snarky.backendless + tuple_lib + with_hash + yojson + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version - h_list.ppx - ppx_let))) + (pps ppx_version h_list.ppx ppx_let))) From a27cc680553a9036ffbe3ef1e8d9e2d8cb907853 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 23:02:53 +0200 Subject: [PATCH 108/178] Revert "Auxiliary commit to revert individual files from ffa2ff3970ecfe3888aa66678a9ed4b89b5f2760" This reverts commit 4f5ee6136b8f368003bc093cb8906d00ecf44d4f. --- buildkite/src/Command/MinaArtifact.dhall | 2 -- buildkite/src/Constants/DebianVersions.dhall | 22 +++++--------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index b810cf02b8b..5e6968df95f 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -117,7 +117,6 @@ let docker_step let deps = DebianVersions.dependsOnStep - (None Text) debVersion profile buildFlags @@ -309,7 +308,6 @@ let publish_to_debian_repo = , key = "publish-deb-pkg" , depends_on = DebianVersions.dependsOnStep - (Some spec.prefix) spec.debVersion spec.profile spec.buildFlags diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index f3042f5f067..627fb3cb659 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -1,7 +1,3 @@ -let Prelude = ../External/Prelude.dhall - -let Optional/default = Prelude.Optional.default - let Profiles = ./Profiles.dhall let BuildFlags = ./BuildFlags.dhall @@ -31,14 +27,13 @@ let lowerName = debVersion let dependsOnStep = - \(prefix : Optional Text) - -> \(debVersion : DebVersion) + \(debVersion : DebVersion) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) -> let profileSuffix = Profiles.toSuffixUppercase profile - let prefix = Optional/default Text "MinaArtifact" prefix + let prefix = "MinaArtifact" in merge { Bookworm = @@ -78,12 +73,7 @@ let dependsOnStep = let dependsOn = \(debVersion : DebVersion) -> \(profile : Profiles.Type) - -> dependsOnStep - (None Text) - debVersion - profile - BuildFlags.Type.None - "build" + -> dependsOnStep debVersion profile BuildFlags.Type.None "build" let minimalDirtyWhen = [ S.exactly "buildkite/src/Constants/DebianVersions" "dhall" @@ -92,10 +82,8 @@ let minimalDirtyWhen = , S.exactly "buildkite/src/Command/MinaArtifact" "dhall" , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") , S.strictlyStart (S.contains "dockerfiles/stages") - , S.exactly "scripts/debian/build" "sh" - , S.exactly "scripts/debian/builder-helpers" "sh" - , S.exactly "scripts/docker/release" "sh" - , S.exactly "scripts/docker/build" "sh" + , S.exactly "scripts/rebuild-deb" "sh" + , S.exactly "scripts/release-docker" "sh" , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" From dc2b839e07781100d334b80048b8e74c8ab70945 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 23:34:31 +0200 Subject: [PATCH 109/178] Revert "Auxiliary commit to revert individual files from ffa2ff3970ecfe3888aa66678a9ed4b89b5f2760" This reverts commit 6f8c914070aa73e7559675a27e05be0d71892c63. --- dockerfiles/Dockerfile-mina-rosetta | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 51c90c78c0f..53355cc049f 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -22,6 +22,14 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. +# Stretch-only Deps: +# libffi6 +# libprocps6 +# libjemalloc1 +# Buster-only Deps: +# libffi7 +# libprocps7 +# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 From fc1b8f40da48a07888f8007cc4042a6a29c98fe1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 2 Oct 2024 08:37:54 +0200 Subject: [PATCH 110/178] remove mina-rosetta debian from rosetta dockerfile --- dockerfiles/Dockerfile-mina-rosetta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 53355cc049f..8c5cd9b82b6 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -99,7 +99,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres From 91df06cec12b5041b6d2770cbb6c1d381216f666 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 2 Oct 2024 16:58:03 +0200 Subject: [PATCH 111/178] remove duplicated package definition --- src/dune-project | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dune-project b/src/dune-project index 67f3aec652f..3c4d5463c4c 100644 --- a/src/dune-project +++ b/src/dune-project @@ -137,7 +137,6 @@ (package (name participating_state)) (package (name patch_archive_test)) (package (name pasta_bindings)) -(package (name patch_archive_test)) (package (name perf_histograms)) (package (name pickles_base)) (package (name pickles)) From 1754af89322c9a1c75851f5c62f024959fd1ae01 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 4 Oct 2024 14:21:11 +0200 Subject: [PATCH 112/178] revert unnecessary changes --- scripts/debian/publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 49f00c575d5..7d37c8f9da5 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eox pipefail +set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m' @@ -42,7 +42,7 @@ S3_REGION_ARG='--s3-region=us-west-2' #>> Checking for existing lock file #>> Repository is locked by another user: at host dc7eaad3c537 #>> Attempting to obtain a lock -#var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." +#/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ --fail-if-exists \ --lock \ From 6b55612a5872178f179a227e3dd0abd187a9fbde Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 4 Oct 2024 14:34:24 +0200 Subject: [PATCH 113/178] fix dirtyWhen for mina artifacts. allow to run debian build on all scripts/debian/* and scripts/docker/* changes --- buildkite/src/Constants/DebianVersions.dhall | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index ba4bb08311b..af12f14134d 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -76,10 +76,8 @@ let minimalDirtyWhen = , S.exactly "buildkite/src/Command/MinaArtifact" "dhall" , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") , S.strictlyStart (S.contains "dockerfiles/stages") - , S.exactly "scripts/debian/build" "sh" - , S.exactly "scripts/debian/builder-helpers" "sh" - , S.exactly "scripts/docker/release" "sh" - , S.exactly "scripts/docker/build" "sh" + , S.strictlyStart (S.contains "scripts/debian") + , S.strictlyStart (S.contains "scripts/docker") , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" From f3eceaf7ce4c19fe75171d00bd0f2f0b7afc75e1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 4 Oct 2024 15:15:21 +0200 Subject: [PATCH 114/178] fix function call --- scripts/debian/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 7d37c8f9da5..062d863e679 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -79,7 +79,7 @@ do deb="${deb%_*}" for i in {1..10}; do - LAST_VERIFY_STATUS=verify_o1test_repo_has_package $deb + LAST_VERIFY_STATUS=$(verify_o1test_repo_has_package "$deb") if [[ $LAST_VERIFY_STATUS == 0 ]]; then echo "succesfully validated that package is uploaded to deb-s3" From dab5052da3343d732c30014390b25ce982e9da03 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 4 Oct 2024 15:56:50 +0200 Subject: [PATCH 115/178] fix return from function --- scripts/debian/publish.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 062d863e679..7df91317cbe 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eo pipefail +set -eox pipefail CLEAR='\033[0m' RED='\033[0;31m' @@ -59,13 +59,6 @@ for _i in {1..10}; do ( "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done -# Verify integrity of debs on remote repo -function verify_o1test_repo_has_package { - sudo apt-get update - ${DEBS3_SHOW} "${1}" "${DEB_VERSION}" ${ARCH} -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" - return $? -} - for deb in $DEB_NAMES do echo "Adding packages.o1test.net ${DEB_CODENAME} ${DEB_RELEASE}" @@ -79,7 +72,10 @@ do deb="${deb%_*}" for i in {1..10}; do - LAST_VERIFY_STATUS=$(verify_o1test_repo_has_package "$deb") + + sudo apt-get update + ${DEBS3_SHOW} "${1}" "${DEB_VERSION}" ${ARCH} -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" + LAST_VERIFY_STATUS=$? if [[ $LAST_VERIFY_STATUS == 0 ]]; then echo "succesfully validated that package is uploaded to deb-s3" From af6365bff360fdb246c9f3da1f523a52fc32d72c Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 4 Oct 2024 16:41:18 +0200 Subject: [PATCH 116/178] fix argument for non-existing function --- scripts/debian/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 7df91317cbe..4367db5b0f2 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -74,7 +74,7 @@ do for i in {1..10}; do sudo apt-get update - ${DEBS3_SHOW} "${1}" "${DEB_VERSION}" ${ARCH} -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" + ${DEBS3_SHOW} "$deb" "${DEB_VERSION}" "${ARCH}" -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" LAST_VERIFY_STATUS=$? if [[ $LAST_VERIFY_STATUS == 0 ]]; then From 2eea1863d52b32d3d21162777278acf1cc2b93f0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 7 Oct 2024 20:39:16 +0200 Subject: [PATCH 117/178] remove fuzzy zkapp test from PR run --- .../src/Jobs/Test/FuzzyZkappTestQuick.dhall | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 buildkite/src/Jobs/Test/FuzzyZkappTestQuick.dhall diff --git a/buildkite/src/Jobs/Test/FuzzyZkappTestQuick.dhall b/buildkite/src/Jobs/Test/FuzzyZkappTestQuick.dhall deleted file mode 100644 index 95eb6c3bb47..00000000000 --- a/buildkite/src/Jobs/Test/FuzzyZkappTestQuick.dhall +++ /dev/null @@ -1,22 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineMode = ../../Pipeline/Mode.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let Command = ../../Command/FuzzyZkappTest.dhall - -in Pipeline.build - ( Command.pipeline - Command.Spec::{ - , job_path = "Test" - , job_name = "FuzzyZkappTestQuick" - , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - , mode = PipelineMode.Type.PullRequest - , additional_dirty_when = - [ S.exactly "buildkite/src/Jobs/Test/FuzzyZkappTestQuick" "dhall" ] - , timeout = 300 - } - ) From 82bcfbc7f6d7ed3e937af92e3cdb52b09d1060c3 Mon Sep 17 00:00:00 2001 From: SanabriaRusso Date: Mon, 14 Oct 2024 11:40:51 +0200 Subject: [PATCH 118/178] Imposing hard timeout of Rosetta integration test --- buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index ec77336a12c..ab0731cd45e 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -65,6 +65,7 @@ in Pipeline.build , label = "Rosetta integration tests Bullseye Long" , key = "rosetta-integration-tests-bullseye-long" , soft_fail = Some (B/SoftFail.Boolean True) + , timeout_in_minutes = Some +90 , target = Size.Small , depends_on = Dockers.dependsOn From 07aac16d2f6828db27b391727982757b0bfd8602 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 16 Oct 2024 00:05:26 +0200 Subject: [PATCH 119/178] allow deb-s3 command to fail to enable retry mechanism --- scripts/debian/publish.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 46c75f3c63f..80fa3c78a3e 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -71,9 +71,11 @@ do for i in {1..10}; do + set +e ${DEBS3_SHOW} "$deb" "${DEB_VERSION}" "${ARCH}" -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" LAST_VERIFY_STATUS=$? - + set -eo pipefail + if [[ $LAST_VERIFY_STATUS == 0 ]]; then echo "succesfully validated that package is uploaded to deb-s3" break From 0d8eb666ebb72c23812411c5efbb6dbdcd6cdcf5 Mon Sep 17 00:00:00 2001 From: SanabriaRusso Date: Wed, 16 Oct 2024 10:23:38 +0200 Subject: [PATCH 120/178] Unstucking Buildkite checks From 01b65d90285a3694d7c1f31c95a8f7c07edbe22b Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 16 Oct 2024 20:14:36 +0200 Subject: [PATCH 121/178] never remove first block as well --- src/test/archive/patch_archive_test/patch_archive_test.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/archive/patch_archive_test/patch_archive_test.ml b/src/test/archive/patch_archive_test/patch_archive_test.ml index 5ba17608041..cdeaa635ddf 100644 --- a/src/test/archive/patch_archive_test/patch_archive_test.ml +++ b/src/test/archive/patch_archive_test/patch_archive_test.ml @@ -77,10 +77,10 @@ let main ~db_uri ~network_data_folder () = let n = List.init missing_blocks_count ~f:(fun _ -> - (* never remove last block as missing-block-guardian can have issues when patching it + (* never remove last and first block as missing-block-guardian can have issues when patching it as it patching only gaps *) - Random.int (List.length extensional_files - 1) ) + Random.int (List.length extensional_files - 2) + 1 ) in let unpatched_extensional_files = From 784206a972b80e50b955fb9d84185d392cc766ad Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 17 Oct 2024 08:12:57 +0200 Subject: [PATCH 122/178] fix Bench/Base.dhall --- buildkite/src/Command/Bench/Base.dhall | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 3d2431943b6..48427e6738d 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -10,6 +10,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall +let Network = ../../Constants/Network.dhall + let Profiles = ../../Constants/Profiles.dhall let Command = ../../Command/Base.dhall @@ -42,6 +44,7 @@ let Spec = , dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard , additionalDirtyWhen = [] : List SelectFiles.Type , yellowThreshold = 0.1 From 4d38f5728c1c5b8da048c52e2c3bc931d5ce02b0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 18 Oct 2024 22:42:53 +0200 Subject: [PATCH 123/178] fix ZkappMetrics.dhall --- buildkite/src/Jobs/Test/ZkappMetrics.dhall | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildkite/src/Jobs/Test/ZkappMetrics.dhall b/buildkite/src/Jobs/Test/ZkappMetrics.dhall index 90117a87a34..e03be0b2b0e 100644 --- a/buildkite/src/Jobs/Test/ZkappMetrics.dhall +++ b/buildkite/src/Jobs/Test/ZkappMetrics.dhall @@ -12,6 +12,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,6 +23,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build From 048a401c48f3a46f25252befcc539c7ea3fee151 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 18 Oct 2024 22:59:04 +0200 Subject: [PATCH 124/178] use devnet instead of berkeley --- buildkite/scripts/bench/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/bench/install.sh b/buildkite/scripts/bench/install.sh index ea06e6ee02b..b9e0292dae0 100644 --- a/buildkite/scripts/bench/install.sh +++ b/buildkite/scripts/bench/install.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh From 2cdc48a505ea83250fd7df89d9fe9356e697d7a2 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Mon, 28 Oct 2024 16:42:42 +0000 Subject: [PATCH 125/178] Update old, invalid dirtyWhen --- buildkite/src/Constants/DebianVersions.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index af12f14134d..4ca7d17a6c3 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -92,7 +92,7 @@ let bullseyeDirtyWhen = [ S.strictlyStart (S.contains "src") , S.strictlyStart (S.contains "automation") , S.strictly (S.contains "Makefile") - , S.exactly "buildkite/scripts/connect-to-berkeley" "sh" + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" , S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" From fb10d7bf2416685450f20790f824fc4777ffdb9e Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 29 Oct 2024 21:08:20 +0100 Subject: [PATCH 126/178] fix dhall --- buildkite/src/Jobs/Lint/Merge.dhall | 8 -------- 1 file changed, 8 deletions(-) diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index c421b5498a7..a2bb91b05a8 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -73,14 +73,6 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } - , Command.build - Command.Config::{ - , commands = - , target = Size.Multi - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } - } , Command.build Command.Config::{ , commands = From 864b6ec37d9f721562a73970fdd19d67e5778aef Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 29 Oct 2024 22:11:40 +0100 Subject: [PATCH 127/178] fix dhall linters --- buildkite/src/Constants/DebianVersions.dhall | 2 +- buildkite/src/Jobs/Test/PatchArchiveTest.dhall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index af12f14134d..4ca7d17a6c3 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -92,7 +92,7 @@ let bullseyeDirtyWhen = [ S.strictlyStart (S.contains "src") , S.strictlyStart (S.contains "automation") , S.strictly (S.contains "Makefile") - , S.exactly "buildkite/scripts/connect-to-berkeley" "sh" + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" , S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index 8bbf02c72ef..67cbfc24f21 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -28,7 +28,7 @@ in Pipeline.build , spec = JobSpec::{ , dirtyWhen = [ S.strictlyStart (S.contains "src") - , S.exactly "scripts/path-archive-test" "sh" + , S.exactly "scripts/patch-archive-test" "sh" , S.exactly "buildkite/src/Jobs/Test/PatchArchiveTest" "dhall" , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall" ] From 89a31fa950e80fc8505eb0baea2b5258d1abe93f Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 5 Nov 2024 23:18:04 +0100 Subject: [PATCH 128/178] fix dirty when --- buildkite/src/Constants/DebianVersions.dhall | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 087d6806af3..694996a1ca8 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -83,7 +83,6 @@ let minimalDirtyWhen = , S.exactly "buildkite/scripts/check-compatibility" "sh" , S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall" , S.exactly "buildkite/scripts/bench/snark_transaction_profiler" "sh" - , S.exactly "scripts/snark_transaction_profiler" "py" , S.exactly "buildkite/scripts/version-linter" "sh" , S.exactly "scripts/version-linter" "py" ] From 7db773bc1736900b77b650e83f7edb45f85fafce Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 11 Nov 2024 10:14:42 +0100 Subject: [PATCH 129/178] update toolchain --- buildkite/src/Constants/ContainerImages.dhall | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index b17bd8ccaff..463af9f56a4 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -4,11 +4,11 @@ -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI { toolchainBase = "codaprotocol/ci-toolchain-base:v3" , minaToolchainBullseye = - "gcr.io/o1labs-192920/mina-toolchain@sha256:1e4b429fdf6a06e88abe9b4c9d54a80a3e818011a94806d070433a95d6af1229" + "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d" , minaToolchainBookworm = - "gcr.io/o1labs-192920/mina-toolchain@sha256:1e4b429fdf6a06e88abe9b4c9d54a80a3e818011a94806d070433a95d6af1229" + "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d" , minaToolchain = - "gcr.io/o1labs-192920/mina-toolchain@sha256:1e4b429fdf6a06e88abe9b4c9d54a80a3e818011a94806d070433a95d6af1229" + "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d" , elixirToolchain = "elixir:1.10-alpine" , nodeToolchain = "node:14.13.1-stretch-slim" , ubuntu2004 = "ubuntu:20.04" From 077b14283998e7c8031d8e48c2ceb3851121ae74 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 17:14:23 +0200 Subject: [PATCH 130/178] publish debians only on stable pipelines --- buildkite/src/Command/MinaArtifact.dhall | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index ddc915278da..322bb180988 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -105,6 +105,7 @@ let build_artifacts let publish_to_debian_repo = \(spec : MinaBuildSpec.Type) + -> \(dependsOn : List Command.TaggedKey.Type) -> Command.build Command.Config::{ , commands = @@ -123,13 +124,7 @@ let publish_to_debian_repo = spec.debVersion} ${Profiles.toSuffixUppercase spec.profile}" , key = "publish-deb-pkg" - , depends_on = - DebianVersions.dependsOnStep - (Some spec.prefix) - spec.debVersion - spec.profile - spec.buildFlags - "build" + , depends_on = dependsOn , target = Size.Small } @@ -318,18 +313,25 @@ let onlyDebianPipeline = \(spec : MinaBuildSpec.Type) -> pipelineBuilder spec - [ build_artifacts spec, publish_to_debian_repo spec ] + [ build_artifacts spec + , publish_to_debian_repo + spec + ( DebianVersions.dependsOnStep + (Some spec.prefix) + spec.debVersion + spec.profile + spec.buildFlags + "build" + ) + ] let pipeline : MinaBuildSpec.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) - -> pipelineBuilder - spec - ( [ build_artifacts spec, publish_to_debian_repo spec ] - # docker_commands spec - ) + -> pipelineBuilder spec ([ build_artifacts spec ] # docker_commands spec) in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline + , publishToDebian = publish_to_debian_repo , MinaBuildSpec = MinaBuildSpec } From 8473b45bc6dd9aa59aaced8d767b86238052a87c Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 17:14:48 +0200 Subject: [PATCH 131/178] publish debians job --- .../src/Jobs/TearDown/PublishDebians.dhall | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 buildkite/src/Jobs/TearDown/PublishDebians.dhall diff --git a/buildkite/src/Jobs/TearDown/PublishDebians.dhall b/buildkite/src/Jobs/TearDown/PublishDebians.dhall new file mode 100644 index 00000000000..c2465a94876 --- /dev/null +++ b/buildkite/src/Jobs/TearDown/PublishDebians.dhall @@ -0,0 +1,45 @@ +let ArtifactPipelines = ../../Command/MinaArtifact.dhall + +let Command = ../../Command/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let DebianChannel = ../../Constants/DebianChannel.dhall + +let DebianVersions = ../../Constants/DebianVersions.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let dependsOn = [] : List Command.TaggedKey.Type + +let specBuilder = + \(debVersion : DebianVersions.DebVersion) + -> ArtifactPipelines.MinaBuildSpec::{ + , channel = DebianChannel.Type.Unstable + , prefix = "DebianPublish" + , debVersion = debVersion + } + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + DebianVersions.dirtyWhen DebianVersions.DebVersion.Bullseye + , path = "TearDown" + , name = "PublishDebians" + , tags = [ PipelineTag.Type.TearDown ] + , mode = PipelineMode.Type.Stable + } + , steps = + [ ArtifactPipelines.publishToDebian + (specBuilder DebianVersions.DebVersion.Bullseye) + dependsOn + , ArtifactPipelines.publishToDebian + (specBuilder DebianVersions.DebVersion.Focal) + dependsOn + ] + } From 2ee2050cf66ed12a24951eb178804675a7127381 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 17:36:48 +0200 Subject: [PATCH 132/178] hardcode build id for testing --- buildkite/scripts/download-artifact-from-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/download-artifact-from-cache.sh b/buildkite/scripts/download-artifact-from-cache.sh index 8257094f4b2..c04c18d53ea 100755 --- a/buildkite/scripts/download-artifact-from-cache.sh +++ b/buildkite/scripts/download-artifact-from-cache.sh @@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then fi DOWNLOAD_BIN=gsutil -PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID} +PREFIX=gs://buildkite_k8s/coda/shared/0192b1f4-720f-48d9-b8a4-5044a7409187 FILE="$1" REMOTE_LOCATION="$2" OPTS=${3:-""} From d709573ffa7c354c62eb9104c5762246576fd6de Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 19:11:44 +0200 Subject: [PATCH 133/178] add debian codename to publish job name --- buildkite/src/Command/MinaArtifact.dhall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 322bb180988..0f7e43f1256 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -123,7 +123,8 @@ let publish_to_debian_repo = "Publish Mina for ${DebianVersions.capitalName spec.debVersion} ${Profiles.toSuffixUppercase spec.profile}" - , key = "publish-deb-pkg" + , key = + "publish-${DebianVersions.lowerName spec.debVersion}-deb-pkg" , depends_on = dependsOn , target = Size.Small } From 0753054f59fda5b324c7cb697155f40ee3e4da32 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 19:47:28 +0200 Subject: [PATCH 134/178] run publish debians sequentially --- .../src/Jobs/TearDown/PublishDebians.dhall | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/buildkite/src/Jobs/TearDown/PublishDebians.dhall b/buildkite/src/Jobs/TearDown/PublishDebians.dhall index c2465a94876..4c6cb44f6bf 100644 --- a/buildkite/src/Jobs/TearDown/PublishDebians.dhall +++ b/buildkite/src/Jobs/TearDown/PublishDebians.dhall @@ -14,13 +14,20 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let JobSpec = ../../Pipeline/JobSpec.dhall -let dependsOn = [] : List Command.TaggedKey.Type +let name = "PublishDebians" + +let dependsOnCodename = + \(codename : DebianVersions.DebVersion) + -> [ { name = name + , key = "publish-${DebianVersions.lowerName codename}-deb-pkg" + } + ] let specBuilder = \(debVersion : DebianVersions.DebVersion) -> ArtifactPipelines.MinaBuildSpec::{ , channel = DebianChannel.Type.Unstable - , prefix = "DebianPublish" + , prefix = name , debVersion = debVersion } @@ -30,16 +37,16 @@ in Pipeline.build , dirtyWhen = DebianVersions.dirtyWhen DebianVersions.DebVersion.Bullseye , path = "TearDown" - , name = "PublishDebians" + , name = name , tags = [ PipelineTag.Type.TearDown ] , mode = PipelineMode.Type.Stable } , steps = [ ArtifactPipelines.publishToDebian (specBuilder DebianVersions.DebVersion.Bullseye) - dependsOn + ([] : List Command.TaggedKey.Type) , ArtifactPipelines.publishToDebian (specBuilder DebianVersions.DebVersion.Focal) - dependsOn + (dependsOnCodename DebianVersions.DebVersion.Bullseye) ] } From f0b8302a80f46ae70f24252e633908fb34b2adc9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 19:48:11 +0200 Subject: [PATCH 135/178] allow duplicates --- scripts/debian/publish.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 624a82d4f33..bc324e6a86c 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -44,7 +44,6 @@ S3_REGION_ARG="--s3-region=us-west-2" #>> Attempting to obtain a lock #/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ - --fail-if-exists \ --lock \ --preserve-versions \ --cache-control=max-age=120" From ee8873c4abd8518d814959c56f94380a5506b13c Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 23 Oct 2024 15:49:33 +0200 Subject: [PATCH 136/178] Revert "hardcode build id for testing" This reverts commit 8fd4947605234b7d6959c13feb01794341bb0997. --- buildkite/scripts/download-artifact-from-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/download-artifact-from-cache.sh b/buildkite/scripts/download-artifact-from-cache.sh index c04c18d53ea..8257094f4b2 100755 --- a/buildkite/scripts/download-artifact-from-cache.sh +++ b/buildkite/scripts/download-artifact-from-cache.sh @@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then fi DOWNLOAD_BIN=gsutil -PREFIX=gs://buildkite_k8s/coda/shared/0192b1f4-720f-48d9-b8a4-5044a7409187 +PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID} FILE="$1" REMOTE_LOCATION="$2" OPTS=${3:-""} From 59fb64e1aa158d920697867ebb4289699e7f8b58 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 14 Nov 2024 18:41:31 +0100 Subject: [PATCH 137/178] Revert "allow duplicates" This reverts commit 13cf26594c531c1f879fd000c4b2305eb21da469. --- scripts/debian/publish.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index bc324e6a86c..624a82d4f33 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -44,6 +44,7 @@ S3_REGION_ARG="--s3-region=us-west-2" #>> Attempting to obtain a lock #/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ + --fail-if-exists \ --lock \ --preserve-versions \ --cache-control=max-age=120" From 2807a3c76d861bbf8e67c5eee86ddfd7a663e105 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 18 Nov 2024 12:09:09 +0100 Subject: [PATCH 138/178] adjust publish_to_debian_repo function --- buildkite/src/Command/MinaArtifact.dhall | 73 +++++++++++++----------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index c0ffc264433..37b59b09b3b 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -123,6 +123,31 @@ let build_artifacts ] } +let publish_to_debian_repo = + \(spec : MinaBuildSpec.Type) + -> \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" + ] + "./buildkite/scripts/debian/publish.sh" + , label = + "Publish Mina for ${DebianVersions.capitalName + spec.debVersion} ${Profiles.toSuffixUppercase + spec.profile}" + , key = "publish-deb-pkg" + , depends_on = dependsOn + , target = Size.Small + } + let docker_step : Artifacts.Type -> MinaBuildSpec.Type -> List DockerImage.ReleaseSpec.Type = \(artifact : Artifacts.Type) @@ -258,34 +283,6 @@ let docker_commands ) flattened_docker_steps -let publish_to_debian_repo = - \(spec : MinaBuildSpec.Type) - -> Command.build - Command.Config::{ - , commands = - Toolchain.select - spec.toolchainSelectMode - spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] - "./buildkite/scripts/debian/publish.sh" - , label = "Debian: Publish ${labelSuffix spec}" - , key = "publish-deb-pkg" - , depends_on = - DebianVersions.dependsOnStep - (Some spec.prefix) - spec.debVersion - spec.network - spec.profile - spec.buildFlags - "build" - , target = Size.Small - } - let pipelineBuilder : MinaBuildSpec.Type -> List Command.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) @@ -306,19 +303,27 @@ let onlyDebianPipeline = \(spec : MinaBuildSpec.Type) -> pipelineBuilder spec - [ build_artifacts spec, publish_to_debian_repo spec ] + [ build_artifacts spec + , publish_to_debian_repo + spec + ( DebianVersions.dependsOnStep + (Some spec.prefix) + spec.debVersion + spec.network + spec.profile + spec.buildFlags + "build" + ) + ] let pipeline : MinaBuildSpec.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) - -> pipelineBuilder - spec - ( [ build_artifacts spec, publish_to_debian_repo spec ] - # docker_commands spec - ) + -> pipelineBuilder spec ([ build_artifacts spec ] # docker_commands spec) in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline + , publishToDebian = publish_to_debian_repo , MinaBuildSpec = MinaBuildSpec , labelSuffix = labelSuffix , keySuffix = keySuffix From 89798350444b6e5a4115f4c257cc52b1363c0002 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 17:14:48 +0200 Subject: [PATCH 139/178] publish debians job --- .../src/Jobs/TearDown/PublishDebians.dhall | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 buildkite/src/Jobs/TearDown/PublishDebians.dhall diff --git a/buildkite/src/Jobs/TearDown/PublishDebians.dhall b/buildkite/src/Jobs/TearDown/PublishDebians.dhall new file mode 100644 index 00000000000..c2465a94876 --- /dev/null +++ b/buildkite/src/Jobs/TearDown/PublishDebians.dhall @@ -0,0 +1,45 @@ +let ArtifactPipelines = ../../Command/MinaArtifact.dhall + +let Command = ../../Command/Base.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let DebianChannel = ../../Constants/DebianChannel.dhall + +let DebianVersions = ../../Constants/DebianVersions.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let dependsOn = [] : List Command.TaggedKey.Type + +let specBuilder = + \(debVersion : DebianVersions.DebVersion) + -> ArtifactPipelines.MinaBuildSpec::{ + , channel = DebianChannel.Type.Unstable + , prefix = "DebianPublish" + , debVersion = debVersion + } + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + DebianVersions.dirtyWhen DebianVersions.DebVersion.Bullseye + , path = "TearDown" + , name = "PublishDebians" + , tags = [ PipelineTag.Type.TearDown ] + , mode = PipelineMode.Type.Stable + } + , steps = + [ ArtifactPipelines.publishToDebian + (specBuilder DebianVersions.DebVersion.Bullseye) + dependsOn + , ArtifactPipelines.publishToDebian + (specBuilder DebianVersions.DebVersion.Focal) + dependsOn + ] + } From e9c21d5208177ab502f7be648c3adf7e95a196e9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 17:36:48 +0200 Subject: [PATCH 140/178] hardcode build id for testing --- buildkite/scripts/download-artifact-from-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/download-artifact-from-cache.sh b/buildkite/scripts/download-artifact-from-cache.sh index 8257094f4b2..c04c18d53ea 100755 --- a/buildkite/scripts/download-artifact-from-cache.sh +++ b/buildkite/scripts/download-artifact-from-cache.sh @@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then fi DOWNLOAD_BIN=gsutil -PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID} +PREFIX=gs://buildkite_k8s/coda/shared/0192b1f4-720f-48d9-b8a4-5044a7409187 FILE="$1" REMOTE_LOCATION="$2" OPTS=${3:-""} From 76556b5611e1fa3d18bcc21f570dbe7767b4ac9d Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 19:11:44 +0200 Subject: [PATCH 141/178] add debian codename to publish job name --- buildkite/src/Command/MinaArtifact.dhall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 37b59b09b3b..e98fdca16a8 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -143,7 +143,8 @@ let publish_to_debian_repo = "Publish Mina for ${DebianVersions.capitalName spec.debVersion} ${Profiles.toSuffixUppercase spec.profile}" - , key = "publish-deb-pkg" + , key = + "publish-${DebianVersions.lowerName spec.debVersion}-deb-pkg" , depends_on = dependsOn , target = Size.Small } From 4ce29667c424c8d1a2487ed2f1abb2c4449f50b7 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 19:47:28 +0200 Subject: [PATCH 142/178] run publish debians sequentially --- .../src/Jobs/TearDown/PublishDebians.dhall | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/buildkite/src/Jobs/TearDown/PublishDebians.dhall b/buildkite/src/Jobs/TearDown/PublishDebians.dhall index c2465a94876..4c6cb44f6bf 100644 --- a/buildkite/src/Jobs/TearDown/PublishDebians.dhall +++ b/buildkite/src/Jobs/TearDown/PublishDebians.dhall @@ -14,13 +14,20 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let JobSpec = ../../Pipeline/JobSpec.dhall -let dependsOn = [] : List Command.TaggedKey.Type +let name = "PublishDebians" + +let dependsOnCodename = + \(codename : DebianVersions.DebVersion) + -> [ { name = name + , key = "publish-${DebianVersions.lowerName codename}-deb-pkg" + } + ] let specBuilder = \(debVersion : DebianVersions.DebVersion) -> ArtifactPipelines.MinaBuildSpec::{ , channel = DebianChannel.Type.Unstable - , prefix = "DebianPublish" + , prefix = name , debVersion = debVersion } @@ -30,16 +37,16 @@ in Pipeline.build , dirtyWhen = DebianVersions.dirtyWhen DebianVersions.DebVersion.Bullseye , path = "TearDown" - , name = "PublishDebians" + , name = name , tags = [ PipelineTag.Type.TearDown ] , mode = PipelineMode.Type.Stable } , steps = [ ArtifactPipelines.publishToDebian (specBuilder DebianVersions.DebVersion.Bullseye) - dependsOn + ([] : List Command.TaggedKey.Type) , ArtifactPipelines.publishToDebian (specBuilder DebianVersions.DebVersion.Focal) - dependsOn + (dependsOnCodename DebianVersions.DebVersion.Bullseye) ] } From f9da5d25a353afd1a194a81c8a2968a67158c76b Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 22 Oct 2024 19:48:11 +0200 Subject: [PATCH 143/178] allow duplicates --- scripts/debian/publish.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 624a82d4f33..bc324e6a86c 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -44,7 +44,6 @@ S3_REGION_ARG="--s3-region=us-west-2" #>> Attempting to obtain a lock #/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ - --fail-if-exists \ --lock \ --preserve-versions \ --cache-control=max-age=120" From f3501b89e65d5917f108c38e34309cbd131354a4 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 23 Oct 2024 15:49:33 +0200 Subject: [PATCH 144/178] Revert "hardcode build id for testing" This reverts commit 8fd4947605234b7d6959c13feb01794341bb0997. --- buildkite/scripts/download-artifact-from-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/download-artifact-from-cache.sh b/buildkite/scripts/download-artifact-from-cache.sh index c04c18d53ea..8257094f4b2 100755 --- a/buildkite/scripts/download-artifact-from-cache.sh +++ b/buildkite/scripts/download-artifact-from-cache.sh @@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then fi DOWNLOAD_BIN=gsutil -PREFIX=gs://buildkite_k8s/coda/shared/0192b1f4-720f-48d9-b8a4-5044a7409187 +PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID} FILE="$1" REMOTE_LOCATION="$2" OPTS=${3:-""} From b4e7d556fb4da7714fe8b0d3b0b0bc02bfae4878 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 14 Nov 2024 18:41:31 +0100 Subject: [PATCH 145/178] Revert "allow duplicates" This reverts commit 13cf26594c531c1f879fd000c4b2305eb21da469. --- scripts/debian/publish.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index bc324e6a86c..624a82d4f33 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -44,6 +44,7 @@ S3_REGION_ARG="--s3-region=us-west-2" #>> Attempting to obtain a lock #/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ + --fail-if-exists \ --lock \ --preserve-versions \ --cache-control=max-age=120" From e2d2a65cf3ffd99bee34c9b3b25ce958e2362570 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 13 Nov 2024 22:24:04 +0100 Subject: [PATCH 146/178] WIP --- buildkite/scripts/debian/install.sh | 6 +- buildkite/scripts/tests/ledger_test_apply.sh | 20 +++++ .../src/Jobs/Test/LedgerTestApplyTx.dhall | 77 +++++++++++++++++++ scripts/tests/ledger_test_apply.sh | 45 +++++++++++ 4 files changed, 145 insertions(+), 3 deletions(-) create mode 100755 buildkite/scripts/tests/ledger_test_apply.sh create mode 100644 buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall create mode 100755 scripts/tests/ledger_test_apply.sh diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index bd9a863f8ef..b0e317c113c 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -8,7 +8,7 @@ fi if [ -z $MINA_DEB_CODENAME ]; then echo "MINA_DEB_CODENAME env var is not defined" - exit -1 + exit 1 fi DEBS=$1 @@ -29,12 +29,12 @@ source ./buildkite/scripts/export-git-env-vars.sh # Download required debians from bucket locally if [ -z "$DEBS" ]; then echo "DEBS env var is empty. It should contains comma delimitered names of debians to install" - exit -1 + exit 1 else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do case $i in - mina-devnet|mina-mainnet|mina-devnet-lightnet) + mina-devnet*|mina-mainnet) # Downaload mina-logproc too source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER ;; diff --git a/buildkite/scripts/tests/ledger_test_apply.sh b/buildkite/scripts/tests/ledger_test_apply.sh new file mode 100755 index 00000000000..4e5e9ec0d36 --- /dev/null +++ b/buildkite/scripts/tests/ledger_test_apply.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eo pipefail + +# Don't prompt for answers during apt-get install +export DEBIAN_FRONTEND=noninteractive + +sudo apt-get update +sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 python3-pip wget + +git config --global --add safe.directory /workdir + +source buildkite/scripts/export-git-env-vars.sh + +source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-instrumented" 1 + +./scripts/tests/ledger_test_apply.sh \ + --mina-app mina \ + --generate-ledger-app mina-generate-random-ledger \ + --runtime-ledger-app mina-create-genesis \ No newline at end of file diff --git a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall new file mode 100644 index 00000000000..4704292f7be --- /dev/null +++ b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall @@ -0,0 +1,77 @@ +let PipelineTag = ../../Pipeline/Tag.dhall + +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall + +let RunInToolchain = ../../Command/RunInToolchain.dhall + +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let BuildFlags = ../../Constants/BuildFlags.dhall + +let Docker = ../../Command/Docker/Type.dhall + +let Size = ../../Command/Size.dhall + +let dependsOn = + DebianVersions.dependsOnStep + (None Text) + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + BuildFlags.Type.Instrumented + "build" + # DebianVersions.dependsOnStep + (None Text) + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + BuildFlags.Type.None + "build" + +let buildTestCmd + : Size -> Command.Type + = \(cmd_target : Size) + -> let key = "ledger-test-apply" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/tests/ledger_test_apply.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" + , label = "Test: Ledger apply txs" + , key = key + , target = cmd_target + , docker = None Docker.Type + , depends_on = dependsOn + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.strictly (S.contains "Makefile") + , S.exactly "buildkite/src/Jobs/Test/LedgerTestApplyTx" "dhall" + , S.exactly "buildkite/scripts/tests/ledger_test_apply" "sh" + , S.exactly "scripts/tests/ledger_test_apply" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "LedgerTestApplyTx" + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + ] + } + , steps = [ buildTestCmd Size.XLarge ] + } diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh new file mode 100755 index 00000000000..d7e948dd88e --- /dev/null +++ b/scripts/tests/ledger_test_apply.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# test mina ledger test-apply +set -x + +export MINA_PRIVKEY_PASS='naughty blue worm' + +MINA_APP=_build/default/src/app/cli/src/mina.exe +RUNTIME_LEDGER_APP=_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe + +TEMP_FOLDER=$(mktemp -d) +ACCOUNTS_FILE=$TEMP_FOLDER/accounts.json +GENESIS_LEDGER=$TEMP_FOLDER/genesis_ledger.config +SENDER_FOLDER=$TEMP_FOLDER/sender + +while [[ "$#" -gt 0 ]]; do case $1 in + -m|--mina-app) MINA_APP="$2"; shift;; + -r|--runtime-ledger-app) RUNTIME_LEDGER_APP="$2"; shift;; + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + + +echo "Exporting ledger to $TEMP_FOLDER" +$MINA_APP ledger test generate-accounts -n 2 --min-balance 100000 --max-balance 1000000 > $ACCOUNTS_FILE + +jq '{ ledger: { name:"random", accounts:( $inputs | .[] ) } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER + +echo "creating runtime ledger in $TEMP_FOLDER" +$RUNTIME_LEDGER_APP --config-file $GENESIS_LEDGER --genesis-dir $TEMP_FOLDER/genesis --hash-output-file $TEMP_FOLDER/genesis/hash.out --ignore-missing + +mkdir $SENDER_FOLDER +chmod 700 $SENDER_FOLDER + +CODA_PRIVKEY=$(cat $ACCOUNTS_FILE | jq -r .[0].sk) + +# Silently passing MINA_PRIVKEY_PASS & CODA_PRIVKEY +mina advanced wrap-key --privkey-path $SENDER_FOLDER + +mkdir $TEMP_FOLDER/genesis/ledger + +tar -zxf $TEMP_FOLDER/genesis/genesis_ledger_*.tar.gz -C $TEMP_FOLDER/genesis/ledger + +echo "running test:" +$MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger --privkey-path $SENDER --num-txs 200 + From db3e56aece7783070eb616ecc11bcd8ded7d0042 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 13 Nov 2024 22:37:23 +0100 Subject: [PATCH 147/178] fix test --- scripts/tests/ledger_test_apply.sh | 13 +++++-------- .../runtime_genesis_ledger.ml | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index d7e948dd88e..5fe84e63229 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -11,7 +11,7 @@ RUNTIME_LEDGER_APP=_build/default/src/app/runtime_genesis_ledger/runtime_genesis TEMP_FOLDER=$(mktemp -d) ACCOUNTS_FILE=$TEMP_FOLDER/accounts.json GENESIS_LEDGER=$TEMP_FOLDER/genesis_ledger.config -SENDER_FOLDER=$TEMP_FOLDER/sender +SENDER=$TEMP_FOLDER/sender while [[ "$#" -gt 0 ]]; do case $1 in -m|--mina-app) MINA_APP="$2"; shift;; @@ -23,23 +23,20 @@ esac; shift; done echo "Exporting ledger to $TEMP_FOLDER" $MINA_APP ledger test generate-accounts -n 2 --min-balance 100000 --max-balance 1000000 > $ACCOUNTS_FILE -jq '{ ledger: { name:"random", accounts:( $inputs | .[] ) } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER +jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER echo "creating runtime ledger in $TEMP_FOLDER" $RUNTIME_LEDGER_APP --config-file $GENESIS_LEDGER --genesis-dir $TEMP_FOLDER/genesis --hash-output-file $TEMP_FOLDER/genesis/hash.out --ignore-missing -mkdir $SENDER_FOLDER -chmod 700 $SENDER_FOLDER - -CODA_PRIVKEY=$(cat $ACCOUNTS_FILE | jq -r .[0].sk) # Silently passing MINA_PRIVKEY_PASS & CODA_PRIVKEY -mina advanced wrap-key --privkey-path $SENDER_FOLDER +CODA_PRIVKEY=$(cat $ACCOUNTS_FILE | jq -r .[0].sk) MINA_PRIVKEY_PASS=$MINA_PRIVKEY_PASS mina advanced wrap-key --privkey-path $SENDER +chmod 700 $SENDER mkdir $TEMP_FOLDER/genesis/ledger tar -zxf $TEMP_FOLDER/genesis/genesis_ledger_*.tar.gz -C $TEMP_FOLDER/genesis/ledger echo "running test:" -$MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger --privkey-path $SENDER --num-txs 200 +$MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger --privkey-path $SENDER --num-txs 200 diff --git a/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml b/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml index 298f50ef14e..0b2a42f4858 100644 --- a/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml +++ b/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml @@ -97,7 +97,7 @@ let load_config_exn ~logger config_file = in if Option.( - is_some config.daemon || is_some config.genesis + is_some config.daemon || Option.value_map ~default:false ~f:is_dirty_proof config.proof) then failwith "Runtime config has unexpected fields" ; let ledger = Option.value_exn ~message:"No ledger provided" config.ledger in From fda500a545cf16635bb7d5e5cdd8c1587b741d94 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 14 Nov 2024 08:28:43 +0100 Subject: [PATCH 148/178] fix script invocation --- buildkite/scripts/tests/ledger_test_apply.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildkite/scripts/tests/ledger_test_apply.sh b/buildkite/scripts/tests/ledger_test_apply.sh index 4e5e9ec0d36..640bbfa014d 100755 --- a/buildkite/scripts/tests/ledger_test_apply.sh +++ b/buildkite/scripts/tests/ledger_test_apply.sh @@ -12,9 +12,8 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-instrumented" 1 +source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1 ./scripts/tests/ledger_test_apply.sh \ --mina-app mina \ - --generate-ledger-app mina-generate-random-ledger \ --runtime-ledger-app mina-create-genesis \ No newline at end of file From 3c4da738d841d3a522b53aee4f9ae2552e363b0a Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Fri, 15 Nov 2024 20:31:08 +0100 Subject: [PATCH 149/178] Update scripts/tests/ledger_test_apply.sh Co-authored-by: George Agapov --- scripts/tests/ledger_test_apply.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index 5fe84e63229..98cfd3cb86a 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -38,5 +38,5 @@ mkdir $TEMP_FOLDER/genesis/ledger tar -zxf $TEMP_FOLDER/genesis/genesis_ledger_*.tar.gz -C $TEMP_FOLDER/genesis/ledger echo "running test:" -$MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger --privkey-path $SENDER --num-txs 200 +time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger --privkey-path $SENDER --num-txs 200 From 09b5ee4ed2e6a624519a30662411d83e1a0dfd4a Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 15 Nov 2024 20:51:59 +0100 Subject: [PATCH 150/178] reverts exit code fix --- buildkite/scripts/debian/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index b0e317c113c..17d5ebd1e61 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -29,7 +29,7 @@ source ./buildkite/scripts/export-git-env-vars.sh # Download required debians from bucket locally if [ -z "$DEBS" ]; then echo "DEBS env var is empty. It should contains comma delimitered names of debians to install" - exit 1 + exit -1 else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do From a566cee3e99e9bc96136e9caf4af01e2b6a06cfc Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 15 Nov 2024 20:52:25 +0100 Subject: [PATCH 151/178] add more docs for the tool. Run job only in nightly --- buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall | 3 +++ scripts/tests/ledger_test_apply.sh | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall index 4704292f7be..dca0b4b89ea 100644 --- a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall +++ b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall @@ -1,5 +1,7 @@ let PipelineTag = ../../Pipeline/Tag.dhall +let PipelineMode = ../../Pipeline/Mode.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall @@ -72,6 +74,7 @@ in Pipeline.build , PipelineTag.Type.Test , PipelineTag.Type.Stable ] + , mode = PipelineMode.Type.Stable } , steps = [ buildTestCmd Size.XLarge ] } diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index 98cfd3cb86a..77f4601a926 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -1,7 +1,11 @@ #!/bin/bash -# test mina ledger test-apply -set -x +# executes mina ledger test-apply command which is a +# Benchmark-like tool that: +# 1. Creates a set number of 9-account-updates transactions +# 2. Applies these transactions to the ledger by calling update_coinbase_stack_and_get_data (which from runs on whale-1-1 seemed to be the major cost center), +# using empty scan state and pending coinbase collections +# Tool might be useful for detecting regression related to mentioned areas export MINA_PRIVKEY_PASS='naughty blue worm' From 668fcfc013caf07d7e83ba9da8c1a7ae5fc067aa Mon Sep 17 00:00:00 2001 From: dkijania Date: Sun, 17 Nov 2024 21:37:34 +0100 Subject: [PATCH 152/178] Revert "Auxiliary commit to revert individual files from 49ce49019071ca5186f48e928d02541f016c1e7d" This reverts commit ee5267e267f4a40c4dab67f61858b8780b3018a8. --- src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml b/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml index 0b2a42f4858..298f50ef14e 100644 --- a/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml +++ b/src/app/runtime_genesis_ledger/runtime_genesis_ledger.ml @@ -97,7 +97,7 @@ let load_config_exn ~logger config_file = in if Option.( - is_some config.daemon + is_some config.daemon || is_some config.genesis || Option.value_map ~default:false ~f:is_dirty_proof config.proof) then failwith "Runtime config has unexpected fields" ; let ledger = Option.value_exn ~message:"No ledger provided" config.ledger in From b3a2f586614c01f61521dd4da9b51ea53b05dd06 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 18 Nov 2024 13:11:55 +0100 Subject: [PATCH 153/178] bump up number of accounts --- scripts/tests/ledger_test_apply.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index 77f4601a926..d11713a3800 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -25,7 +25,7 @@ esac; shift; done echo "Exporting ledger to $TEMP_FOLDER" -$MINA_APP ledger test generate-accounts -n 2 --min-balance 100000 --max-balance 1000000 > $ACCOUNTS_FILE +$MINA_APP ledger test generate-accounts -n 200000 --min-balance 100000 --max-balance 1000000 > $ACCOUNTS_FILE jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER From a5ee0c0b771087f74b13d87ea71f856aa3494ce2 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 19:16:22 +0100 Subject: [PATCH 154/178] moved test apply to pull request --- buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall | 2 +- scripts/tests/ledger_test_apply.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall index dca0b4b89ea..c0a4b2c26cc 100644 --- a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall +++ b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall @@ -74,7 +74,7 @@ in Pipeline.build , PipelineTag.Type.Test , PipelineTag.Type.Stable ] - , mode = PipelineMode.Type.Stable + , mode = PipelineMode.Type.PullRequest } , steps = [ buildTestCmd Size.XLarge ] } diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index d11713a3800..332ff6e6fda 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -25,7 +25,7 @@ esac; shift; done echo "Exporting ledger to $TEMP_FOLDER" -$MINA_APP ledger test generate-accounts -n 200000 --min-balance 100000 --max-balance 1000000 > $ACCOUNTS_FILE +$MINA_APP ledger test generate-accounts -n 200000 --min-balance 10000 --max-balance 1000000 > $ACCOUNTS_FILE jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER From 421e1f1546abfddd366438965ad0ce52dceb4799 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 20:01:13 +0100 Subject: [PATCH 155/178] remove magic config_file --- buildkite/scripts/tests/ledger_test_apply.sh | 3 +++ scripts/tests/ledger_test_apply.sh | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/buildkite/scripts/tests/ledger_test_apply.sh b/buildkite/scripts/tests/ledger_test_apply.sh index 640bbfa014d..d72c20154b7 100755 --- a/buildkite/scripts/tests/ledger_test_apply.sh +++ b/buildkite/scripts/tests/ledger_test_apply.sh @@ -14,6 +14,9 @@ source buildkite/scripts/export-git-env-vars.sh source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1 +echo "removing magic config files" +sudo rm -f /var/lib/coda/config_* + ./scripts/tests/ledger_test_apply.sh \ --mina-app mina \ --runtime-ledger-app mina-create-genesis \ No newline at end of file diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index 332ff6e6fda..a60eff9f104 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -7,6 +7,8 @@ # using empty scan state and pending coinbase collections # Tool might be useful for detecting regression related to mentioned areas +set -exo pipefail + export MINA_PRIVKEY_PASS='naughty blue worm' MINA_APP=_build/default/src/app/cli/src/mina.exe @@ -29,7 +31,9 @@ $MINA_APP ledger test generate-accounts -n 200000 --min-balance 10000 --max-bala jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER + echo "creating runtime ledger in $TEMP_FOLDER" + $RUNTIME_LEDGER_APP --config-file $GENESIS_LEDGER --genesis-dir $TEMP_FOLDER/genesis --hash-output-file $TEMP_FOLDER/genesis/hash.out --ignore-missing From 47ce3af95393d7bd4a7bed0e1992ee98b61d77f0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 21:02:02 +0100 Subject: [PATCH 156/178] adapt ledger before testing --- scripts/tests/ledger_test_apply.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index a60eff9f104..79f3a99b519 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -27,10 +27,14 @@ esac; shift; done echo "Exporting ledger to $TEMP_FOLDER" -$MINA_APP ledger test generate-accounts -n 200000 --min-balance 10000 --max-balance 1000000 > $ACCOUNTS_FILE +$MINA_APP ledger test generate-accounts -n 20000 --min-balance 1000 --max-balance 10000 > $ACCOUNTS_FILE -jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER +echo "Adapt ledger for tests" +# give more MINA to first account which will be sending founds +jq '(.[0] | .balance ) |= "20000000"' $ACCOUNTS_FILE +# construct correct ledger file +jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER echo "creating runtime ledger in $TEMP_FOLDER" From 2f683ed250507520734ddd211316dbcf9e050686 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 21:48:11 +0100 Subject: [PATCH 157/178] update correctly accounts file --- scripts/tests/ledger_test_apply.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index 79f3a99b519..f726b8afa58 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -32,7 +32,7 @@ $MINA_APP ledger test generate-accounts -n 20000 --min-balance 1000 --max-balanc echo "Adapt ledger for tests" # give more MINA to first account which will be sending founds -jq '(.[0] | .balance ) |= "20000000"' $ACCOUNTS_FILE +jq '(.[0] | .balance ) |= "20000000"' $ACCOUNTS_FILE > $ACCOUNTS_FILE # construct correct ledger file jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER From 6c7017776e01ad46f4ff9ae0607dd92193b7e4b3 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 08:58:27 +0100 Subject: [PATCH 158/178] update correct ledger json --- scripts/tests/ledger_test_apply.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index f726b8afa58..0875669c8af 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -16,6 +16,7 @@ RUNTIME_LEDGER_APP=_build/default/src/app/runtime_genesis_ledger/runtime_genesis TEMP_FOLDER=$(mktemp -d) ACCOUNTS_FILE=$TEMP_FOLDER/accounts.json +TEMP_ACCOUNTS_FILE=$TEMP_FOLDER/accounts_tmp.json GENESIS_LEDGER=$TEMP_FOLDER/genesis_ledger.config SENDER=$TEMP_FOLDER/sender @@ -32,7 +33,9 @@ $MINA_APP ledger test generate-accounts -n 20000 --min-balance 1000 --max-balanc echo "Adapt ledger for tests" # give more MINA to first account which will be sending founds -jq '(.[0] | .balance ) |= "20000000"' $ACCOUNTS_FILE > $ACCOUNTS_FILE +jq '(.[0] | .balance ) |= "20000000"' $ACCOUNTS_FILE > $TEMP_ACCOUNTS_FILE +mv $TEMP_ACCOUNTS_FILE $ACCOUNTS_FILE + # construct correct ledger file jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER From 6784bdd6936433649830f2bdb379edce71788bea Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 18:43:58 +0100 Subject: [PATCH 159/178] revert unrelated change --- buildkite/scripts/debian/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index 17d5ebd1e61..dfe1c425367 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -8,7 +8,7 @@ fi if [ -z $MINA_DEB_CODENAME ]; then echo "MINA_DEB_CODENAME env var is not defined" - exit 1 + exit -1 fi DEBS=$1 From 09345af6a43888493a37b95ea0b90d9f962d9ade Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 18:49:12 +0100 Subject: [PATCH 160/178] add comment regarding 20k accounts and optimize jq command --- scripts/tests/ledger_test_apply.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh index 0875669c8af..8c04578aff9 100755 --- a/scripts/tests/ledger_test_apply.sh +++ b/scripts/tests/ledger_test_apply.sh @@ -28,6 +28,7 @@ esac; shift; done echo "Exporting ledger to $TEMP_FOLDER" +echo "20k accounts is way less than the size of a mainnet ledger (200k), but good enough for testing" $MINA_APP ledger test generate-accounts -n 20000 --min-balance 1000 --max-balance 10000 > $ACCOUNTS_FILE echo "Adapt ledger for tests" @@ -37,7 +38,7 @@ jq '(.[0] | .balance ) |= "20000000"' $ACCOUNTS_FILE > $TEMP_ACCOUNTS_FILE mv $TEMP_ACCOUNTS_FILE $ACCOUNTS_FILE # construct correct ledger file -jq '{ ledger: { accounts:( $inputs | .[] ) , "add_genesis_winner": false } }' $ACCOUNTS_FILE --slurpfile inputs $ACCOUNTS_FILE > $GENESIS_LEDGER +jq '{ ledger: { accounts: . , "add_genesis_winner": false } }' < $ACCOUNTS_FILE > $GENESIS_LEDGER echo "creating runtime ledger in $TEMP_FOLDER" From 7ffbc4f02f204647735f55c3b50e50ceff2a96bc Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 20:43:43 +0100 Subject: [PATCH 161/178] fix LedgerTestApplyTx.dhall --- buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall index c0a4b2c26cc..9cbab00b41b 100644 --- a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall +++ b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall @@ -18,6 +18,8 @@ let Profiles = ../../Constants/Profiles.dhall let BuildFlags = ../../Constants/BuildFlags.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -26,12 +28,14 @@ let dependsOn = DebianVersions.dependsOnStep (None Text) DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard BuildFlags.Type.Instrumented "build" # DebianVersions.dependsOnStep (None Text) DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard BuildFlags.Type.None "build" From b0e5eb79c71737b90d8037f546356b03f9ca6480 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 17:30:23 +0100 Subject: [PATCH 162/178] use devnet --- buildkite/scripts/tests/ledger_test_apply.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/tests/ledger_test_apply.sh b/buildkite/scripts/tests/ledger_test_apply.sh index d72c20154b7..ce8304244a6 100755 --- a/buildkite/scripts/tests/ledger_test_apply.sh +++ b/buildkite/scripts/tests/ledger_test_apply.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1 +source buildkite/scripts/debian/install.sh "mina-devnet-instrumented" 1 echo "removing magic config files" sudo rm -f /var/lib/coda/config_* From 53a71ebfa31e1fd3a3d323030f9a16e8355c3911 Mon Sep 17 00:00:00 2001 From: leopardracer <136604165+leopardracer@users.noreply.github.com> Date: Sun, 1 Dec 2024 19:39:18 +0200 Subject: [PATCH 163/178] Update README-dev.md --- README-dev.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-dev.md b/README-dev.md index 9e0478ae2aa..61617d1c4bf 100644 --- a/README-dev.md +++ b/README-dev.md @@ -261,7 +261,7 @@ Debian package can be build locally by using below commands: ``` make build ``` -2. Build debian for mina-devet (example for ubuntu 18.04): +2. Build debian for mina-devnet (example for ubuntu 18.04): ``` ./scripts/debian/build.sh daemon_devnet ``` From 7b68eec02f16a2297d5d41663bbfb5a411c911d5 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 10:16:02 +0100 Subject: [PATCH 164/178] fix docker tag and exit on error fix cleanup method name fix rosetta-connectivity.sh name increase timeout for rosetta connectivity test add connectivity tests export postgres variables in docker-start.sh use single-run option for missing-block-guardian us mina missing block guardian script --- .../src/Command/Rosetta/Connectivity.dhall | 110 ++++++++++++++++++ buildkite/src/Constants/DockerVersions.dhall | 13 ++- .../src/Jobs/Test/RosettaDevnetConnect.dhall | 10 ++ .../src/Jobs/Test/RosettaMainnetConnect.dhall | 10 ++ scripts/tests/rosetta-connectivity.sh | 105 +++++++++++++++++ 5 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 buildkite/src/Command/Rosetta/Connectivity.dhall create mode 100644 buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall create mode 100644 buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall create mode 100755 scripts/tests/rosetta-connectivity.sh diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall new file mode 100644 index 00000000000..fdd2e18c94d --- /dev/null +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -0,0 +1,110 @@ +let Cmd = ../../Lib/Cmds.dhall + +let B = ../../External/Buildkite.dhall + +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall + +let Size = ../../Command/Size.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + +let Dockers = ../../Constants/DockerVersions.dhall + +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +let Spec = + { Type = + { dockerType : Dockers.Type + , network : Network.Type + , mode : PipelineMode.Type + , additionalDirtyWhen : List S.Type + , softFail : B/SoftFail + } + , default = + { dockerType = Dockers.Type.Bullseye + , network = Network.Type.Devnet + , mode = PipelineMode.Type.PullRequest + , additionalDirtyWhen = [] : List S.Type + , softFail = B/SoftFail.Boolean False + } + } + +let command + : Spec.Type -> Command.Type + = \(spec : Spec.Type) + -> Command.build + Command.Config::{ + , commands = + [ Cmd.chain + [ "export MINA_DEB_CODENAME=${Dockers.lowerName + spec.dockerType}" + , "source ./buildkite/scripts/export-git-env-vars.sh" + , "scripts/tests/rosetta-connectivity.sh --network ${Network.lowerName + spec.network} --tag \\\${MINA_DOCKER_TAG}" + ] + ] + , label = + "Rosetta ${Network.lowerName spec.network} connectivity test " + , key = + "rosetta-${Network.lowerName spec.network}-connectivity-test" + , target = Size.XLarge + , soft_fail = Some spec.softFail + , depends_on = + Dockers.dependsOn + spec.dockerType + spec.network + Profiles.Type.Standard + Artifacts.Type.Rosetta + } + +let pipeline + : Spec.Type -> Pipeline.Config.Type + = \(spec : Spec.Type) + -> Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly + "buildkite/src/Jobs/Test/RosettaIntegrationTests" + "dhall" + , S.exactly + "buildkite/src/Jobs/Test/Rosetta${Network.capitalName + spec.network}Connect" + "dhall" + , S.exactly + "buildkite/src/Command/Rosetta/Connectivity" + "dhall" + , S.exactly "scripts/tests/rosetta-connectivity" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + , S.exactly + "buildkite/scripts/rosetta-integration-tests-full" + "sh" + ] + # spec.additionalDirtyWhen + , path = "Test" + , name = "Rosetta${Network.capitalName spec.network}Connect" + , mode = spec.mode + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + ] + } + , steps = [ command spec ] + } + +in { command = command, pipeline = pipeline, Spec = Spec } diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 926a077b77d..eb2dfeb9600 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -28,8 +28,9 @@ let lowerName = } docker -let dependsOn = +let dependsOnStep = \(docker : Docker) + -> \(prefix : Text) -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) @@ -37,8 +38,6 @@ let dependsOn = let profileSuffix = "${Profiles.toSuffixUppercase profile}" - let prefix = "MinaArtifact" - let suffix = "docker-image" let key = "${Artifacts.lowerName binary}-${suffix}" @@ -77,8 +76,16 @@ let dependsOn = } docker +let dependsOn = + \(docker : Docker) + -> \(network : Network.Type) + -> \(profile : Profiles.Type) + -> \(binary : Artifacts.Type) + -> dependsOnStep docker "MinaArtifact" network profile binary + in { Type = Docker , capitalName = capitalName , lowerName = lowerName , dependsOn = dependsOn + , dependsOnStep = dependsOnStep } diff --git a/buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall b/buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall new file mode 100644 index 00000000000..a41f4cb5484 --- /dev/null +++ b/buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall @@ -0,0 +1,10 @@ +let Pipeline = ../../Pipeline/Dsl.dhall + +let Network = ../../Constants/Network.dhall + +let Connectivity = ../../Command/Rosetta/Connectivity.dhall + +in Pipeline.build + ( Connectivity.pipeline + Connectivity.Spec::{ network = Network.Type.Devnet } + ) diff --git a/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall new file mode 100644 index 00000000000..8aa6616d614 --- /dev/null +++ b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall @@ -0,0 +1,10 @@ +let Pipeline = ../../Pipeline/Dsl.dhall + +let Network = ../../Constants/Network.dhall + +let Connectivity = ../../Command/Rosetta/Connectivity.dhall + +in Pipeline.build + ( Connectivity.pipeline + Connectivity.Spec::{ network = Network.Type.Mainnet } + ) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh new file mode 100755 index 00000000000..853b885bb07 --- /dev/null +++ b/scripts/tests/rosetta-connectivity.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +# end to end test for rosetta connectivity with given network +set -exo pipefail +CLEAR='\033[0m' +RED='\033[0;31m' + +NETWORK=devnet + +while [[ "$#" -gt 0 ]]; do case $1 in + -n|--network) NETWORK="$2"; shift;; + -t|--tag) TAG="$2"; shift;; + -h|--help) usage; exit 0;; + *) echo "Unknown parameter passed: $1"; usage; exit 1;; +esac; shift; done + + +function usage() { + if [[ -n "$1" ]]; then + echo -e "${RED}☞ $1${CLEAR}\n"; + fi + echo "Usage: $0 [-t docker-tag] [-n network]" + echo " -t, --version The version to be used in the docker image tag" + echo " -n, --network The network configuration to use (devnet or mainnet). Default=$NETWORK" + echo " -h, --help Show help" + echo "" + echo "Example: $0 --network devnet --tag 3.0.3-bullseye-berkeley " + +} + +if [[ -z "$TAG" ]]; then usage "Docker tag is not set!"; usage; exit 1; fi; + +stop_docker() { + docker stop "$container_id" && docker rm "$container_id" +} + +cleanup_on_error() { + echo "Error occurred. Stopping and removing Docker container..." + stop_docker + exit 1 +} + +cleanup_on_exit() { + stop_docker + exit 0 +} + + +container_id=$(docker run -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK ) + +# Trap errors and call the cleanup function +trap cleanup_on_error ERR +trap cleanup_on_exit EXIT + +# Command to run the process +process_command="docker logs $container_id -f" + +# Pattern to wait for in the output +pattern="Added block with state hash" + +# Timeout duration in seconds +timeout_duration=900 + +# Required number of matches +required_matches=10 + +# Count of pattern matches +match_count=0 + +# Start time +start_time=$(date +%s) + +# Run the rosetta docker and check its output +$process_command | while read -r line; do + # Get the current time + current_time=$(date +%s) + + # Check if the timeout has been exceeded + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout_duration" ]; then + echo "Timeout reached! The pattern ('$pattern') was not found $required_matches times." + exit 1 + fi + + # If the pattern is found, increment the match count + if [[ "$line" =~ $pattern ]]; then + ((match_count++)) + echo "Pattern found ($match_count/$required_matches): $line" + fi + + # If the required number of matches is reached, exit the loop + if [ "$match_count" -ge "$required_matches" ]; then + echo "Pattern found $required_matches times. Exiting..." + break + fi +done + +# Check if we met the required match count +if [ "$match_count" -ge "$required_matches" ]; then + echo "Successfully found the pattern ('$pattern') $required_matches times." +else + echo "Failed to find the pattern ('$pattern') $required_matches times within the timeout." + exit 1 +fi + From 56c21311026b2c31609a20a58d70b1715fd75051 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 21:24:14 +0100 Subject: [PATCH 165/178] fix rosetta script --- scripts/tests/rosetta-connectivity.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh index 853b885bb07..a138f133807 100755 --- a/scripts/tests/rosetta-connectivity.sh +++ b/scripts/tests/rosetta-connectivity.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # end to end test for rosetta connectivity with given network -set -exo pipefail +set -euo pipefail CLEAR='\033[0m' RED='\033[0;31m' @@ -40,17 +40,10 @@ cleanup_on_error() { exit 1 } -cleanup_on_exit() { - stop_docker - exit 0 -} - - container_id=$(docker run -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK ) # Trap errors and call the cleanup function trap cleanup_on_error ERR -trap cleanup_on_exit EXIT # Command to run the process process_command="docker logs $container_id -f" @@ -85,7 +78,7 @@ $process_command | while read -r line; do # If the pattern is found, increment the match count if [[ "$line" =~ $pattern ]]; then ((match_count++)) - echo "Pattern found ($match_count/$required_matches): $line" + echo "Pattern found ($match_count of $required_matches): $line" fi # If the required number of matches is reached, exit the loop @@ -103,3 +96,4 @@ else exit 1 fi +stop_docker \ No newline at end of file From d298b8977d1e3f5dbe53de605a239ab29455459f Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 22:22:38 +0100 Subject: [PATCH 166/178] fix match_count increment --- scripts/tests/rosetta-connectivity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh index a138f133807..8e2f98a1ee8 100755 --- a/scripts/tests/rosetta-connectivity.sh +++ b/scripts/tests/rosetta-connectivity.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # end to end test for rosetta connectivity with given network -set -euo pipefail +set -euox pipefail CLEAR='\033[0m' RED='\033[0;31m' @@ -77,7 +77,7 @@ $process_command | while read -r line; do # If the pattern is found, increment the match count if [[ "$line" =~ $pattern ]]; then - ((match_count++)) + match_count=$((match_count+1)) echo "Pattern found ($match_count of $required_matches): $line" fi From f0315c28ddb4086e71d397e047fd0e0f1dea1f71 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 20:15:44 +0100 Subject: [PATCH 167/178] fix loop which listens for docker logs --- scripts/tests/rosetta-connectivity.sh | 28 +++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh index 8e2f98a1ee8..c3226f0225f 100755 --- a/scripts/tests/rosetta-connectivity.sh +++ b/scripts/tests/rosetta-connectivity.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # end to end test for rosetta connectivity with given network -set -euox pipefail +set -x CLEAR='\033[0m' RED='\033[0;31m' @@ -30,20 +30,13 @@ function usage() { if [[ -z "$TAG" ]]; then usage "Docker tag is not set!"; usage; exit 1; fi; +container_id=$(docker run -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK ) + stop_docker() { - docker stop "$container_id" && docker rm "$container_id" + { docker stop "$container_id" ; docker rm "$container_id" ; } || true } -cleanup_on_error() { - echo "Error occurred. Stopping and removing Docker container..." - stop_docker - exit 1 -} - -container_id=$(docker run -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK ) - -# Trap errors and call the cleanup function -trap cleanup_on_error ERR +trap stop_docker ERR # Command to run the process process_command="docker logs $container_id -f" @@ -64,7 +57,7 @@ match_count=0 start_time=$(date +%s) # Run the rosetta docker and check its output -$process_command | while read -r line; do +while IFS= read -t 10 -r line; do # Get the current time current_time=$(date +%s) @@ -86,14 +79,15 @@ $process_command | while read -r line; do echo "Pattern found $required_matches times. Exiting..." break fi -done +done < <($process_command) + +stop_docker # Check if we met the required match count if [ "$match_count" -ge "$required_matches" ]; then echo "Successfully found the pattern ('$pattern') $required_matches times." + exit 0 else echo "Failed to find the pattern ('$pattern') $required_matches times within the timeout." exit 1 -fi - -stop_docker \ No newline at end of file +fi \ No newline at end of file From c6ba18294d1104b23186c29e8b7f7f5df22c3714 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 22:29:48 +0100 Subject: [PATCH 168/178] increase wait timeout for next line in docker output --- scripts/tests/rosetta-connectivity.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh index c3226f0225f..bd0cb97739e 100755 --- a/scripts/tests/rosetta-connectivity.sh +++ b/scripts/tests/rosetta-connectivity.sh @@ -56,8 +56,11 @@ match_count=0 # Start time start_time=$(date +%s) +# Wait timeout for next line in output +next_line_timeout=60 + # Run the rosetta docker and check its output -while IFS= read -t 10 -r line; do +while IFS= read -t $next_line_timeout -r line; do # Get the current time current_time=$(date +%s) From 7f564864077c2fbfc96d22808977f71afb9ba0b9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 16:31:26 +0100 Subject: [PATCH 169/178] increase timeout for mainnet --- buildkite/src/Command/Rosetta/Connectivity.dhall | 5 ++++- buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall | 2 +- scripts/tests/rosetta-connectivity.sh | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index fdd2e18c94d..0ad195d8bc6 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -33,6 +33,7 @@ let Spec = , mode : PipelineMode.Type , additionalDirtyWhen : List S.Type , softFail : B/SoftFail + , timeout : Natural } , default = { dockerType = Dockers.Type.Bullseye @@ -40,6 +41,7 @@ let Spec = , mode = PipelineMode.Type.PullRequest , additionalDirtyWhen = [] : List S.Type , softFail = B/SoftFail.Boolean False + , timeout = 900 } } @@ -54,7 +56,8 @@ let command spec.dockerType}" , "source ./buildkite/scripts/export-git-env-vars.sh" , "scripts/tests/rosetta-connectivity.sh --network ${Network.lowerName - spec.network} --tag \\\${MINA_DOCKER_TAG}" + spec.network} --tag \\\${MINA_DOCKER_TAG} --timeout ${Natural/show + spec.timeout}" ] ] , label = diff --git a/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall index 8aa6616d614..c21e242f30f 100644 --- a/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall +++ b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall @@ -6,5 +6,5 @@ let Connectivity = ../../Command/Rosetta/Connectivity.dhall in Pipeline.build ( Connectivity.pipeline - Connectivity.Spec::{ network = Network.Type.Mainnet } + Connectivity.Spec::{ network = Network.Type.Mainnet, timeout = 1200 } ) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh index bd0cb97739e..d48bb918941 100755 --- a/scripts/tests/rosetta-connectivity.sh +++ b/scripts/tests/rosetta-connectivity.sh @@ -6,10 +6,12 @@ CLEAR='\033[0m' RED='\033[0;31m' NETWORK=devnet +TIMEOUT=900 while [[ "$#" -gt 0 ]]; do case $1 in -n|--network) NETWORK="$2"; shift;; -t|--tag) TAG="$2"; shift;; + --timeout) TIMEOUT="$2"; shift;; -h|--help) usage; exit 0;; *) echo "Unknown parameter passed: $1"; usage; exit 1;; esac; shift; done @@ -45,7 +47,7 @@ process_command="docker logs $container_id -f" pattern="Added block with state hash" # Timeout duration in seconds -timeout_duration=900 +timeout_duration=$TIMEOUT # Required number of matches required_matches=10 From a86081a52e731d7d06738194b0273ff4498f17a0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 19:04:58 +0100 Subject: [PATCH 170/178] increase timeout event more --- buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall index c21e242f30f..7c081efd7c2 100644 --- a/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall +++ b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall @@ -6,5 +6,5 @@ let Connectivity = ../../Command/Rosetta/Connectivity.dhall in Pipeline.build ( Connectivity.pipeline - Connectivity.Spec::{ network = Network.Type.Mainnet, timeout = 1200 } + Connectivity.Spec::{ network = Network.Type.Mainnet, timeout = 2400 } ) From 5e1f78169ea9ad6fc8252ab0259b73309511e652 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 2 Dec 2024 17:58:02 +0100 Subject: [PATCH 171/178] increase timeout --- buildkite/src/Command/Rosetta/Connectivity.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index 0ad195d8bc6..9c44cc24e51 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -41,7 +41,7 @@ let Spec = , mode = PipelineMode.Type.PullRequest , additionalDirtyWhen = [] : List S.Type , softFail = B/SoftFail.Boolean False - , timeout = 900 + , timeout = 1000 } } From de9a50bfff6cb4764b73830d5d19670149c8628e Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 3 Dec 2024 09:08:58 +0100 Subject: [PATCH 172/178] run connectivity tests only in stable pipelines --- buildkite/src/Command/Rosetta/Connectivity.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index 9c44cc24e51..0335aeeced1 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -38,7 +38,7 @@ let Spec = , default = { dockerType = Dockers.Type.Bullseye , network = Network.Type.Devnet - , mode = PipelineMode.Type.PullRequest + , mode = PipelineMode.Type.Stable , additionalDirtyWhen = [] : List S.Type , softFail = B/SoftFail.Boolean False , timeout = 1000 From e345d120125af2ec83c2e92db5fe80500f27deb3 Mon Sep 17 00:00:00 2001 From: Dmytrol <46675332+Dimitrolito@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:19:26 +0200 Subject: [PATCH 173/178] typos README.md --- buildkite/src/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/README.md b/buildkite/src/README.md index d020eb0d2c2..1ef17d334ee 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -1,6 +1,6 @@ # Buildkite CI -This folder contains all dhall code which is an backbone for our CI related code for buildkite. +This folder contains all dhall code which is a backbone for our CI related code for buildkite. # Structure @@ -204,4 +204,4 @@ We want only to move dockers from gcr to dockerhub without changing version. Cur - "NEW_VERSION=3.0.0-dc6bf78" - "CODENAMES=Focal,Bullseye" - "PUBLISH=1" -``` \ No newline at end of file +``` From a50025e091bb168d4060addd6d50c9d8ac3fa15d Mon Sep 17 00:00:00 2001 From: Dmytrol <46675332+Dimitrolito@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:20:16 +0200 Subject: [PATCH 174/178] typos README.md --- buildkite/src/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/README.md b/buildkite/src/README.md index 1ef17d334ee..1643697b1c0 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -53,7 +53,7 @@ User defined value which describe current pipeline chunk of jobs to be executed. - coverage gathering - which gathers coverage artifacts and uploads it to coveralls.io To reach above pipeline configuration below configuration can be provided: -(non important attributes where omitted) +(non-important attributes where omitted) ``` steps: - commands: From f6b1a216882e3eff4ced7a033039fe6dd9b1c3b1 Mon Sep 17 00:00:00 2001 From: Dmytrol <46675332+Dimitrolito@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:21:05 +0200 Subject: [PATCH 175/178] typos README.md --- buildkite/src/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/README.md b/buildkite/src/README.md index 1643697b1c0..372da8d329c 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -195,7 +195,7 @@ Pipeline with create 6 jobs for each Docker and Debian component separately. #### Promoting dockers form gcr to dockerhub -We want only to move dockers from gcr to dockerhub without changing version. Current implementation of pipeline is not user friendly so we need to still define `FROM_VERSION` and `TO_VERSION`. They should be equal. +We want only to move dockers from gcr to dockerhub without changing version. Current implementation of pipeline is not user-friendly so we need to still define `FROM_VERSION` and `TO_VERSION`. They should be equal. ``` - "DOCKERS=Archive,Daemon,Rosetta" From 84512ce89b6f40c9267f64112c0d45ae9166826c Mon Sep 17 00:00:00 2001 From: Dmytrol <46675332+Dimitrolito@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:22:45 +0200 Subject: [PATCH 176/178] typos 0016-transition-frontier-persistence.md --- rfcs/0016-transition-frontier-persistence.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/0016-transition-frontier-persistence.md b/rfcs/0016-transition-frontier-persistence.md index 8a719a907a4..90a67d971d8 100644 --- a/rfcs/0016-transition-frontier-persistence.md +++ b/rfcs/0016-transition-frontier-persistence.md @@ -8,7 +8,7 @@ This RFC proposes a new system for persisting the transition frontier's state to ## Motivation [motivation]: #motivation -The Transition Frontier is too large of a data structure to just blindly serialize and write to disk. Under non optimal network scenarios, we expect the upper bound of the data structure to be >100Gb. Even if the structure were smaller, we cannot write the structure out to disk every time we mutate it as the speed of the transition frontier data structure is critical to the systems ability to prevent DDoS attacks. Therefore, a more robust and effecient system is required to persist the Transition Frontier to disk without negatively effecting the speed of operations on the in memory copy of the Transition Frontier. +The Transition Frontier is too large of a data structure to just blindly serialize and write to disk. Under non-optimal network scenarios, we expect the upper bound of the data structure to be >100Gb. Even if the structure were smaller, we cannot write the structure out to disk every time we mutate it as the speed of the transition frontier data structure is critical to the systems ability to prevent DDoS attacks. Therefore, a more robust and effecient system is required to persist the Transition Frontier to disk without negatively effecting the speed of operations on the in memory copy of the Transition Frontier. ## Detailed design [detailed-design]: #detailed-design From 1dd81fd2e1b96f47a63756ee9a6c630946a3c952 Mon Sep 17 00:00:00 2001 From: Dmytrol <46675332+Dimitrolito@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:23:42 +0200 Subject: [PATCH 177/178] typos 0016-transition-frontier-persistence.md --- rfcs/0016-transition-frontier-persistence.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/0016-transition-frontier-persistence.md b/rfcs/0016-transition-frontier-persistence.md index 90a67d971d8..81a227202d4 100644 --- a/rfcs/0016-transition-frontier-persistence.md +++ b/rfcs/0016-transition-frontier-persistence.md @@ -30,7 +30,7 @@ As actions are performed on the Transition Frontier, diffs are emitted and store Having two different mechanisms for writing the same data can be tricky as there can be bugs in one of the two mechanisms that would cause the data structures to become desynchronized. In order to help prevent this, we can introduce an incremental hash on top of the Transition Frontier which can be updated upon each diff application. This hash will give a direct and easy way to compare the structural equality of the two data structures. Being incremental, however, also means that the order of diff application needs to be the same across both data structures, so care needs to be taken with that ordering. Therefore, in a sense, this hash will represent the structure and content of the data structure, as well as the order in which actions were taken to get there. We only care about the former in our case, and the latter is just a consequence of the hash being incremental. -In order to calculate this hash correctly, we need to introduce a new concept to a diff, which is that of a diff mutant. Each diff represents some mutation to perform on the Transition Frontier, however not every diff will contain the enough information by itself to encapsulate the state of the data structure after the mutation occurs. For example, setting a balance on an account in two implementations of the data structure does not guarantee that the accounts in each a equal as there are other fields on the account besides that. This is where the concept of a diff mutant comes in. The mutant of a diff is the set of all modified values in the data structure after the diff has been applied. Using this, we can create a proper incremental diff which will truly ensure our data structures are in sync. +In order to calculate this hash correctly, we need to introduce a new concept to a diff, which is that of a diff mutant. Each diff represents some mutation to perform on the Transition Frontier, however not every diff will contain the enough information by itself to encapsulate the state of the data structure after the mutation occurs. For example, setting a balance on an account in two implementations of the data structure does not guarantee that the accounts in each an equal as there are other fields on the account besides that. This is where the concept of a diff mutant comes in. The mutant of a diff is the set of all modified values in the data structure after the diff has been applied. Using this, we can create a proper incremental diff which will truly ensure our data structures are in sync. These hashes will be Sha256 as there is no reason to use the Pedersen hashing algorithm we use in the rest of our code since none of this information needs to be snarked. The formula for calculating a new hash `h'` given an old hash `h` and a diff `diff` is as follows: `h' = sha256 h diff (Diff.mutant diff)`. From 168177c6ae5c60320ef5f734bf8609b49de7841c Mon Sep 17 00:00:00 2001 From: Dmytrol <46675332+Dimitrolito@users.noreply.github.com> Date: Sun, 15 Dec 2024 15:24:28 +0200 Subject: [PATCH 178/178] typos 0020-transition-frontier-extensions-2.md --- rfcs/0020-transition-frontier-extensions-2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfcs/0020-transition-frontier-extensions-2.md b/rfcs/0020-transition-frontier-extensions-2.md index c495a2fe2fb..15df4da40f4 100644 --- a/rfcs/0020-transition-frontier-extensions-2.md +++ b/rfcs/0020-transition-frontier-extensions-2.md @@ -28,7 +28,7 @@ See [#1585](https://github.com/CodaProtocol/coda/pull/1585) for early discussion ### Extensions Redefined -A Transition Frontier Extension is an stateful, incremental view on the state of a Transiton Frontier. When a Transition Frontier is initialized, all of its extensions are also initialized using the Transition Frontier's root. Every mutation performed is represented as a list of diffs, and when the Transition Frontier updates, each Extension is notified of this list of diffs synchronously. Transition Frontier Extensions will notify the Transition Frontier if there was a update to the Extension's view when handling the diffs. If an Extension's view is updated, then a synchronous event is broadcast internally with the new view of that Extension. A Transition Frontier Extension has access to the Transition Frontier so that it can query and calculate information it requires when it handles diffs. +A Transition Frontier Extension is a stateful, incremental view on the state of a Transiton Frontier. When a Transition Frontier is initialized, all of its extensions are also initialized using the Transition Frontier's root. Every mutation performed is represented as a list of diffs, and when the Transition Frontier updates, each Extension is notified of this list of diffs synchronously. Transition Frontier Extensions will notify the Transition Frontier if there was a update to the Extension's view when handling the diffs. If an Extension's view is updated, then a synchronous event is broadcast internally with the new view of that Extension. A Transition Frontier Extension has access to the Transition Frontier so that it can query and calculate information it requires when it handles diffs. ### Extension Guidelines