[rampup] Stop txn/network after slot feature

buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall

@@ -36,6 +36,7 @@ in Pipeline.build Pipeline.Config::{
     TestExecutive.execute "zkapps" dependsOn,
     TestExecutive.execute "zkapps-timing" dependsOn,
     TestExecutive.execute "zkapps-nonce" dependsOn,
-    TestExecutive.execute "verification-key" dependsOn
+    TestExecutive.execute "verification-key" dependsOn,
+    TestExecutive.execute "slot-end" dependsOn
   ]
 }

scripts/prepare-test-ledger.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+# Usage: ./scripts/prepare-test-ledger.sh <BP key 1> <BP key 2> ... <BP key n>
+
+# Number of keys in ledger that won't be re-delegated
+NUM_GHOST_KEYS=${NUM_GHOST_KEYS:-0}
+KEY_BALANCE=100000
+
+echo "Script assumes mainnet's start was at epoch 0 on 17th March 2021, if it's not the case, update the script please" >&2
+
+if [[ $# -eq 0 ]]; then
+  echo "Usage: ./scripts/prepare-test-ledger.sh <BP key 1> <BP key 2> ... <BP key n>" >&2
+  exit 1
+fi
+
+num_keys=$#
+num_keys_and_ghost=$((num_keys + NUM_GHOST_KEYS))
+
+now=$(date +%s)
+mainnet_start=$(date --date='2021-03-17 00:00:00' -u +%s)
+epoch_now=$(( (now-mainnet_start)/7140/180 ))
+
+ledger_file="$epoch_now.json"
+
+echo "Current epoch: $epoch_now" >&2
+
+if [[ ! -f "$ledger_file" ]]; then
+  ledgers_url="https://storage.googleapis.com/storage/v1/b/mina-staking-ledgers/o?maxResults=1&prefix=staking-$epoch_now"
+  ledger_url=$(curl "$ledgers_url" | jq -r .items[0].mediaLink)
+
+  wget -O $ledger_file "$ledger_url"
+fi
+
+keys_=""
+for key in "$@"; do
+  keys_="\"$key\",$keys_"
+done
+keys_="${keys_:0:-1}"
+
+# jq filter to exclude block PKs from the ledger
+pre_filter="[.[] | select(.pk | IN($keys_) | not)]"
+
+##########################################################
+# Calculate and print approximate new balances
+##########################################################
+
+num_accounts=$(<$ledger_file jq "$pre_filter | length")
+total_balance=$(<$ledger_file jq "$pre_filter | [.[].balance | tonumber] | add | round")
+new_total_balance=$((total_balance+num_keys*KEY_BALANCE))
+echo "Total accounts: $num_accounts, balance: $total_balance, new balance: $new_total_balance" >&2
+
+function make_balance_expr(){
+  i=$1
+  key=$2
+  echo "$key: (((([.[range($i; $num_accounts; $num_keys_and_ghost)].balance | tonumber] | add) + $KEY_BALANCE)/$new_total_balance*10000|round/100 | tostring) + \"%\")"
+}
+
+balance_expr=$({ i=0; while [[ $i -lt $num_keys ]]; do
+  j=$((i+1))
+  make_balance_expr $i "${!j}"
+  i=$j
+done } | tr "\n" "," | head -c -1)
+
+echo "Balance map:" >&2
+jq <$ledger_file "$pre_filter | {$balance_expr}" 1>&2
+
+##########################################################
+# Build new ledger
+##########################################################
+
+function make_expr(){
+  i=$1
+  key=$2
+
+  # Substitute delegate of some ledger keys with the BP key
+  echo ".[range($i; $num_accounts; $num_keys_and_ghost)].delegate = \"$key\""
+  echo ".[$((i+num_accounts))] = {pk:\"$key\", balance:\"$KEY_BALANCE\"}"
+}
+
+expr=$({ i=0; while [[ $i -lt $num_keys ]]; do
+  j=$((i+1))
+  make_expr $i "${!j}"
+  i=$j
+done } | tr "\n" "|" | head -c -1)
+
+jq <$ledger_file "$pre_filter | $expr"

src/app/test_executive/dune

@@ -59,6 +59,7 @@
    zkapps_examples
    cache_dir
    snarky.backendless
+   block_time
  )
  (instrumentation (backend bisect_ppx))
  (preprocess (pps ppx_mina ppx_jane ppx_deriving_yojson ppx_mina ppx_version)))

src/app/test_executive/slot_end_test.ml

@@ -0,0 +1,204 @@
+open Core
+open Integration_test_lib
+
+module Make (Inputs : Intf.Test.Inputs_intf) = struct
+  open Inputs
+  open Engine
+  open Dsl
+
+  open Test_common.Make (Inputs)
+
+  (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *)
+  type network = Network.t
+
+  type node = Network.Node.t
+
+  type dsl = Dsl.t
+
+  let num_extra_keys = 100
+
+  let slot_tx_end = 5
+
+  let slot_chain_end = 8
+
+  let sender_account_prefix = "sender-account-"
+
+  let config =
+    let open Test_config in
+    { default with
+      requires_graphql = true
+    ; genesis_ledger =
+        [ { Test_Account.account_name = "receiver-key"
+          ; balance = "9999999"
+          ; timing = Untimed
+          }
+        ; { account_name = "sender-1-key"; balance = "0"; timing = Untimed }
+        ; { account_name = "sender-2-key"; balance = "0"; timing = Untimed }
+        ; { account_name = "sender-3-key"; balance = "0"; timing = Untimed }
+        ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed }
+        ]
+        @ List.init num_extra_keys ~f:(fun i ->
+              { Test_Account.account_name =
+                  sprintf "%s-%d" sender_account_prefix i
+              ; balance = "1000"
+              ; timing = Untimed
+              } )
+    ; block_producers =
+        [ { node_name = "receiver"; account_name = "receiver-key" }
+        ; { node_name = "sender-1"; account_name = "sender-1-key" }
+        ; { node_name = "sender-2"; account_name = "sender-2-key" }
+        ; { node_name = "sender-3"; account_name = "sender-3-key" }
+        ]
+    ; snark_coordinator =
+        Some
+          { node_name = "snark-node"
+          ; account_name = "snark-node-key"
+          ; worker_nodes = 4
+          }
+    ; txpool_max_size = 10_000_000
+    ; snark_worker_fee = "0.0002"
+    ; num_archive_nodes = 0
+    ; proof_config =
+        { proof_config_default with
+          work_delay = Some 1
+        ; transaction_capacity =
+            Some Runtime_config.Proof_keys.Transaction_capacity.small
+        }
+    ; slot_tx_end = Some slot_tx_end
+    ; slot_chain_end = Some slot_chain_end
+    }
+
+  let fee = Currency.Fee.of_nanomina_int_exn 10_000_000
+
+  let amount = Currency.Amount.of_nanomina_int_exn 10_000_000
+
+  let tx_delay_ms = 5000
+
+  let run network t =
+    let open Malleable_error.Let_syntax in
+    let logger = Logger.create () in
+    let num_slots = slot_chain_end + 2 in
+    let receiver =
+      String.Map.find_exn (Network.block_producers network) "receiver"
+    in
+    let%bind receiver_pub_key = pub_key_of_node receiver in
+    let bp_senders =
+      String.Map.remove (Network.block_producers network) "receiver"
+      |> String.Map.data
+    in
+    let sender_kps =
+      String.Map.fold (Network.genesis_keypairs network) ~init:[]
+        ~f:(fun ~key ~data acc ->
+          if String.is_prefix key ~prefix:sender_account_prefix then data :: acc
+          else acc )
+    in
+    let sender_priv_keys =
+      List.map sender_kps ~f:(fun kp -> kp.keypair.private_key)
+    in
+    let pk_to_string = Signature_lib.Public_key.Compressed.to_base58_check in
+    [%log info] "receiver: %s" (pk_to_string receiver_pub_key) ;
+    let%bind () =
+      Malleable_error.List.iter sender_kps ~f:(fun s ->
+          let pk = s.keypair.public_key |> Signature_lib.Public_key.compress in
+          return ([%log info] "sender: %s" (pk_to_string pk)) )
+    in
+    let window_ms =
+      (Network.constraint_constants network).block_window_duration_ms
+    in
+    let all_nodes = Network.all_mina_nodes network in
+    let%bind () =
+      wait_for t
+        (Wait_condition.nodes_to_initialize (String.Map.data all_nodes))
+    in
+    let genesis_timestamp =
+      Block_time.to_time_exn
+      @@ Block_time.of_int64
+           (Network.genesis_constants network).protocol.genesis_state_timestamp
+    in
+    let end_t =
+      Time.add genesis_timestamp
+        (Time.Span.of_ms @@ float_of_int (num_slots * window_ms))
+    in
+    let slot_tx_end =
+      Mina_numbers.Global_slot_since_hard_fork.of_int slot_tx_end
+    in
+    let slot_chain_end =
+      Mina_numbers.Global_slot_since_hard_fork.of_int slot_chain_end
+    in
+    let%bind () =
+      section_hard "spawn transaction sending"
+        (let num_payments = num_slots * window_ms / tx_delay_ms in
+         let repeat_count = Unsigned.UInt32.of_int num_payments in
+         let repeat_delay_ms = Unsigned.UInt32.of_int tx_delay_ms in
+         let num_sender_keys = List.length sender_priv_keys in
+         let n_bp_senders = List.length bp_senders in
+         let keys_per_sender = num_sender_keys / n_bp_senders in
+         [%log info]
+           "will now send %d payments from as many accounts. %d nodes will \
+            send %d payments each from distinct keys"
+           num_payments n_bp_senders keys_per_sender ;
+         Malleable_error.List.fold ~init:sender_priv_keys bp_senders
+           ~f:(fun keys node ->
+             let keys0, rest = List.split_n keys keys_per_sender in
+             Integration_test_lib.Graphql_requests.must_send_test_payments
+               ~repeat_count ~repeat_delay_ms ~logger ~senders:keys0
+               ~receiver_pub_key ~amount ~fee
+               (Network.Node.get_ingress_uri node)
+             >>| const rest )
+         >>| const () )
+    in
+    let%bind () =
+      section
+        (Printf.sprintf "wait until slot %d" num_slots)
+        Async.(at end_t >>= const Malleable_error.ok_unit)
+    in
+    let ok_if_true s =
+      Malleable_error.ok_if_true ~error:(Error.of_string s) ~error_type:`Soft
+    in
+    let%bind blocks =
+      Integration_test_lib.Graphql_requests
+      .must_get_best_chain_for_slot_end_test ~max_length:(2 * num_slots) ~logger
+        (Network.Node.get_ingress_uri receiver)
+    in
+    let%bind () =
+      section "blocks produced before slot_tx_end"
+        ( ok_if_true "only empty blocks were produced before slot_tx_end"
+        @@ List.exists blocks ~f:(fun block ->
+               Mina_numbers.Global_slot_since_hard_fork.(
+                 block.slot < slot_tx_end)
+               && ( block.command_transaction_count <> 0
+                  || block.snark_work_count <> 0
+                  || Currency.Amount.(block.coinbase <> zero) ) ) )
+    in
+    let%bind () =
+      section "blocks produced after slot_tx_end"
+        (Malleable_error.List.iter blocks ~f:(fun block ->
+             let msg =
+               Printf.sprintf
+                 "non-empty block after slot_tx_end. block slot since genesis: \
+                  %s, txn count: %d, snark work count: %d, coinbase: %s"
+                 (Mina_numbers.Global_slot_since_hard_fork.to_string block.slot)
+                 block.command_transaction_count block.snark_work_count
+                 (Currency.Amount.to_string block.coinbase)
+             in
+             ok_if_true msg
+               ( Mina_numbers.Global_slot_since_hard_fork.(
+                   block.slot < slot_tx_end)
+               || block.command_transaction_count = 0
+                  && block.snark_work_count = 0
+                  && Currency.Amount.(block.coinbase = zero) ) ) )
+    in
+    let%bind () =
+      section "blocks produced before slot_chain_end"
+        ( ok_if_true "no block produced before slot_chain_end"
+        @@ List.exists blocks ~f:(fun block ->
+               Mina_numbers.Global_slot_since_hard_fork.(
+                 block.slot < slot_chain_end) ) )
+    in
+    section "no blocks produced after slot_chain_end"
+      ( ok_if_true "blocks produced after slot_chain_end"
+      @@ not
+      @@ List.exists blocks ~f:(fun block ->
+             Mina_numbers.Global_slot_since_hard_fork.(
+               block.slot >= slot_chain_end) ) )
+end

src/app/test_executive/slot_end_test.mli

@@ -0,0 +1 @@
+module Make : Integration_test_lib.Intf.Test.Functor_intf

src/app/test_executive/test_executive.ml

@@ -66,6 +66,7 @@ let tests : test list =
     , (module Block_production_priority.Make : Intf.Test.Functor_intf) )
   ; ("block-reward", (module Block_reward_test.Make : Intf.Test.Functor_intf))
   ; ("hard-fork", (module Hard_fork.Make : Intf.Test.Functor_intf))
+  ; ("slot-end", (module Slot_end_test.Make : Intf.Test.Functor_intf))
   ]
 
 let report_test_errors ~log_error_set ~internal_error_set =

src/config/debug.mlh

@@ -34,3 +34,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/dev.mlh

@@ -36,3 +36,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/dev_medium_curves.mlh

@@ -34,3 +34,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/dev_snark.mlh

@@ -34,3 +34,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/devnet.mlh

@@ -47,3 +47,5 @@
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/fake_hash.mlh

@@ -34,3 +34,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/fork.mlh

@@ -2,3 +2,5 @@
 [%%undef fork_previous_state_hash]
 [%%undef fork_genesis_slot]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/fork_at_3757.mlh

@@ -2,3 +2,5 @@
 [%%define fork_previous_state_hash "3NKR3QYJ7qwxiGgX39umahgdT8BH5yXBQwQtpYZdvodCXcsndK7f"]
 [%%define fork_genesis_slot 12796]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/fuzz_medium.mlh

@@ -33,3 +33,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/fuzz_small.mlh

@@ -33,3 +33,5 @@
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/integration_tests.mlh

@@ -6,3 +6,6 @@
    snark keys, a valid genesis proof, etc. *)
 [%%import "/src/config/proof_level/none.mlh"]
 [%%undef zkapp_cmd_limit]
+
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]

src/config/lightnet.mlh

@@ -47,3 +47,5 @@
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
 [%%undef zkapp_cmd_limit]
+[%%undef slot_tx_end]
+[%%undef slot_chain_end]