Merge branch 'develop' into merge-back-to-develop-2025-01-23

README-ci-failures.md

@@ -46,7 +46,7 @@ In this case, you should:
    [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain)
    where they have been automatically uploaded. Look for your branch name and
    commit hash in the second column. You should find several images suffixed
-   with `-bullseye` and `-focal` (Debian versions).
+   with `-bullseye` or `-focal` (Debian versions).
 3. For each such image, retrieve its full name and hash by hovering its link and
    clicking the `Copy full image name` tooltip that appears (or retrieve it on
    the image's page).

README-dev.md

@@ -261,7 +261,7 @@ Debian package can be build locally by using below commands:
 ```
 make build
 ```
-2. Build debian for mina-devet (example for ubuntu 18.04):
+2. Build debian for mina-devnet (example for ubuntu 18.04):
 ```
 ./scripts/debian/build.sh daemon_devnet
 ```

automation/services/watchdog/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.16-slim-buster
+FROM python:3.9.16-slim-bullseye
 
 ARG GCLOUDSDK_DOWNLOAD_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-296.0.1-linux-x86_64.tar.gz"
 ARG WATCHMAN_DEB_URL="http://ftp.us.debian.org/debian/pool/main/w/watchman/watchman_4.9.0-5+b1_amd64.deb"

buildkite/Makefile

@@ -32,4 +32,4 @@ check_dups: dump_pipelines
 check_names: dump_pipelines
 	python3 scripts/dhall/checker.py --root "$(TMP)" names
 
-all: check_syntax lint format check_deps check_dirty check_dups check_names
+all: check_syntax lint format check_deps check_dirty check_dups check_names

buildkite/scripts/bench/install.sh

@@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive
 sudo apt-get update
 sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3
 
-TESTNET_NAME="berkeley"
+TESTNET_NAME="devnet"
 
 git config --global --add safe.directory /workdir
 source buildkite/scripts/export-git-env-vars.sh

buildkite/scripts/check-compatibility.sh

@@ -10,7 +10,7 @@ function get_shas {
 
 function image_tag {
     SHA=$1
-    IMAGE_TAG="$SHA-bullseye-berkeley"
+    IMAGE_TAG="$SHA-bullseye-devnet"
 }
 
 function download-docker {
@@ -22,17 +22,25 @@ function download-docker {
 function try_docker_shas {
     DOCKER_SHAS=$1
     GOT_DOCKER=0
-
     for sha in $DOCKER_SHAS; do
-	download-docker $sha
-	if [ $? -eq 0 ] ; then
-	    GOT_DOCKER=1
-	    image_tag $sha
-	    break
-	else
-	    echo "No docker available for SHA=$sha"
-	fi
+
+        set +e
+        download-docker $sha
+
+        if [ $? -eq 0 ] ; then
+            GOT_DOCKER=1
+            image_tag $sha
+            break
+        else
+            echo "No docker available for SHA=$sha"
+        fi
+        set -e
     done
+
+    if [[ $GOT_DOCKER == 0 ]]; then
+        echo "docker cannot be found for given shas: $DOCKER_SHAS"
+        exit 1
+    fi
 }
 
 function image_id {

buildkite/scripts/connect-to-testnet.sh

@@ -7,53 +7,46 @@ if [[ $# -ne 3 ]]; then
     exit 1
 fi
 
-TESTNET_VERSION_NAME="berkeley"
 TESTNET_NAME=$1
 WAIT_BETWEEN_POLLING_GRAPHQL=$2
 WAIT_AFTER_FINAL_CHECK=$3
 
-case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
-  rampup|berkeley|release/2.0.0|develop)
-  ;;
-  *)
-    echo "Not pulling against rampup, not running the connect test"
-    exit 0 ;;
-esac
-
 # Don't prompt for answers during apt-get install
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get update
-apt-get install -y git apt-transport-https ca-certificates tzdata curl
+sudo apt-get update
+sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq
 
 git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}"
+source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1
 
 # Remove lockfile if present
-rm ~/.mina-config/.mina-lock ||:
+sudo rm ~/.mina-config/.mina-lock ||:
+
+sudo mkdir -p /root/libp2p-keys/
 
-mkdir -p /root/libp2p-keys/
-# Pre-generated random password for this quick test
-export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf
-mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key
 # Set permissions on the keypair so the daemon doesn't complain
-chmod -R 0700 /root/libp2p-keys/
+sudo chmod -R 0700 /root/libp2p-keys/
+# Pre-generated random password for this quick test
+sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key
 
 # Restart in the background
-mina daemon \
-  --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \
-  --libp2p-keypair "/root/libp2p-keys/key" \
-& # -background
+sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \
+    TESTNET_NAME=$TESTNET_NAME \
+  bash -c "mina daemon \
+  --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \
+  --libp2p-keypair \"/root/libp2p-keys/key\" \
+  --seed &" # -background
 
 # Attempt to connect to the GraphQL client every 10s for up to 8 minutes
 num_status_retries=24
 for ((i=1;i<=$num_status_retries;i++)); do
   sleep $WAIT_BETWEEN_POLLING_GRAPHQL
   set +e
-  mina client status
+  sudo mina client status
   status_exit_code=$?
   set -e
   if [ $status_exit_code -eq 0 ]; then
@@ -63,13 +56,18 @@ for ((i=1;i<=$num_status_retries;i++)); do
   fi
 done
 
-# Check that the daemon has connected to peers and is still up after 2 mins
-sleep $WAIT_AFTER_FINAL_CHECK
-mina client status
-if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then
-    echo "Found some peers"
-else
-    echo "No peers found"
-    exit 1
-fi
+peer_retries=10
+for ((i=1;i<=$peer_retries;i++)); do
+  peer_count=$(sudo mina advanced get-peers | wc -l)
+  sudo mina client status
+
+  if [ "$peer_count" -gt 0 ]; then
+      echo "Found some peers"
+      exit 0;      
+  else
+      echo "No peers found"
+  fi
+  sleep $WAIT_AFTER_FINAL_CHECK
+done
 
+exit 1;

buildkite/scripts/debian/install.sh

@@ -34,7 +34,7 @@ else
   debs=(${DEBS//,/ })
   for i in "${debs[@]}"; do
     case $i in
-      mina-berkeley*|mina-devnet|mina-mainnet)
+      mina-devnet*|mina-mainnet)
         # Downaload mina-logproc too
         source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER
       ;;

buildkite/scripts/dump-mina-type-shapes.sh

@@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive
 sudo apt-get update
 sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl
 
-TESTNET_NAME="berkeley"
+TESTNET_NAME="devnet"
 
 git config --global --add safe.directory /workdir
 source buildkite/scripts/export-git-env-vars.sh

buildkite/scripts/rosetta-integration-tests.sh

@@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then
   send_zkapp_transactions &
 fi
 
-next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13)
-sleep_time=$((($next_block_time - $curr_time) / 1000))
+mina client status --json
+
+next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13)
+sleep_time=$((($next_block_time - $curr_time) / 10000000))
 echo "Sleeping for ${sleep_time}s until next block is created..."
-sleep ${sleep_time}
+sleep $sleep_time
 
 # Mina Rosetta Checks (spec construction data perf)
 echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} =============="

buildkite/scripts/run-test-executive-cloud.sh

@@ -2,7 +2,7 @@
 set -o pipefail -x
 
 TEST_NAME="$1"
-MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley"
+MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet"
 ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG"
 
 if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then

buildkite/scripts/run-test-executive-local.sh

@@ -26,7 +26,7 @@ cleanup
 
 TEST_NAME="$1"
 
-MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley"
+MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet"
 ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG"
 
 if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then
@@ -42,7 +42,7 @@ rm -f /etc/apt/sources.list.d/hashicorp.list
 apt-get update
 apt-get install -y git apt-transport-https ca-certificates tzdata curl
 
-TESTNET_NAME="berkeley"
+TESTNET_NAME="devnet"
 
 git config --global --add safe.directory /workdir
 

buildkite/scripts/run_promote_build_job.sh

@@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)"
 if [[ -n "$DEBIANS" ]]; then 
     if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi;
     if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
-    if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi;
+    if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi;
     if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi;
     if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi;
     if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi;

buildkite/scripts/run_verify_promoted_build_job.sh

@@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)"
 if [[ -n "$DEBIANS" ]]; then 
     if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi;
     if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
-    if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi;
+    if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi;
     if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi;
     if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi;
     if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi;

buildkite/scripts/single-node-tests.sh

@@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1
+source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1
 
 export MINA_LIBP2P_PASS="naughty blue worm"
 export MINA_PRIVKEY_PASS="naughty blue worm"

buildkite/scripts/tests/ledger_test_apply.sh

@@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1
+source buildkite/scripts/debian/install.sh "mina-devnet-instrumented" 1
 
 echo "removing magic config files"
 sudo rm -f /var/lib/coda/config_*

buildkite/scripts/version-linter.sh

@@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then
     exit 1
 fi
 
-TESTNET_NAME="${TESTNET_NAME:-berkeley}"
+TESTNET_NAME="${TESTNET_NAME:-devnet}"
 
 # Don't prompt for answers during apt-get install
 export DEBIAN_FRONTEND=noninteractive

buildkite/src/Command/Bench/Base.dhall

@@ -10,6 +10,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall
 
 let RunInToolchain = ../../Command/RunInToolchain.dhall
 
+let Network = ../../Constants/Network.dhall
+
 let Profiles = ../../Constants/Profiles.dhall
 
 let Command = ../../Command/Base.dhall
@@ -42,6 +44,7 @@ let Spec =
           , dependsOn =
               DebianVersions.dependsOn
                 DebianVersions.DebVersion.Bullseye
+                Network.Type.Devnet
                 Profiles.Type.Standard
           , additionalDirtyWhen = [] : List SelectFiles.Type
           , yellowThreshold = 0.1