testing(flatkv): evm migration int testings

.github/workflows/integration-test.yml

@@ -187,6 +187,43 @@ jobs:
               "./integration_test/contracts/verify_flatkv_partial_loss_fails_loudly.sh",
             ],
           },
+          {
+            # FlatKV EVM migrate cluster coverage.
+            #
+            # The cluster boots with GIGA_MIGRATE_FROM_MEMIAVL=true so
+            # every validator starts in sc-write-mode = memiavl_only,
+            # i.e. v0 (FlatKV not yet allocated). This is the inverse of
+            # the FlatKV Integration row above (which boots in
+            # test_only_dual_write) and is what makes the migration
+            # script's pre-flip check meaningful: if the matrix env ever
+            # silently lands the cluster in any mode other than
+            # memiavl_only, the script will fail loudly at the pre-flip
+            # grep instead of "succeeding" with a no-op migration.
+            #
+            # Steps:
+            #   1  Deposit an EVM fixture while in v0 so the migration
+            #      has real account+code+storage to drain. The fixture
+            #      writes roughly 4000 storage keys by default so the
+            #      migration spans multiple batches instead of trivially
+            #      completing against an empty or smoke-sized tree.
+            #   2  Coordinated stop -> sed sc-write-mode -> restart on
+            #      all 4 validators, then poll seidb migrate-evm-status
+            #      until every validator reports completion. Cross-
+            #      validator FlatKV digest agreement is asserted at a
+            #      shared post-migration height; any non-determinism in
+            #      the batch copier would surface here as a digest
+            #      mismatch.
+            #   3  Re-run the fixture round-trip check against the
+            #      now-FlatKV-backed EVM state, confirming pre-migration
+            #      data survives the migration intact (read transparency).
+            name: "FlatKV EVM Migrate",
+            env: "GIGA_MIGRATE_FROM_MEMIAVL=true",
+            scripts: [
+              "docker exec sei-node-0 integration_test/contracts/deploy_flatkv_evm_fixture.sh",
+              "./integration_test/contracts/verify_flatkv_evm_migrate.sh",
+              "docker exec sei-node-0 integration_test/contracts/verify_flatkv_evm_store.sh",
+            ],
+          },
           {
             name: "EVM Module",
             env: "GIGA_STORAGE=true",

Makefile

@@ -295,7 +295,8 @@ CLUSTER_ENV_VARS = DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROU
 	GIGA_OCC=$(GIGA_OCC) \
 	RECEIPT_BACKEND=$(RECEIPT_BACKEND) \
 	AUTOBAHN=$(AUTOBAHN) \
-	GIGA_STORAGE=$(GIGA_STORAGE)
+	GIGA_STORAGE=$(GIGA_STORAGE) \
+	GIGA_MIGRATE_FROM_MEMIAVL=$(GIGA_MIGRATE_FROM_MEMIAVL)
 
 # Run a 4-node docker containers
 docker-cluster-start: docker-cluster-stop build-docker-node
@@ -321,7 +322,7 @@ docker-cluster-start-skipbuild: docker-cluster-stop build-docker-node
 		else \
 			DETACH_FLAG=""; \
 		fi; \
-		DOCKER_PLATFORM=$(DOCKER_PLATFORM) USERID=$(shell id -u) GROUPID=$(shell id -g) GOCACHE=$(shell go env GOCACHE) NUM_ACCOUNTS=10 SKIP_BUILD=true docker compose up $$DETACH_FLAG
+		$(CLUSTER_ENV_VARS) SKIP_BUILD=true docker compose up $$DETACH_FLAG
 .PHONY: localnet-start
 
 # Stop 4-node docker containers

app/seidb.go

@@ -28,6 +28,14 @@ const (
 	FlagSCHistoricalProofRateLimit   = "state-commit.sc-historical-proof-rate-limit"
 	FlagSCHistoricalProofBurst       = "state-commit.sc-historical-proof-burst"
 	FlagSCWriteMode                  = "state-commit.sc-write-mode"
+	// Per-block batch size used by the MigrationManager when sc-write-mode
+	// is one of the in-flight modes (migrate_evm, migrate_bank,
+	// migrate_all_but_bank). Optional: when unset in app.toml the field
+	// stays at DefaultStateCommitConfig().KeysToMigratePerBlock (= 1024),
+	// which is appropriate for production drains. Lowering it spreads the
+	// migration across more blocks, which is useful for tests that need to
+	// exercise the resume / hybrid-read path mid-flight.
+	FlagSCKeysToMigratePerBlock = "state-commit.sc-keys-to-migrate-per-block"
 
 	// SS Store configs
 	FlagSSEnable            = "state-store.ss-enable"
@@ -119,6 +127,16 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig {
 	if v := appOpts.Get(FlagSCHistoricalProofBurst); v != nil {
 		scConfig.HistoricalProofBurst = cast.ToInt(v)
 	}
+	// Guard with v != nil so that an absent app.toml entry preserves the
+	// default of 1024 instead of clobbering it to 0, which would fail
+	// StateCommitConfig.Validate ("keys-to-migrate-per-block must be > 0")
+	// and bring the node down at startup the first time write-mode is
+	// flipped to a migration mode.
+	if v := appOpts.Get(FlagSCKeysToMigratePerBlock); v != nil {
+		if n := cast.ToInt(v); n > 0 {
+			scConfig.KeysToMigratePerBlock = n
+		}
+	}
 
 	return scConfig
 }

docker/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     volumes:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -54,6 +55,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     volumes:
       - "${PROJECT_HOME}:/sei-protocol/sei-chain:Z"
       - "${PROJECT_HOME}/../sei-tendermint:/sei-protocol/sei-tendermint:Z"
@@ -83,6 +85,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     ports:
       - "26662-26664:26656-26658"
       - "9094-9095:9090-9091"
@@ -116,6 +119,7 @@ services:
       - RECEIPT_BACKEND
       - AUTOBAHN
       - GIGA_STORAGE
+      - GIGA_MIGRATE_FROM_MEMIAVL
     ports:
       - "26665-26667:26656-26658"
       - "9096-9097:9090-9091"

docker/localnode/config/app.toml

@@ -236,6 +236,13 @@ sc-snapshot-writer-limit = 2
 # CacheSize defines the size of the LRU cache for each store on top of the tree, default to 100000.
 sc-cache-size = 1000
 
+# KeysToMigratePerBlock controls how many EVM keys the in-flight migration
+# (sc-write-mode = migrate_evm / migrate_bank / migrate_all_but_bank) drains
+# from memiavl into flatkv per block. Default 1024 is appropriate for
+# production drains; tests lower it to spread the migration across more
+# blocks and exercise the resume / hybrid-read path.
+sc-keys-to-migrate-per-block = 1024
+
 [state-store]
 
 # Enable defines if the state-store should be enabled for historical queries.

docker/localnode/scripts/step4_config_override.sh

@@ -5,6 +5,13 @@ GIGA_EXECUTOR=${GIGA_EXECUTOR:-false}
 GIGA_OCC=${GIGA_OCC:-false}
 AUTOBAHN=${AUTOBAHN:-false}
 GIGA_STORAGE=${GIGA_STORAGE:-false}
+# GIGA_MIGRATE_FROM_MEMIAVL=true boots the cluster in v0 (memiavl_only):
+# memiavl is the sole SC backend, FlatKV is not allocated. This is the
+# starting point for the FlatKV EVM migrate cluster test, which drives a
+# real workload in this mode and then performs a coordinated stop/flip/
+# restart into migrate_evm. Mutually exclusive with GIGA_STORAGE=true;
+# the script picks the more specific override if both are set.
+GIGA_MIGRATE_FROM_MEMIAVL=${GIGA_MIGRATE_FROM_MEMIAVL:-false}
 
 APP_CONFIG_FILE="build/generated/node_$NODE_ID/app.toml"
 TENDERMINT_CONFIG_FILE="build/generated/node_$NODE_ID/config.toml"
@@ -23,11 +30,31 @@ sed -i.bak -e "s|^snapshot-directory *=.*|snapshot-directory = \"./build/generat
 # Enable slow mode
 sed -i.bak -e 's/slow = .*/slow = true/' ~/.sei/config/app.toml
 
+# Boot the cluster in v0 (memiavl_only) for the FlatKV EVM migrate test.
+# Doing this here keeps the override surface narrow: the test runner
+# only has to set one env var to ship a v0-shaped config, and the
+# follow-up flip script just rewrites sc-write-mode in place during the
+# coordinated stop.
+if [ "$GIGA_MIGRATE_FROM_MEMIAVL" = "true" ]; then
+  echo "Booting node $NODE_ID in memiavl_only mode (FlatKV EVM migrate starting point)..."
+  if grep -q '^sc-write-mode[[:space:]]*=' ~/.sei/config/app.toml; then
+    sed -i 's/^sc-write-mode[[:space:]]*=.*/sc-write-mode = "memiavl_only"/' ~/.sei/config/app.toml
+  else
+    sed -i '/^\[state-store\]/i sc-write-mode = "memiavl_only"' ~/.sei/config/app.toml
+  fi
+  # The EVM SS split is irrelevant in this mode (flatkv is not allocated),
+  # but explicitly disabling it keeps app.toml self-describing in case an
+  # operator inspects it post-flip.
+  sed -i 's/^evm-ss-split[[:space:]]*=.*/evm-ss-split = false/' ~/.sei/config/app.toml
+fi
+
 # Enable Giga Storage: FlatKV SC dual-write + EVM SS split.
 # When GIGA_STORAGE=true we also default the receipt backend to parquet; callers
 # can still override this by setting RECEIPT_BACKEND explicitly.
 # Set GIGA_STORAGE=false to disable.
-if [ "$GIGA_STORAGE" = "true" ]; then
+# GIGA_MIGRATE_FROM_MEMIAVL takes precedence: if both are set, the memiavl-only
+# block above ran first and the test runner is responsible for the migration.
+if [ "$GIGA_STORAGE" = "true" ] && [ "$GIGA_MIGRATE_FROM_MEMIAVL" != "true" ]; then
   RECEIPT_BACKEND=${RECEIPT_BACKEND:-parquet}
   echo "Enabling Giga Storage for node $NODE_ID..."
 
@@ -36,14 +63,14 @@ if [ "$GIGA_STORAGE" = "true" ]; then
   # from the memiavl tree via GetChildStoreByName. dual-write keeps memiavl
   # up-to-date for reads while also populating FlatKV. This mode is for test
   # clusters only — never deploy to testnet/mainnet.
-  if grep -q "sc-write-mode" ~/.sei/config/app.toml; then
-    sed -i 's/sc-write-mode = .*/sc-write-mode = "test_only_dual_write"/' ~/.sei/config/app.toml
+  if grep -q '^sc-write-mode[[:space:]]*=' ~/.sei/config/app.toml; then
+    sed -i 's/^sc-write-mode[[:space:]]*=.*/sc-write-mode = "test_only_dual_write"/' ~/.sei/config/app.toml
   else
     sed -i '/^\[state-store\]/i sc-write-mode = "test_only_dual_write"' ~/.sei/config/app.toml
   fi
 
   # --- SS layer: enable EVM split ---
-  sed -i 's/evm-ss-split = .*/evm-ss-split = true/' ~/.sei/config/app.toml
+  sed -i 's/^evm-ss-split[[:space:]]*=.*/evm-ss-split = true/' ~/.sei/config/app.toml
 fi
 
 # Enable Giga Executor if requested

integration_test/contracts/deploy_flatkv_evm_fixture.sh

@@ -11,6 +11,8 @@ CHAIN_ID=${FLATKV_EVM_FIXTURE_CHAIN_ID:-sei}
 RECIPIENT_ADDR=${FLATKV_EVM_FIXTURE_RECIPIENT:-0x70997970C51812dc3A010C7d01b50e0d17dc79C8}
 MISSING_ADDR=${FLATKV_EVM_FIXTURE_MISSING:-0xc1cadaffffffffffffffffffffffffffffffffff}
 TRANSFER_VALUE_WEI=${FLATKV_EVM_FIXTURE_TRANSFER_VALUE_WEI:-1}
+BULK_STORAGE_KEYS=${FLATKV_EVM_BULK_STORAGE_KEYS:-4000}
+BULK_STORAGE_KEYS_PER_CONTRACT=${FLATKV_EVM_BULK_STORAGE_KEYS_PER_CONTRACT:-50}
 KEYRING_ARGS=()
 if [ -n "${FLATKV_EVM_FIXTURE_KEYRING_BACKEND:-}" ]; then
   KEYRING_ARGS+=(--keyring-backend "$FLATKV_EVM_FIXTURE_KEYRING_BACKEND")
@@ -106,6 +108,39 @@ require_success_receipt() {
   fi
 }
 
+write_bulk_storage_contract() {
+  local output=$1
+  local start_slot=$2
+  local count=$3
+
+  python3 - "$output" "$start_slot" "$count" <<'PY'
+import sys
+
+output = sys.argv[1]
+start_slot = int(sys.argv[2])
+count = int(sys.argv[3])
+
+if start_slot < 0 or count < 0 or start_slot + count > 65536:
+    raise SystemExit(f"slot range [{start_slot}, {start_slot + count}) is outside PUSH2 range")
+
+code = bytearray()
+for slot in range(start_slot, start_slot + count):
+    # sstore(slot, slot + 1). Use fixed-width PUSH32/PUSH2 so the emitted
+    # constructor is deterministic and does not need an assembler dependency.
+    code.append(0x7F)  # PUSH32
+    code.extend((slot + 1).to_bytes(32, "big"))
+    code.append(0x61)  # PUSH2
+    code.extend(slot.to_bytes(2, "big"))
+    code.append(0x55)  # SSTORE
+
+# Return empty runtime; the test only needs persisted storage rows.
+code.extend(bytes.fromhex("60006000f3"))
+
+with open(output, "w", encoding="utf-8") as fh:
+    fh.write(code.hex())
+PY
+}
+
 echo "Generating FlatKV EVM historical fixture via $RPC_URL..."
 wait_for_evm_rpc
 
@@ -198,9 +233,54 @@ missing_storage_expected=$(query_storage "$MISSING_ADDR" "$STORAGE_SLOT_ZERO" "$
 write_fixture "flatkv_evm_missing_balance_expected.txt" "$missing_balance_expected"
 write_fixture "flatkv_evm_missing_storage_expected.txt" "$missing_storage_expected"
 
+write_fixture "flatkv_evm_bulk_storage_keys.txt" "$BULK_STORAGE_KEYS"
+if [ "$BULK_STORAGE_KEYS" -gt 0 ]; then
+  if [ "$BULK_STORAGE_KEYS_PER_CONTRACT" -le 0 ]; then
+    echo "FLATKV_EVM_BULK_STORAGE_KEYS_PER_CONTRACT must be positive" >&2
+    exit 1
+  fi
+
+  echo "Deploying bulk storage fixture: total_slots=$BULK_STORAGE_KEYS slots_per_contract=$BULK_STORAGE_KEYS_PER_CONTRACT"
+  deployed_bulk=0
+  while [ "$deployed_bulk" -lt "$BULK_STORAGE_KEYS" ]; do
+    remaining=$((BULK_STORAGE_KEYS - deployed_bulk))
+    batch_size=$BULK_STORAGE_KEYS_PER_CONTRACT
+    if [ "$remaining" -lt "$batch_size" ]; then
+      batch_size=$remaining
+    fi
+
+    bulk_contract_hex_file="/tmp/flatkv_evm_bulk_storage_${deployed_bulk}.hex"
+    write_bulk_storage_contract "$bulk_contract_hex_file" "$deployed_bulk" "$batch_size"
+
+    if ! bulk_out=$(run_seid tx evm deploy "$bulk_contract_hex_file" \
+      --from "$FROM" \
+      "${KEYRING_ARGS[@]}" \
+      --chain-id "$CHAIN_ID" \
+      --evm-rpc "$RPC_URL" \
+      -b sync \
+      -y 2>&1); then
+      echo "FlatKV EVM bulk storage deploy command failed at slot offset $deployed_bulk:" >&2
+      printf "%s\n" "$bulk_out" >&2
+      exit 1
+    fi
+    bulk_tx=$(printf "%s\n" "$bulk_out" | extract_tx_hash || true)
+    if [ -z "$bulk_tx" ]; then
+      echo "Failed to extract FlatKV EVM bulk storage tx hash at slot offset $deployed_bulk:" >&2
+      printf "%s\n" "$bulk_out" >&2
+      exit 1
+    fi
+    bulk_receipt=$(wait_for_receipt "$bulk_tx" 120)
+    require_success_receipt "bulk storage deployment" "$bulk_receipt"
+
+    deployed_bulk=$((deployed_bulk + batch_size))
+    echo "  bulk storage slots committed: $deployed_bulk/$BULK_STORAGE_KEYS"
+  done
+fi
+
 latest_height=$(block_number)
 write_fixture "flatkv_evm_latest_fixture_block_height.txt" "$latest_height"
 
 echo "FlatKV EVM fixture generated:"
 echo "  recipient=$RECIPIENT_ADDR balance_height=$balance_height balance=$balance_expected"
 echo "  contract=$contract_addr contract_height=$contract_height storage=$storage_expected"
+echo "  bulk_storage_keys=$BULK_STORAGE_KEYS"