elastic scaling RFC 103 end-to-end tests (#6452)

Adds two new zombienet-sdk tests:

- one which verifies that elastic scaling works correctly both with the
MVP and the new RFC 103 implementation which sends the core selector as
a UMP signal.
- one which spawns a parachain with two collators, but only one of them
is using v2 receipts. The parachain will still make progress but without
full throughput

Also enables the V2 receipts node feature for testnet genesis config.

Part of #5049

.gitlab/pipeline/zombienet/polkadot.yml

@@ -190,6 +190,7 @@ zombienet-polkadot-elastic-scaling-0002-elastic-scaling-doesnt-break-parachains:
       --local-dir="${LOCAL_DIR}/elastic_scaling"
       --test="0002-elastic-scaling-doesnt-break-parachains.zndsl"
 
+
 .zombienet-polkadot-functional-0012-spam-statement-distribution-requests:
   extends:
     - .zombienet-polkadot-common
@@ -386,3 +387,35 @@ zombienet-polkadot-malus-0001-dispute-valid:
     - unset NEXTEST_FAILURE_OUTPUT
     - unset NEXTEST_SUCCESS_OUTPUT
     - cargo nextest run --archive-file ./artifacts/polkadot-zombienet-tests.tar.zst --no-capture -- smoke::coretime_revenue::coretime_revenue_test
+
+zombienet-polkadot-elastic-scaling-slot-based-3cores:
+  extends:
+    - .zombienet-polkadot-common
+  needs:
+    - job: build-polkadot-zombienet-tests
+      artifacts: true
+  before_script:
+    - !reference [ ".zombienet-polkadot-common", "before_script" ]
+    - export POLKADOT_IMAGE="${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+    - export CUMULUS_IMAGE="docker.io/paritypr/test-parachain:${PIPELINE_IMAGE_TAG}"
+  script:
+    # we want to use `--no-capture` in zombienet tests.
+    - unset NEXTEST_FAILURE_OUTPUT
+    - unset NEXTEST_SUCCESS_OUTPUT
+    - cargo nextest run --archive-file ./artifacts/polkadot-zombienet-tests.tar.zst --no-capture -- elastic_scaling::slot_based_3cores::slot_based_3cores_test
+
+zombienet-polkadot-elastic-scaling-mixed-receipt-versions:
+  extends:
+    - .zombienet-polkadot-common
+  needs:
+    - job: build-polkadot-zombienet-tests
+      artifacts: true
+  before_script:
+    - !reference [ ".zombienet-polkadot-common", "before_script" ]
+    - export POLKADOT_IMAGE="${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+    - export CUMULUS_IMAGE="docker.io/paritypr/test-parachain:${PIPELINE_IMAGE_TAG}"
+  script:
+    # we want to use `--no-capture` in zombienet tests.
+    - unset NEXTEST_FAILURE_OUTPUT
+    - unset NEXTEST_SUCCESS_OUTPUT
+    - cargo nextest run --archive-file ./artifacts/polkadot-zombienet-tests.tar.zst --no-capture -- elastic_scaling::mixed_receipt_versions::mixed_receipt_versions_test

cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml

@@ -51,7 +51,7 @@ polkadot-runtime-common = { workspace = true }
 # Cumulus
 cumulus-pallet-aura-ext = { workspace = true }
 pallet-message-queue = { workspace = true }
-cumulus-pallet-parachain-system = { workspace = true, features = ["experimental-ump-signals"] }
+cumulus-pallet-parachain-system = { workspace = true }
 cumulus-pallet-xcm = { workspace = true }
 cumulus-pallet-xcmp-queue = { workspace = true }
 cumulus-ping = { workspace = true }

cumulus/test/runtime/Cargo.toml

@@ -96,3 +96,4 @@ std = [
 ]
 increment-spec-version = []
 elastic-scaling = []
+experimental-ump-signals = ["cumulus-pallet-parachain-system/experimental-ump-signals"]

cumulus/test/runtime/build.rs

@@ -29,6 +29,14 @@ fn main() {
 		.with_current_project()
 		.enable_feature("elastic-scaling")
 		.import_memory()
+		.set_file_name("wasm_binary_elastic_scaling_mvp.rs")
+		.build();
+
+	WasmBuilder::new()
+		.with_current_project()
+		.enable_feature("elastic-scaling")
+		.enable_feature("experimental-ump-signals")
+		.import_memory()
 		.set_file_name("wasm_binary_elastic_scaling.rs")
 		.build();
 }

cumulus/test/runtime/src/lib.rs

@@ -27,6 +27,11 @@ pub mod wasm_spec_version_incremented {
 	include!(concat!(env!("OUT_DIR"), "/wasm_binary_spec_version_incremented.rs"));
 }
 
+pub mod elastic_scaling_mvp {
+	#[cfg(feature = "std")]
+	include!(concat!(env!("OUT_DIR"), "/wasm_binary_elastic_scaling_mvp.rs"));
+}
+
 pub mod elastic_scaling {
 	#[cfg(feature = "std")]
 	include!(concat!(env!("OUT_DIR"), "/wasm_binary_elastic_scaling.rs"));

cumulus/test/service/src/chain_spec.rs

@@ -116,3 +116,13 @@ pub fn get_elastic_scaling_chain_spec(id: Option<ParaId>) -> ChainSpec {
 			.expect("WASM binary was not built, please build it!"),
 	)
 }
+
+/// Get the chain spec for a specific parachain ID.
+pub fn get_elastic_scaling_mvp_chain_spec(id: Option<ParaId>) -> ChainSpec {
+	get_chain_spec_with_extra_endowed(
+		id,
+		Default::default(),
+		cumulus_test_runtime::elastic_scaling_mvp::WASM_BINARY
+			.expect("WASM binary was not built, please build it!"),
+	)
+}

cumulus/test/service/src/cli.rs

@@ -262,10 +262,16 @@ impl SubstrateCli for TestCollatorCli {
 				tracing::info!("Using default test service chain spec.");
 				Box::new(cumulus_test_service::get_chain_spec(Some(ParaId::from(2000)))) as Box<_>
 			},
+			"elastic-scaling-mvp" => {
+				tracing::info!("Using elastic-scaling mvp chain spec.");
+				Box::new(cumulus_test_service::get_elastic_scaling_mvp_chain_spec(Some(
+					ParaId::from(2100),
+				))) as Box<_>
+			},
 			"elastic-scaling" => {
 				tracing::info!("Using elastic-scaling chain spec.");
 				Box::new(cumulus_test_service::get_elastic_scaling_chain_spec(Some(ParaId::from(
-					2100,
+					2200,
 				)))) as Box<_>
 			},
 			path => {

polkadot/runtime/rococo/src/genesis_config_presets.rs

@@ -129,7 +129,9 @@ fn default_parachains_host_configuration(
 			allowed_ancestry_len: 2,
 		},
 		node_features: bitvec::vec::BitVec::from_element(
-			1u8 << (FeatureIndex::ElasticScalingMVP as usize),
+			1u8 << (FeatureIndex::ElasticScalingMVP as usize) |
+				1u8 << (FeatureIndex::EnableAssignmentsV2 as usize) |
+				1u8 << (FeatureIndex::CandidateReceiptV2 as usize),
 		),
 		scheduler_params: SchedulerParams {
 			lookahead: 2,

polkadot/runtime/westend/src/genesis_config_presets.rs

@@ -133,7 +133,8 @@ fn default_parachains_host_configuration(
 		},
 		node_features: bitvec::vec::BitVec::from_element(
 			1u8 << (FeatureIndex::ElasticScalingMVP as usize) |
-				1u8 << (FeatureIndex::EnableAssignmentsV2 as usize),
+				1u8 << (FeatureIndex::EnableAssignmentsV2 as usize) |
+				1u8 << (FeatureIndex::CandidateReceiptV2 as usize),
 		),
 		scheduler_params: SchedulerParams {
 			lookahead: 2,

polkadot/zombienet-sdk-tests/build.rs

@@ -25,39 +25,47 @@ fn make_env_key(k: &str) -> String {
 	replace_dashes(&k.to_ascii_uppercase())
 }
 
+fn wasm_sub_path(chain: &str) -> String {
+	let (package, runtime_name) =
+		if let Some(cumulus_test_runtime) = chain.strip_prefix("cumulus-test-runtime-") {
+			(
+				"cumulus-test-runtime".to_string(),
+				format!("wasm_binary_{}.rs", replace_dashes(cumulus_test_runtime)),
+			)
+		} else {
+			(format!("{chain}-runtime"), replace_dashes(&format!("{chain}-runtime")))
+		};
+
+	format!("{}/{}.wasm", package, runtime_name)
+}
+
 fn find_wasm(chain: &str) -> Option<PathBuf> {
 	const PROFILES: [&str; 2] = ["release", "testnet"];
 	let manifest_path = env::var("CARGO_WORKSPACE_ROOT_DIR").unwrap();
 	let manifest_path = manifest_path.strip_suffix('/').unwrap();
 	debug_output!("manifest_path is  : {}", manifest_path);
-	let package = format!("{chain}-runtime");
+
+	let sub_path = wasm_sub_path(chain);
+
 	let profile = PROFILES.into_iter().find(|p| {
-		let full_path = format!(
-			"{}/target/{}/wbuild/{}/{}.wasm",
-			manifest_path,
-			p,
-			&package,
-			replace_dashes(&package)
-		);
+		let full_path = format!("{}/target/{}/wbuild/{}", manifest_path, p, sub_path);
 		debug_output!("checking wasm at : {}", full_path);
 		matches!(path::PathBuf::from(&full_path).try_exists(), Ok(true))
 	});
 
 	debug_output!("profile is : {:?}", profile);
 	profile.map(|profile| {
-		PathBuf::from(&format!(
-			"{}/target/{}/wbuild/{}/{}.wasm",
-			manifest_path,
-			profile,
-			&package,
-			replace_dashes(&package)
-		))
+		PathBuf::from(&format!("{}/target/{}/wbuild/{}", manifest_path, profile, sub_path))
 	})
 }
 
 // based on https://gist.github.com/s0me0ne-unkn0wn/bbd83fe32ce10327086adbf13e750eec
 fn build_wasm(chain: &str) -> PathBuf {
-	let package = format!("{chain}-runtime");
+	let package = if chain.starts_with("cumulus-test-runtime-") {
+		String::from("cumulus-test-runtime")
+	} else {
+		format!("{chain}-runtime")
+	};
 
 	let cargo = env::var("CARGO").unwrap();
 	let target = env::var("TARGET").unwrap();
@@ -81,11 +89,7 @@ fn build_wasm(chain: &str) -> PathBuf {
 		.status()
 		.unwrap();
 
-	let wasm_path = &format!(
-		"{target_dir}/{target}/release/wbuild/{}/{}.wasm",
-		&package,
-		replace_dashes(&package)
-	);
+	let wasm_path = &format!("{target_dir}/{target}/release/wbuild/{}", wasm_sub_path(chain));
 	PathBuf::from(wasm_path)
 }
 
@@ -128,6 +132,10 @@ fn main() {
 	const METADATA_DIR: &str = "metadata-files";
 	const CHAINS: [&str; 2] = ["rococo", "coretime-rococo"];
 
+	// Add some cumulus test runtimes if needed. Formatted like
+	// "cumulus-test-runtime-elastic-scaling".
+	const CUMULUS_TEST_RUNTIMES: [&str; 0] = [];
+
 	let metadata_path = format!("{manifest_path}/{METADATA_DIR}");
 
 	for chain in CHAINS {
@@ -145,6 +153,21 @@ fn main() {
 		};
 	}
 
+	for chain in CUMULUS_TEST_RUNTIMES {
+		let full_path = format!("{metadata_path}/{chain}-local.scale");
+		let output_path = path::PathBuf::from(&full_path);
+
+		match output_path.try_exists() {
+			Ok(true) => {
+				debug_output!("got: {}", full_path);
+			},
+			_ => {
+				debug_output!("needs: {}", full_path);
+				fetch_metadata_file(chain, &output_path);
+			},
+		};
+	}
+
 	substrate_build_script_utils::generate_cargo_keys();
 	substrate_build_script_utils::rerun_if_git_head_changed();
 	println!("cargo:rerun-if-changed={}", metadata_path);

polkadot/zombienet-sdk-tests/tests/elastic_scaling/helpers.rs

@@ -0,0 +1,60 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::rococo;
+use std::{collections::HashMap, ops::Range};
+use subxt::{OnlineClient, PolkadotConfig};
+
+// Helper function for asserting the throughput of parachains (total number of backed candidates in
+// a window of relay chain blocks), after the first session change.
+pub async fn assert_para_throughput(
+	relay_client: &OnlineClient<PolkadotConfig>,
+	stop_at: u32,
+	expected_candidate_ranges: HashMap<u32, Range<u32>>,
+) -> Result<(), anyhow::Error> {
+	let mut blocks_sub = relay_client.blocks().subscribe_finalized().await?;
+	let mut candidate_count: HashMap<u32, u32> = HashMap::new();
+	let mut current_block_count = 0;
+	let mut had_first_session_change = false;
+
+	while let Some(block) = blocks_sub.next().await {
+		let block = block?;
+		log::debug!("Finalized relay chain block {}", block.number());
+		let events = block.events().await?;
+		let is_session_change = events.has::<rococo::session::events::NewSession>()?;
+
+		if !had_first_session_change && is_session_change {
+			had_first_session_change = true;
+		}
+
+		if had_first_session_change && !is_session_change {
+			current_block_count += 1;
+
+			for event in events.find::<rococo::para_inclusion::events::CandidateBacked>() {
+				*(candidate_count.entry(event?.0.descriptor.para_id.0).or_default()) += 1;
+			}
+		}
+
+		if current_block_count == stop_at {
+			break;
+		}
+	}
+
+	log::info!(
+		"Reached {} finalized relay chain blocks that contain backed candidates. The per-parachain distribution is: {:#?}",
+		stop_at,
+		candidate_count
+	);
+
+	for (para_id, expected_candidate_range) in expected_candidate_ranges {
+		let actual = candidate_count
+			.get(&para_id)
+			.expect("ParaId did not have any backed candidates");
+		assert!(
+			expected_candidate_range.contains(actual),
+			"Candidate count {actual} not within range {expected_candidate_range:?}"
+		);
+	}
+
+	Ok(())
+}