feat: 🔀 Merge upstream into wasmtime-precompile PR

cumulus/test/service/src/lib.rs

@@ -893,6 +893,7 @@ pub fn node_config(
 		data_path: root,
 		base_path,
 		wasm_runtime_overrides: None,
+		wasmtime_precompiled: None,
 	})
 }
 

polkadot/node/core/pvf/common/src/executor_interface.rs

@@ -191,7 +191,7 @@ pub fn prepare(
 	executor_params: &ExecutorParams,
 ) -> Result<Vec<u8>, sc_executor_common::error::WasmError> {
 	let (semantics, _) = params_to_wasmtime_semantics(executor_params);
-	sc_executor_wasmtime::prepare_runtime_artifact(blob, &semantics)
+	sc_executor_wasmtime::prepare_runtime_artifact(blob, Default::default(), &semantics)
 }
 
 /// Available host functions. We leave out:

polkadot/node/test/service/src/lib.rs

@@ -208,6 +208,7 @@ pub fn node_config(
 			..ExecutorConfiguration::default()
 		},
 		wasm_runtime_overrides: Default::default(),
+		wasmtime_precompiled: Default::default(),
 		rpc: RpcConfiguration {
 			addr: Default::default(),
 			max_request_size: Default::default(),

substrate/bin/node/cli/benches/block_production.rs

@@ -108,6 +108,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		announce_block: true,
 		data_path: base_path.path().into(),
 		base_path,
+		wasmtime_precompiled: None,
 		wasm_runtime_overrides: None,
 	};
 

substrate/bin/node/cli/benches/transaction_pool.rs

@@ -94,6 +94,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		announce_block: true,
 		data_path: base_path.path().into(),
 		base_path,
+		wasmtime_precompiled: None,
 		wasm_runtime_overrides: None,
 	};
 

substrate/bin/node/cli/src/cli.rs

@@ -100,4 +100,7 @@ pub enum Subcommand {
 
 	/// Db meta columns information.
 	ChainInfo(sc_cli::ChainInfoCmd),
+
+	/// Precompile the WASM runtime into native code
+	PrecompileWasm(sc_cli::PrecompileWasmCmd),
 }

substrate/bin/node/cli/src/command.rs

@@ -228,5 +228,13 @@ pub fn run() -> Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.sync_run(|config| cmd.run::<Block>(&config))
 		},
+		Some(Subcommand::PrecompileWasm(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { task_manager, backend, .. } =
+					service::new_partial(&config, None)?;
+				Ok((cmd.run(backend, config.chain_spec), task_manager))
+			})
+		},
 	}
 }

substrate/client/chain-spec/src/genesis_block.rs

@@ -23,7 +23,11 @@ use std::{marker::PhantomData, sync::Arc};
 use codec::Encode;
 use sc_client_api::{backend::Backend, BlockImportOperation};
 use sc_executor::RuntimeVersionOf;
-use sp_core::storage::{well_known_keys, StateVersion, Storage};
+use sp_core::{
+	storage::{well_known_keys, StateVersion, Storage},
+	Blake2Hasher,
+};
+
 use sp_runtime::{
 	traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, Zero},
 	BuildStorage,

substrate/client/cli/Cargo.toml

@@ -37,6 +37,7 @@ bip39 = { package = "parity-bip39", version = "2.0.1", features = ["rand"] }
 tokio = { features = ["parking_lot", "rt-multi-thread", "signal"], workspace = true, default-features = true }
 sc-client-api = { workspace = true, default-features = true }
 sc-client-db = { workspace = true }
+sc-executor = { workspace = true, default-features = true }
 sc-keystore = { workspace = true, default-features = true }
 sc-mixnet = { workspace = true, default-features = true }
 sc-network = { workspace = true, default-features = true }
@@ -51,6 +52,8 @@ sp-keyring = { workspace = true, default-features = true }
 sp-keystore = { workspace = true, default-features = true }
 sp-panic-handler = { workspace = true, default-features = true }
 sp-runtime = { workspace = true, default-features = true }
+sp-state-machine = { workspace = true, default-features = true }
+sp-storage = { workspace = true, default-features = true }
 sp-version = { workspace = true, default-features = true }
 
 [dev-dependencies]

substrate/client/cli/src/commands/mod.rs

@@ -30,6 +30,7 @@ mod insert_key;
 mod inspect_key;
 mod inspect_node_key;
 mod key;
+mod precompile_wasm_cmd;
 mod purge_chain_cmd;
 mod revert_cmd;
 mod run_cmd;
@@ -44,6 +45,6 @@ pub use self::{
 	export_blocks_cmd::ExportBlocksCmd, export_state_cmd::ExportStateCmd, generate::GenerateCmd,
 	generate_node_key::GenerateKeyCmdCommon, import_blocks_cmd::ImportBlocksCmd,
 	insert_key::InsertKeyCmd, inspect_key::InspectKeyCmd, inspect_node_key::InspectNodeKeyCmd,
-	key::KeySubcommand, purge_chain_cmd::PurgeChainCmd, revert_cmd::RevertCmd, run_cmd::RunCmd,
-	sign::SignCmd, vanity::VanityCmd, verify::VerifyCmd,
+	key::KeySubcommand, precompile_wasm_cmd::PrecompileWasmCmd, purge_chain_cmd::PurgeChainCmd,
+	revert_cmd::RevertCmd, run_cmd::RunCmd, sign::SignCmd, vanity::VanityCmd, verify::VerifyCmd,
 };

substrate/client/cli/src/commands/precompile_wasm_cmd.rs

@@ -0,0 +1,156 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+	arg_enums::{
+		execution_method_from_cli, WasmExecutionMethod, WasmtimeInstantiationStrategy,
+		DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
+	},
+	error::{self, Error},
+	params::{DatabaseParams, PruningParams, SharedParams},
+	CliConfiguration,
+};
+
+use clap::Parser;
+use sc_client_api::{Backend, HeaderBackend};
+use sc_executor::{
+	precompile_and_serialize_versioned_wasm_runtime, HeapAllocStrategy, DEFAULT_HEAP_ALLOC_PAGES,
+};
+use sc_service::ChainSpec;
+use sp_core::traits::RuntimeCode;
+use sp_runtime::traits::{Block as BlockT, Hash, Header};
+use sp_state_machine::backend::BackendRuntimeCode;
+use std::{fmt::Debug, path::PathBuf, sync::Arc};
+
+/// The `precompile-wasm` command used to serialize a precompiled WASM module.
+///
+/// The WASM code precompiled will be the one used at the latest finalized block
+/// this node is aware of, if this node has the state for that finalized block in
+/// its storage. If that's not the case, it will use the WASM code from the chain
+/// spec passed as parameter when running the node.
+#[derive(Debug, Parser)]
+pub struct PrecompileWasmCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub database_params: DatabaseParams,
+
+	/// The default number of 64KB pages to ever allocate for Wasm execution.
+	///  
+	/// Don't alter this unless you know what you're doing.
+	#[arg(long, value_name = "COUNT")]
+	pub default_heap_pages: Option<u32>,
+
+	/// Path to the directory where precompiled artifact will be written.
+	#[arg()]
+	pub output_dir: PathBuf,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub pruning_params: PruningParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	/// The WASM instantiation method to use.
+	///
+	/// Only has an effect when `wasm-execution` is set to `compiled`.
+	/// The copy-on-write strategies are only supported on Linux.
+	/// If the copy-on-write variant of a strategy is unsupported
+	/// the executor will fall back to the non-CoW equivalent.
+	/// The fastest (and the default) strategy available is `pooling-copy-on-write`.
+	/// The `legacy-instance-reuse` strategy is deprecated and will
+	/// be removed in the future. It should only be used in case of
+	/// issues with the default instantiation strategy.
+	#[arg(
+		long,
+		value_name = "STRATEGY",
+		default_value_t = DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
+		value_enum,
+	)]
+	pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy,
+}
+
+impl PrecompileWasmCmd {
+	/// Run the precompile-wasm command
+	pub async fn run<B, BA>(&self, backend: Arc<BA>, spec: Box<dyn ChainSpec>) -> error::Result<()>
+	where
+		B: BlockT,
+		BA: Backend<B>,
+	{
+		let heap_pages = self.default_heap_pages.unwrap_or(DEFAULT_HEAP_ALLOC_PAGES);
+
+		let blockchain_info = backend.blockchain().info();
+
+		if backend.have_state_at(blockchain_info.finalized_hash, blockchain_info.finalized_number) {
+			let state = backend.state_at(backend.blockchain().info().finalized_hash)?;
+
+			precompile_and_serialize_versioned_wasm_runtime(
+				HeapAllocStrategy::Static { extra_pages: heap_pages },
+				&BackendRuntimeCode::new(&state).runtime_code()?,
+				execution_method_from_cli(
+					WasmExecutionMethod::Compiled,
+					self.wasmtime_instantiation_strategy,
+				),
+				&self.output_dir,
+			)
+			.map_err(|e| Error::Application(Box::new(e)))?;
+		} else {
+			let storage = spec.as_storage_builder().build_storage()?;
+			if let Some(wasm_bytecode) = storage.top.get(sp_storage::well_known_keys::CODE) {
+				let runtime_code = RuntimeCode {
+					code_fetcher: &sp_core::traits::WrappedRuntimeCode(
+						wasm_bytecode.as_slice().into(),
+					),
+					hash: <<B::Header as Header>::Hashing as Hash>::hash(&wasm_bytecode)
+						.as_ref()
+						.to_vec(),
+					heap_pages: Some(heap_pages as u64),
+				};
+				precompile_and_serialize_versioned_wasm_runtime(
+					HeapAllocStrategy::Static { extra_pages: heap_pages },
+					&runtime_code,
+					execution_method_from_cli(
+						WasmExecutionMethod::Compiled,
+						self.wasmtime_instantiation_strategy,
+					),
+					&self.output_dir,
+				)
+				.map_err(|e| Error::Application(Box::new(e)))?;
+			} else {
+				return Err(Error::Input(format!("The chain spec used does not contain a wasm bytecode in the `:code` storage key")));
+			}
+		}
+
+		Ok(())
+	}
+}
+
+impl CliConfiguration for PrecompileWasmCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+
+	fn pruning_params(&self) -> Option<&PruningParams> {
+		Some(&self.pruning_params)
+	}
+
+	fn database_params(&self) -> Option<&DatabaseParams> {
+		Some(&self.database_params)
+	}
+}

substrate/client/cli/src/config.rs

@@ -294,6 +294,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		Ok(self.import_params().map(|x| x.wasm_method()).unwrap_or_default())
 	}
 
+	/// Get the path where WASM precompiled artifacts live.
+	///
+	/// By default this is `None`.
+	fn wasmtime_precompiled(&self) -> Option<PathBuf> {
+		self.import_params().map(|x| x.wasmtime_precompiled()).unwrap_or_default()
+	}
+
 	/// Get the path where WASM overrides live.
 	///
 	/// By default this is `None`.
@@ -532,6 +539,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			blocks_pruning: self.blocks_pruning()?,
 			executor: ExecutorConfiguration {
 				wasm_method: self.wasm_method()?,
+				wasmtime_precompiled: self.wasmtime_precompiled(),
 				default_heap_pages: self.default_heap_pages()?,
 				max_runtime_instances,
 				runtime_cache_size,

substrate/client/cli/src/params/import_params.rs

@@ -65,10 +65,28 @@ pub struct ImportParams {
 	)]
 	pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy,
 
+	/// Specify the path where local precompiled WASM runtimes are stored.
+	/// Only has an effect when `wasm-execution` is set to `compiled`.
+	///
+	/// The precompiled runtimes must have been generated using the `precompile-wasm`
+	/// subcommand with the same version of wasmtime and the exact same configuration.
+	/// The file name must end with the hash of the configuration. This hash must match, otherwise
+	/// the runtime will be recompiled.
+	#[arg(
+        long,
+        value_name = "PATH",
+        value_parser = ImportParams::validate_existing_directory,
+    )]
+	pub wasmtime_precompiled: Option<PathBuf>,
+
 	/// Specify the path where local WASM runtimes are stored.
 	///
 	/// These runtimes will override on-chain runtimes when the version matches.
-	#[arg(long, value_name = "PATH")]
+	#[arg(
+        long,
+        value_name = "PATH",
+        value_parser = ImportParams::validate_existing_directory,
+    )]
 	pub wasm_runtime_overrides: Option<PathBuf>,
 
 	#[allow(missing_docs)]
@@ -99,11 +117,27 @@ impl ImportParams {
 		crate::execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy)
 	}
 
+	/// Enable using precompiled WASM module with locally-stored artifacts
+	/// by specifying the path where artifacts are stored.
+	pub fn wasmtime_precompiled(&self) -> Option<PathBuf> {
+		self.wasmtime_precompiled.clone()
+	}
+
 	/// Enable overriding on-chain WASM with locally-stored WASM
 	/// by specifying the path where local WASM is stored.
 	pub fn wasm_runtime_overrides(&self) -> Option<PathBuf> {
 		self.wasm_runtime_overrides.clone()
 	}
+
+	/// Validate that the path is a valid directory.
+	fn validate_existing_directory(path: &str) -> Result<PathBuf, String> {
+		let path_buf = PathBuf::from(path);
+		if path_buf.is_dir() {
+			Ok(path_buf)
+		} else {
+			Err(format!("The path '{}' is not a valid directory", path))
+		}
+	}
 }
 
 /// Execution strategies parameters.

substrate/client/cli/src/runner.rs

@@ -254,6 +254,7 @@ mod tests {
 				trie_cache_maximum_size: None,
 				state_pruning: None,
 				blocks_pruning: sc_client_db::BlocksPruning::KeepAll,
+				wasmtime_precompiled: None,
 				chain_spec: Box::new(
 					GenericChainSpec::<NoExtension, ()>::builder(
 						Default::default(),

substrate/client/executor/benches/bench.rs

@@ -70,9 +70,12 @@ fn initialize(
 			};
 
 			if precompile {
-				let precompiled_blob =
-					sc_executor_wasmtime::prepare_runtime_artifact(blob, &config.semantics)
-						.unwrap();
+				let precompiled_blob = sc_executor_wasmtime::prepare_runtime_artifact(
+					blob,
+					Default::default(),
+					&config.semantics,
+				)
+				.unwrap();
 
 				// Create a fresh temporary directory to make absolutely sure
 				// we'll use the right module.
@@ -84,7 +87,7 @@ fn initialize(
 				unsafe {
 					sc_executor_wasmtime::create_runtime_from_artifact::<
 						sp_io::SubstrateHostFunctions,
-					>(&path, config)
+					>(&path, sc_executor_wasmtime::ModuleVersionStrategy::default(), config)
 				}
 			} else {
 				sc_executor_wasmtime::create_runtime::<sp_io::SubstrateHostFunctions>(blob, config)

substrate/client/executor/common/Cargo.toml

@@ -17,6 +17,7 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+codec = { workspace = true, default-features = true }
 thiserror = { workspace = true }
 wasm-instrument = { workspace = true, default-features = true }
 sc-allocator = { workspace = true, default-features = true }

substrate/client/executor/common/src/wasm_runtime.rs

@@ -74,7 +74,7 @@ pub trait WasmInstance: Send {
 /// Defines the heap pages allocation strategy the wasm runtime should use.
 ///
 /// A heap page is defined as 64KiB of memory.
-#[derive(Debug, Copy, Clone, PartialEq, Hash, Eq)]
+#[derive(Debug, Copy, Clone, PartialEq, Hash, Eq, codec::Encode)]
 pub enum HeapAllocStrategy {
 	/// Allocate a static number of heap pages.
 	///