add command PrecompileWasmCmd

This cherry-pick should match this pull request:

paritytech#1641

cumulus/test/service/src/lib.rs

@@ -893,6 +893,7 @@ pub fn node_config(
 		base_path,
 		informant_output_format: Default::default(),
 		wasm_runtime_overrides: None,
+		wasmtime_precompiled: None,
 		runtime_cache_size: 2,
 	})
 }

polkadot/node/core/pvf/common/src/executor_interface.rs

@@ -191,7 +191,7 @@ pub fn prepare(
 	executor_params: &ExecutorParams,
 ) -> Result<Vec<u8>, sc_executor_common::error::WasmError> {
 	let (semantics, _) = params_to_wasmtime_semantics(executor_params);
-	sc_executor_wasmtime::prepare_runtime_artifact(blob, &semantics)
+	sc_executor_wasmtime::prepare_runtime_artifact(blob, Default::default(), &semantics)
 }
 
 /// Available host functions. We leave out:

polkadot/node/test/service/src/lib.rs

@@ -204,6 +204,7 @@ pub fn node_config(
 			instantiation_strategy: WasmtimeInstantiationStrategy::PoolingCopyOnWrite,
 		},
 		wasm_runtime_overrides: Default::default(),
+		wasmtime_precompiled: Default::default(),
 		rpc_addr: Default::default(),
 		rpc_max_request_size: Default::default(),
 		rpc_max_response_size: Default::default(),

substrate/bin/node/cli/benches/block_production.rs

@@ -105,6 +105,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		data_path: base_path.path().into(),
 		base_path,
 		informant_output_format: Default::default(),
+		wasmtime_precompiled: None,
 		wasm_runtime_overrides: None,
 	};
 

substrate/bin/node/cli/benches/transaction_pool.rs

@@ -100,6 +100,7 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		data_path: base_path.path().into(),
 		base_path,
 		informant_output_format: Default::default(),
+		wasmtime_precompiled: None,
 		wasm_runtime_overrides: None,
 	};
 

substrate/bin/node/cli/src/cli.rs

@@ -99,4 +99,7 @@ pub enum Subcommand {
 
 	/// Db meta columns information.
 	ChainInfo(sc_cli::ChainInfoCmd),
+
+	/// Precompile the WASM runtime into native code
+	PrecompileWasm(sc_cli::PrecompileWasmCmd),
 }

substrate/bin/node/cli/src/command.rs

@@ -227,5 +227,13 @@ pub fn run() -> Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.sync_run(|config| cmd.run::<Block>(&config))
 		},
+		Some(Subcommand::PrecompileWasm(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				let PartialComponents { task_manager, backend, .. } =
+					new_partial(&config, None)?;
+				Ok((cmd.run(backend, config.chain_spec), task_manager))
+			})
+		},
 	}
 }

substrate/client/chain-spec/src/genesis_block.rs

@@ -23,7 +23,10 @@ use std::{marker::PhantomData, sync::Arc};
 use codec::Encode;
 use sc_client_api::{backend::Backend, BlockImportOperation};
 use sc_executor::RuntimeVersionOf;
-use sp_core::storage::{well_known_keys, StateVersion, Storage};
+use sp_core::{
+	storage::{well_known_keys, StateVersion, Storage},
+};
+
 use sp_runtime::{
 	traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, Zero},
 	BuildStorage,

substrate/client/cli/Cargo.toml

@@ -65,6 +65,12 @@ sp-runtime.workspace = true
 sp-runtime.default-features = true
 sp-version.workspace = true
 sp-version.default-features = true
+sp-state-machine.workspace = true
+sp-state-machine.default-features = true
+sc-executor.workspace = true
+sc-executor.default-features = true
+sp-storage.workspace = true
+sp-storage.default-features = true
 
 [dev-dependencies]
 tempfile = { workspace = true }

substrate/client/cli/src/commands/mod.rs

@@ -30,6 +30,7 @@ mod insert_key;
 mod inspect_key;
 mod inspect_node_key;
 mod key;
+mod precompile_wasm_cmd;
 mod purge_chain_cmd;
 mod revert_cmd;
 mod run_cmd;
@@ -44,6 +45,6 @@ pub use self::{
 	export_blocks_cmd::ExportBlocksCmd, export_state_cmd::ExportStateCmd, generate::GenerateCmd,
 	generate_node_key::GenerateKeyCmdCommon, import_blocks_cmd::ImportBlocksCmd,
 	insert_key::InsertKeyCmd, inspect_key::InspectKeyCmd, inspect_node_key::InspectNodeKeyCmd,
-	key::KeySubcommand, purge_chain_cmd::PurgeChainCmd, revert_cmd::RevertCmd, run_cmd::RunCmd,
-	sign::SignCmd, vanity::VanityCmd, verify::VerifyCmd,
+	key::KeySubcommand, precompile_wasm_cmd::PrecompileWasmCmd, purge_chain_cmd::PurgeChainCmd,
+	revert_cmd::RevertCmd, run_cmd::RunCmd, sign::SignCmd, vanity::VanityCmd, verify::VerifyCmd,
 };

substrate/client/cli/src/commands/precompile_wasm_cmd.rs

@@ -0,0 +1,143 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+    arg_enums::{DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, execution_method_from_cli, WasmExecutionMethod, WasmtimeInstantiationStrategy},
+	error::{self, Error},
+	params::{DatabaseParams, PruningParams, SharedParams},
+    CliConfiguration
+};
+
+use clap::Parser;
+use sc_client_api::{Backend, HeaderBackend};
+use sc_executor::{
+	HeapAllocStrategy, DEFAULT_HEAP_ALLOC_PAGES, precompile_and_serialize_versioned_wasm_runtime,
+};
+use sc_service::ChainSpec;
+use sp_core::traits::RuntimeCode;
+use sp_runtime::traits::Block as BlockT;
+use sp_state_machine::backend::BackendRuntimeCode;
+use std::{fmt::Debug, path::PathBuf, sync::Arc};
+
+/// The `precompile-wasm` command used to serialize a precompiled WASM module.
+#[derive(Debug, Parser)]
+pub struct PrecompileWasmCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub database_params: DatabaseParams,
+
+	/// The default number of 64KB pages to ever allocate for Wasm execution.
+	/// Don't alter this unless you know what you're doing.
+	#[arg(long, value_name = "COUNT")]
+	pub default_heap_pages: Option<u32>,
+
+    /// path to the directory where precompiled artifact will be written
+    #[arg()]
+    pub output_dir: PathBuf,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub pruning_params: PruningParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	/// The WASM instantiation method to use.
+	/// Only has an effect when `wasm-execution` is set to `compiled`.
+	/// The copy-on-write strategies are only supported on Linux.
+	/// If the copy-on-write variant of a strategy is unsupported
+	/// the executor will fall back to the non-CoW equivalent.
+	/// The fastest (and the default) strategy available is `pooling-copy-on-write`.
+	/// The `legacy-instance-reuse` strategy is deprecated and will
+	/// be removed in the future. It should only be used in case of
+	/// issues with the default instantiation strategy.
+	#[arg(
+		long,
+		value_name = "STRATEGY",
+		default_value_t = DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
+		value_enum,
+	)]
+	pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy,
+}
+
+impl PrecompileWasmCmd {
+	/// Run the precompile-wasm command
+	pub async fn run<B, BA>(&self, backend: Arc<BA>, spec: Box<dyn ChainSpec>) -> error::Result<()>
+	where
+        B: BlockT,
+        BA: Backend<B>
+	{
+		let heap_pages = self.default_heap_pages.unwrap_or(DEFAULT_HEAP_ALLOC_PAGES);
+
+		let blockchain_info = backend.blockchain().info();
+
+		if backend.have_state_at(blockchain_info.finalized_hash, blockchain_info.finalized_number) {
+			let state = backend.state_at(backend.blockchain().info().finalized_hash)?;
+
+			precompile_and_serialize_versioned_wasm_runtime(
+				HeapAllocStrategy::Static { extra_pages: heap_pages },
+				&BackendRuntimeCode::new(&state).runtime_code()?,
+				execution_method_from_cli(
+					WasmExecutionMethod::Compiled,
+					self.wasmtime_instantiation_strategy,
+				),
+				&self.output_dir,
+			)
+			.map_err(|e| Error::Application(Box::new(e)))?;
+		} else {
+			let storage = spec.as_storage_builder().build_storage()?;
+			if let Some(wasm_bytecode) = storage.top.get(sp_storage::well_known_keys::CODE) {
+				let runtime_code = RuntimeCode {
+					code_fetcher: &sp_core::traits::WrappedRuntimeCode(
+						wasm_bytecode.as_slice().into(),
+					),
+					hash: sp_core::blake2_256(&wasm_bytecode).to_vec(),
+					heap_pages: Some(heap_pages as u64),
+				};
+				precompile_and_serialize_versioned_wasm_runtime(
+					HeapAllocStrategy::Static { extra_pages: heap_pages },
+					&runtime_code,
+					execution_method_from_cli(
+						WasmExecutionMethod::Compiled,
+						self.wasmtime_instantiation_strategy,
+					),
+					&self.output_dir,
+				)
+				.map_err(|e| Error::Application(Box::new(e)))?;
+			}
+		}
+
+
+        Ok(())
+    }
+}
+
+impl CliConfiguration for PrecompileWasmCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+
+	fn pruning_params(&self) -> Option<&PruningParams> {
+		Some(&self.pruning_params)
+	}
+
+	fn database_params(&self) -> Option<&DatabaseParams> {
+		Some(&self.database_params)
+	}
+}

substrate/client/cli/src/config.rs

@@ -287,6 +287,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		Ok(self.import_params().map(|x| x.wasm_method()).unwrap_or_default())
 	}
 
+	/// Get the path where WASM precompiled artifacts live.
+	///
+	/// By default this is `None`.
+	fn wasmtime_precompiled(&self) -> Option<PathBuf> {
+		self.import_params().map(|x| x.wasmtime_precompiled()).unwrap_or_default()
+	}
+
 	/// Get the path where WASM overrides live.
 	///
 	/// By default this is `None`.
@@ -520,6 +527,7 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			state_pruning: self.state_pruning()?,
 			blocks_pruning: self.blocks_pruning()?,
 			wasm_method: self.wasm_method()?,
+			wasmtime_precompiled: self.wasmtime_precompiled(),
 			wasm_runtime_overrides: self.wasm_runtime_overrides(),
 			rpc_addr: self.rpc_addr(DCV::rpc_listen_port())?,
 			rpc_methods: self.rpc_methods()?,

substrate/client/cli/src/params/import_params.rs

@@ -65,6 +65,16 @@ pub struct ImportParams {
 	)]
 	pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy,
 
+	/// Specify the path where local precompiled WASM runtimes are stored.
+	/// Only has an effect when `wasm-execution` is set to `compiled`.
+	/// 
+	/// The precompiled runtimes must have been generated using the `precompile-runtimes` subcommand
+	/// with the same version of wasmtime and the exact same configuration. 
+	/// The file name must end with the hash of the configuration. This hash must match, otherwise
+	/// the runtime will be recompiled.
+	#[arg(long, value_name = "PATH")]
+	pub wasmtime_precompiled: Option<PathBuf>,
+
 	/// Specify the path where local WASM runtimes are stored.
 	///
 	/// These runtimes will override on-chain runtimes when the version matches.
@@ -107,6 +117,12 @@ impl ImportParams {
 		crate::execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy)
 	}
 
+	/// Enable using precompiled WASM module with locally-stored artifacts
+	/// by specifying the path where artifacts are stored.
+	pub fn wasmtime_precompiled(&self) -> Option<PathBuf> {
+		self.wasmtime_precompiled.clone()
+	}
+
 	/// Enable overriding on-chain WASM with locally-stored WASM
 	/// by specifying the path where local WASM is stored.
 	pub fn wasm_runtime_overrides(&self) -> Option<PathBuf> {

substrate/client/cli/src/runner.rs

@@ -263,6 +263,7 @@ mod tests {
 					.build(),
 				),
 				wasm_method: Default::default(),
+				wasmtime_precompiled: None,
 				wasm_runtime_overrides: None,
 				rpc_addr: None,
 				rpc_max_connections: Default::default(),

substrate/client/executor/Cargo.toml

@@ -17,6 +17,7 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+log = { workspace = true, default-features = true }
 parking_lot = { workspace = true, default-features = true }
 schnellru = { workspace = true }
 tracing = { workspace = true, default-features = true }

substrate/client/executor/benches/bench.rs

@@ -66,12 +66,13 @@ fn initialize(
 					wasm_bulk_memory: false,
 					wasm_reference_types: false,
 					wasm_simd: false,
+					module_version_strategy: Default::default(),
 				},
 			};
 
 			if precompile {
 				let precompiled_blob =
-					sc_executor_wasmtime::prepare_runtime_artifact(blob, &config.semantics)
+					sc_executor_wasmtime::prepare_runtime_artifact(blob, Default::default(), &config.semantics)
 						.unwrap();
 
 				// Create a fresh temporary directory to make absolutely sure
@@ -84,7 +85,7 @@ fn initialize(
 				unsafe {
 					sc_executor_wasmtime::create_runtime_from_artifact::<
 						sp_io::SubstrateHostFunctions,
-					>(&path, config)
+					>(&path, Default::defaut(), config)
 				}
 			} else {
 				sc_executor_wasmtime::create_runtime::<sp_io::SubstrateHostFunctions>(blob, config)

substrate/client/executor/common/Cargo.toml

@@ -17,6 +17,7 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+codec = { package = "parity-scale-codec", version = "3.6.1" }
 thiserror = { workspace = true }
 wasm-instrument = { workspace = true, default-features = true }
 sc-allocator.workspace = true

substrate/client/executor/common/src/wasm_runtime.rs

@@ -74,7 +74,7 @@ pub trait WasmInstance: Send {
 /// Defines the heap pages allocation strategy the wasm runtime should use.
 ///
 /// A heap page is defined as 64KiB of memory.
-#[derive(Debug, Copy, Clone, PartialEq, Hash, Eq)]
+#[derive(Debug, Copy, Clone, PartialEq, Hash, Eq, codec::Encode)]
 pub enum HeapAllocStrategy {
 	/// Allocate a static number of heap pages.
 	///