diff --git a/Cargo.lock b/Cargo.lock index 5229e8c02..ba15ffaf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1864,11 +1864,13 @@ dependencies = [ "mio", "nix", "oci-spec", + "once_cell", "path-clean 1.0.1", "prctl", "procfs", "quickcheck", "rand", + "regex", "rust-criu", "serde", "serde_json", diff --git a/crates/libcontainer/Cargo.toml b/crates/libcontainer/Cargo.toml index 5db20176f..130d79c56 100644 --- a/crates/libcontainer/Cargo.toml +++ b/crates/libcontainer/Cargo.toml @@ -33,6 +33,7 @@ mio = { version = "0.8.6", features = ["os-ext", "os-poll"] } nix = "0.26.2" path-clean = "1.0.1" oci-spec = { version = "^0.6.0", features = ["runtime"] } +once_cell = "1.17.1" procfs = "0.15.1" prctl = "1.0.0" libcgroups = { version = "0.0.5", path = "../libcgroups", default-features = false } @@ -42,6 +43,7 @@ serde_json = "1.0" syscalls = "0.6.10" rust-criu = "0.4.0" clone3 = "0.2.3" +regex = "1.7.3" [dev-dependencies] oci-spec = { version = "^0.6.0", features = ["proptests", "runtime"] } diff --git a/crates/libcontainer/src/container/builder_impl.rs b/crates/libcontainer/src/container/builder_impl.rs index 4410b3783..8745960ef 100644 --- a/crates/libcontainer/src/container/builder_impl.rs +++ b/crates/libcontainer/src/container/builder_impl.rs @@ -5,6 +5,7 @@ use crate::{ process::{ self, args::{ContainerArgs, ContainerType}, + intel_rdt::delete_resctrl_subdirectory, }, rootless::Rootless, syscall::Syscall, @@ -137,7 +138,8 @@ impl<'a> ContainerBuilderImpl<'a> { executor_manager: &self.executor_manager, }; - let init_pid = process::container_main_process::container_main_process(&container_args)?; + let (init_pid, need_to_clean_up_intel_rdt_dir) = + process::container_main_process::container_main_process(&container_args)?; // if file to write the pid to is specified, write pid of the child if let Some(pid_file) = &self.pid_file { @@ -150,6 +152,7 @@ impl<'a> ContainerBuilderImpl<'a> { .set_status(ContainerStatus::Created) .set_creator(nix::unistd::geteuid().as_raw()) .set_pid(init_pid.as_raw()) + .set_clean_up_intel_rdt_directory(need_to_clean_up_intel_rdt_dir) .save() .context("Failed to save container state")?; } @@ -171,11 +174,23 @@ impl<'a> ContainerBuilderImpl<'a> { )?; let mut errors = Vec::new(); + if let Err(e) = cmanager.remove().context("failed to remove cgroup") { errors.push(e.to_string()); } if let Some(container) = &self.container { + if let Some(true) = container.clean_up_intel_rdt_subdirectory() { + if let Err(e) = delete_resctrl_subdirectory(container.id()).with_context(|| { + format!( + "failed to delete resctrl subdirectory: {:?}", + container.id() + ) + }) { + errors.push(e.to_string()); + } + } + if container.root.exists() { if let Err(e) = fs::remove_dir_all(&container.root) .with_context(|| format!("could not delete {:?}", container.root)) diff --git a/crates/libcontainer/src/container/container.rs b/crates/libcontainer/src/container/container.rs index 598277a8b..536994502 100644 --- a/crates/libcontainer/src/container/container.rs +++ b/crates/libcontainer/src/container/container.rs @@ -126,6 +126,15 @@ impl Container { self } + pub fn set_clean_up_intel_rdt_directory(&mut self, clean_up: bool) -> &mut Self { + self.state.clean_up_intel_rdt_subdirectory = Some(clean_up); + self + } + + pub fn clean_up_intel_rdt_subdirectory(&self) -> Option { + self.state.clean_up_intel_rdt_subdirectory + } + pub fn status(&self) -> ContainerStatus { self.state.status } diff --git a/crates/libcontainer/src/container/container_delete.rs b/crates/libcontainer/src/container/container_delete.rs index 5350567e5..63a125286 100644 --- a/crates/libcontainer/src/container/container_delete.rs +++ b/crates/libcontainer/src/container/container_delete.rs @@ -1,6 +1,7 @@ use super::{Container, ContainerStatus}; use crate::config::YoukiConfig; use crate::hooks; +use crate::process::intel_rdt::delete_resctrl_subdirectory; use anyhow::{bail, Context, Result}; use libcgroups; use nix::sys::signal; @@ -66,6 +67,14 @@ impl Container { // Once reached here, the container is verified that it can be deleted. debug_assert!(self.status().can_delete()); + if let Some(true) = &self.clean_up_intel_rdt_subdirectory() { + if let Err(err) = delete_resctrl_subdirectory(self.id()) { + log::warn!( + "failed to delete resctrl subdirectory due to: {err:?}, continue to delete" + ); + } + } + if self.root.exists() { match YoukiConfig::load(&self.root) { Ok(config) => { diff --git a/crates/libcontainer/src/container/state.rs b/crates/libcontainer/src/container/state.rs index 1146ce09d..bd8284a83 100644 --- a/crates/libcontainer/src/container/state.rs +++ b/crates/libcontainer/src/container/state.rs @@ -98,6 +98,8 @@ pub struct State { pub creator: Option, // Specifies if systemd should be used to manage cgroups pub use_systemd: Option, + // Specifies if the Intel RDT subdirectory needs be cleaned up. + pub clean_up_intel_rdt_subdirectory: Option, } impl State { @@ -119,6 +121,7 @@ impl State { created: None, creator: None, use_systemd: None, + clean_up_intel_rdt_subdirectory: None, } } diff --git a/crates/libcontainer/src/process/args.rs b/crates/libcontainer/src/process/args.rs index 3048fc6bc..98ddb8543 100644 --- a/crates/libcontainer/src/process/args.rs +++ b/crates/libcontainer/src/process/args.rs @@ -26,7 +26,7 @@ pub struct ContainerArgs<'a> { pub console_socket: Option, /// The Unix Domain Socket to communicate container start pub notify_socket: NotifyListener, - /// File descriptos preserved/passed to the container init process. + /// File descriptors preserved/passed to the container init process. pub preserve_fds: i32, /// Container state pub container: &'a Option, diff --git a/crates/libcontainer/src/process/container_main_process.rs b/crates/libcontainer/src/process/container_main_process.rs index 33a024281..eb6ecc7d1 100644 --- a/crates/libcontainer/src/process/container_main_process.rs +++ b/crates/libcontainer/src/process/container_main_process.rs @@ -1,6 +1,9 @@ use crate::{ container::ContainerProcessState, - process::{args::ContainerArgs, channel, container_intermediate_process, fork}, + process::{ + args::ContainerArgs, channel, container_intermediate_process, fork, + intel_rdt::setup_intel_rdt, + }, rootless::Rootless, utils, }; @@ -20,7 +23,7 @@ use oci_spec::runtime; #[cfg(feature = "libseccomp")] use std::{io::IoSlice, path::Path}; -pub fn container_main_process(container_args: &ContainerArgs) -> Result { +pub fn container_main_process(container_args: &ContainerArgs) -> Result<(Pid, bool)> { // We use a set of channels to communicate between parent and child process. // Each channel is uni-directional. Because we will pass these channel to // cloned process, we have to be deligent about closing any unused channel. @@ -68,6 +71,7 @@ pub fn container_main_process(container_args: &ContainerArgs) -> Result { // The intermediate process will send the init pid once it forks the init // process. The intermediate process should exit after this point. let init_pid = main_receiver.wait_for_intermediate_ready()?; + let mut need_to_clean_up_intel_rdt_subdirectory = false; if let Some(linux) = container_args.spec.linux() { if let Some(seccomp) = linux.seccomp() { @@ -89,6 +93,14 @@ pub fn container_main_process(container_args: &ContainerArgs) -> Result { sync_seccomp(seccomp, &state, init_sender, main_receiver) .context("failed to sync seccomp with init")?; } + if let Some(intel_rdt) = linux.intel_rdt() { + let container_id = container_args + .container + .as_ref() + .map(|container| container.id()); + need_to_clean_up_intel_rdt_subdirectory = + setup_intel_rdt(container_id, &init_pid, intel_rdt)?; + } } // We don't need to send anything to the init process after this point, so @@ -124,7 +136,7 @@ pub fn container_main_process(container_args: &ContainerArgs) -> Result { Err(err) => bail!("failed to wait for intermediate process: {err}"), }; - Ok(init_pid) + Ok((init_pid, need_to_clean_up_intel_rdt_subdirectory)) } #[cfg(feature = "libseccomp")] diff --git a/crates/libcontainer/src/process/intel_rdt.rs b/crates/libcontainer/src/process/intel_rdt.rs new file mode 100644 index 000000000..62ca4ad8d --- /dev/null +++ b/crates/libcontainer/src/process/intel_rdt.rs @@ -0,0 +1,523 @@ +use once_cell::sync::Lazy; +use regex::Regex; +use std::collections::HashMap; +use std::io::Write; +use std::{ + fs::{self, OpenOptions}, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; +use nix::unistd::Pid; +use oci_spec::runtime::LinuxIntelRdt; +use procfs::process::Process; + +pub fn delete_resctrl_subdirectory(id: &str) -> Result<()> { + let dir = find_resctrl_mount_point().context("failed to find a mounted resctrl file system")?; + let container_resctrl_path = dir.join(id).canonicalize()?; + + match container_resctrl_path.parent() { + // Make sure the container_id really exists and the directory + // is inside the resctrl fs. + Some(parent) => { + if parent == dir && container_resctrl_path.exists() { + fs::remove_dir(container_resctrl_path)?; + } else { + bail!("No resctrl subdirectory found for container id"); + } + } + None => bail!("No parent for resctrl dir"), + } + Ok(()) +} + +/// Finds the resctrl mount path by looking at the process mountinfo data. +pub fn find_resctrl_mount_point() -> Result { + let process = Process::myself()?; + let mount_infos = process.mountinfo()?; + + for mount_info in mount_infos.iter() { + // "resctrl" type fs can be mounted only once. + if mount_info.fs_type == "resctrl" { + let path = mount_info.mount_point.clone().canonicalize()?; + return Ok(path); + } + } + + bail!("resctrl mount point not found"); +} + +/// Adds container PID to the tasks file in the correct resctrl +/// pseudo-filesystem subdirectory. Creates the directory if needed based on +/// the rules in Linux OCI runtime config spec. +fn write_container_pid_to_resctrl_tasks( + path: &Path, + id: &str, + init_pid: Pid, + only_clos_id_set: bool, +) -> Result { + let tasks = path.to_owned().join(id).join("tasks"); + let dir = tasks.parent(); + match dir { + None => bail!("invalid resctrl directory"), + Some(resctrl_container_dir) => { + let mut created_dir = false; + if !resctrl_container_dir.exists() { + if only_clos_id_set { + // Directory doesn't exist and only clos_id is set: error out. + bail!("resctrl closID directory didn't exist"); + } + fs::create_dir_all(resctrl_container_dir)?; + created_dir = true; + } + // TODO(ipuustin): File doesn't need to be created, but it's easier + // to test this way. Fix the tests so that the fake resctrl + // filesystem is pre-populated. + let mut file = OpenOptions::new().create(true).append(true).open(tasks)?; + write!(file, "{init_pid}")?; + Ok(created_dir) + } + } +} + +/// Merges the two schemas together, removing lines starting with "MB:" from +/// l3_cache_schema if mem_bw_schema is also specified. +fn combine_l3_cache_and_mem_bw_schemas( + l3_cache_schema: &Option, + mem_bw_schema: &Option, +) -> Option { + if l3_cache_schema.is_some() && mem_bw_schema.is_some() { + // Combine the results. Filter out "MB:"-lines from l3_cache_schema + let real_l3_cache_schema = l3_cache_schema.as_ref().unwrap(); + let real_mem_bw_schema = mem_bw_schema.as_ref().unwrap(); + let mut output: Vec<&str> = vec![]; + + for line in real_l3_cache_schema.lines() { + if line.starts_with("MB:") { + continue; + } + output.push(line); + } + output.push(real_mem_bw_schema); + return Some(output.join("\n")); + } else if l3_cache_schema.is_some() { + // Apprarently the "MB:"-lines don't need to be removed in this case? + return l3_cache_schema.to_owned(); + } else if mem_bw_schema.is_some() { + return mem_bw_schema.to_owned(); + } + None +} + +#[derive(PartialEq)] +enum LineType { + L3Line, + L3DataLine, + L3CodeLine, + MbLine, + Unknown, +} + +#[derive(PartialEq)] +struct ParsedLine { + line_type: LineType, + tokens: HashMap, +} + +/// Parse tokens ("1=7000") from a "MB:" line. +fn parse_mb_line(line: &str) -> Result> { + let mut token_map = HashMap::new(); + + static MB_VALIDATE_RE: Lazy = Lazy::new(|| { + Regex::new(r"^MB:(?:\s|;)*(?:\w+\s*=\s*\w+)?(?:(?:\s*;+\s*)+\w+\s*=\s*\w+)*(?:\s|;)*$") + .unwrap() + }); + static MB_CAPTURE_RE: Lazy = Lazy::new(|| Regex::new(r"(\w+)\s*=\s*(\w+)").unwrap()); + + if !MB_VALIDATE_RE.is_match(line) { + bail!("MB line doesn't match validation") + } + + for token in MB_CAPTURE_RE.captures_iter(line) { + match (token.get(1), token.get(2)) { + (Some(key), Some(value)) => { + token_map.insert(key.as_str().to_string(), value.as_str().to_string()); + } + _ => bail!("MB token has wrong number of fields"), + } + } + + Ok(token_map) +} + +/// Parse tokens ("0=ffff") from a L3{,CODE,DATA} line. +fn parse_l3_line(line: &str) -> Result> { + let mut token_map = HashMap::new(); + + static L3_VALIDATE_RE: Lazy = Lazy::new(|| { + Regex::new(r"^(?:L3|L3DATA|L3CODE):(?:\s|;)*(?:\w+\s*=\s*[[:xdigit:]]+)?(?:(?:\s*;+\s*)+\w+\s*=\s*[[:xdigit:]]+)*(?:\s|;)*$").unwrap() + }); + static L3_CAPTURE_RE: Lazy = + Lazy::new(|| Regex::new(r"(\w+)\s*=\s*0*([[:xdigit:]]+)").unwrap()); + // ^ + // +-------------+ + // | + // The capture regexp also removes leading zeros from mask values. + + if !L3_VALIDATE_RE.is_match(line) { + bail!("L3 line doesn't match validation") + } + + for token in L3_CAPTURE_RE.captures_iter(line) { + match (token.get(1), token.get(2)) { + (Some(key), Some(value)) => { + token_map.insert(key.as_str().to_string(), value.as_str().to_string()); + } + _ => bail!("L3 token has wrong number of fields"), + } + } + + Ok(token_map) +} + +/// Get the resctrl line type. We only support L3{,CODE,DATA} and MB. +fn get_line_type(line: &str) -> LineType { + if line.starts_with("L3:") { + return LineType::L3Line; + } + if line.starts_with("L3CODE:") { + return LineType::L3CodeLine; + } + if line.starts_with("L3DATA:") { + return LineType::L3DataLine; + } + if line.starts_with("MB:") { + return LineType::MbLine; + } + + // Empty or unknown line. + LineType::Unknown +} + +/// Parse a resctrl line. +fn parse_line(line: &str) -> Option> { + let line_type = get_line_type(line); + + let maybe_tokens = match line_type { + LineType::L3Line => parse_l3_line(line).map(Some), + LineType::L3DataLine => parse_l3_line(line).map(Some), + LineType::L3CodeLine => parse_l3_line(line).map(Some), + LineType::MbLine => parse_mb_line(line).map(Some), + LineType::Unknown => Ok(None), + }; + + match maybe_tokens { + Err(err) => Some(Err(err)), + Ok(None) => None, + Ok(Some(tokens)) => Some(Ok(ParsedLine { line_type, tokens })), + } +} + +/// Compare two sets of parsed lines. Do this both ways because of possible +/// duplicate lines, meaning that the vector lengths may be different. +fn compare_lines(first_lines: &[ParsedLine], second_lines: &[ParsedLine]) -> bool { + first_lines.iter().all(|line| second_lines.contains(line)) + && second_lines.iter().all(|line| first_lines.contains(line)) +} + +/// Compares that two strings have the same set of lines (even if the lines are +/// in different order). +fn is_same_schema(combined_schema: &str, existing_schema: &str) -> Result { + // Parse the strings first to lines and then to structs. Also filter + // out lines that are non-L3{DATA,CODE} and non-MB. + let combined = combined_schema + .lines() + .filter_map(parse_line) + .collect::>>()?; + let existing = existing_schema + .lines() + .filter_map(parse_line) + .collect::>>()?; + + // Compare the two sets of parsed lines. + Ok(compare_lines(&combined, &existing)) +} + +/// Combines the l3_cache_schema and mem_bw_schema values together with the +/// rules given in Linux OCI runtime config spec. If clos_id_was_set parameter +/// is true and the directory wasn't created, the rules say that the schemas +/// need to be compared with the existing value and an error must be generated +/// if they don't match. +fn write_resctrl_schemata( + path: &Path, + id: &str, + l3_cache_schema: &Option, + mem_bw_schema: &Option, + clos_id_was_set: bool, + created_dir: bool, +) -> Result<()> { + let schemata = path.to_owned().join(id).join("schemata"); + let maybe_combined_schema = combine_l3_cache_and_mem_bw_schemas(l3_cache_schema, mem_bw_schema); + + if let Some(combined_schema) = maybe_combined_schema { + if clos_id_was_set && !created_dir { + // Compare existing schema and error out if no match. + let data = fs::read_to_string(&schemata)?; + if !is_same_schema(&combined_schema, &data)? { + bail!("existing schemata found but data did not match"); + } + } else { + // Write the combined schema to the schemata file. + // TODO(ipuustin): File doesn't need to be created, but it's easier + // to test this way. Fix the tests so that the fake resctrl + // filesystem is pre-populated. + let mut file = OpenOptions::new().create(true).write(true).open(schemata)?; + // Prevent write!() from writing the newline with a separate call. + let schema_with_newline = combined_schema + "\n"; + write!(file, "{schema_with_newline}")?; + } + } + + Ok(()) +} + +/// Sets up Intel RDT configuration for the container process based on the +/// OCI config. The result bool tells whether or not we need to clean up +/// the created subdirectory. +pub fn setup_intel_rdt( + maybe_container_id: Option<&str>, + init_pid: &Pid, + intel_rdt: &LinuxIntelRdt, +) -> Result { + // Find mounted resctrl filesystem, error out if it can't be found. + let path = + find_resctrl_mount_point().context("failed to find a mounted resctrl file system")?; + let clos_id_set = intel_rdt.clos_id().is_some(); + let only_clos_id_set = + clos_id_set && intel_rdt.l3_cache_schema().is_none() && intel_rdt.mem_bw_schema().is_none(); + let id = match (intel_rdt.clos_id(), maybe_container_id) { + (Some(clos_id), _) => clos_id, + (None, Some(container_id)) => container_id, + (None, None) => bail!("failed to find ID for resctrl"), + }; + + let created_dir = write_container_pid_to_resctrl_tasks(&path, id, *init_pid, only_clos_id_set) + .context("failed to write container pid to resctrl tasks file")?; + write_resctrl_schemata( + &path, + id, + intel_rdt.l3_cache_schema(), + intel_rdt.mem_bw_schema(), + clos_id_set, + created_dir, + ) + .context("failed to write schemas to resctrl schemata file")?; + + // If closID is not set and the runtime has created the sub-directory, + // the runtime MUST remove the sub-directory when the container is deleted. + let need_to_delete_directory = !clos_id_set && created_dir; + + Ok(need_to_delete_directory) +} + +#[cfg(test)] +mod test { + use crate::utils::create_temp_dir; + + use super::*; + use anyhow::Result; + + #[test] + fn test_combine_schemas() -> Result<()> { + let res = combine_l3_cache_and_mem_bw_schemas(&None, &None); + assert!(res.is_none()); + + let l3_1 = "L3:0=f;1=f0"; + let bw_1 = "MB:0=70;1=20"; + + let res = combine_l3_cache_and_mem_bw_schemas(&Some(l3_1.to_owned()), &None); + assert!(res.is_some()); + assert!(res.unwrap() == "L3:0=f;1=f0"); + + let res = combine_l3_cache_and_mem_bw_schemas(&None, &Some(bw_1.to_owned())); + assert!(res.is_some()); + assert!(res.unwrap() == "MB:0=70;1=20"); + + let res = + combine_l3_cache_and_mem_bw_schemas(&Some(l3_1.to_owned()), &Some(bw_1.to_owned())); + assert!(res.is_some()); + let val = res.unwrap(); + assert!(val.lines().any(|line| line == "MB:0=70;1=20")); + assert!(val.lines().any(|line| line == "L3:0=f;1=f0")); + + let l3_2 = "L3:0=f;1=f0\nL3:2=f\n;MB:0=20;1=70"; + let res = + combine_l3_cache_and_mem_bw_schemas(&Some(l3_2.to_owned()), &Some(bw_1.to_owned())); + assert!(res.is_some()); + let val = res.unwrap(); + assert!(val.lines().any(|line| line == "MB:0=70;1=20")); + assert!(val.lines().any(|line| line == "L3:0=f;1=f0")); + assert!(val.lines().any(|line| line == "L3:2=f")); + assert!(!val.lines().any(|line| line == "MB:0=20;1=70")); + + Ok(()) + } + + #[test] + fn test_is_same_schema() -> Result<()> { + // Exact same schemas. + assert!(is_same_schema("L3:0=f;1=f0", "L3:0=f;1=f0")?); + assert!(is_same_schema("L3DATA:0=f;1=f0", "L3DATA:0=f;1=f0")?); + assert!(is_same_schema("L3CODE:0=f;1=f0", "L3CODE:0=f;1=f0")?); + assert!(is_same_schema("MB:0=bar;1=f0", "MB:0=bar;1=f0")?); + assert!(is_same_schema("L3:", "L3:")?); + assert!(is_same_schema("MB:", "MB:")?); + + // Different schemas. + assert!(!is_same_schema("L3:0=f;1=f0", "L3:2=f")?); + assert!(!is_same_schema("MB:0=bar;1=f0", "MB:0=foo;1=f0")?); + assert!(!is_same_schema("L3DATA:0=f;1=f0", "L3CODE:2=f")?); + assert!(!is_same_schema("L3DATA:0=f;1=f0", "L3CODE:2=f")?); + assert!(!is_same_schema("L3DATA:0=f", "L3CODE:0=f")?); + assert!(!is_same_schema("L3:0=f", "L3DATA:0=f")?); + assert!(!is_same_schema("L3CODE:0=f", "L3:0=f")?); + assert!(!is_same_schema("MB:0=f", "L3:0=f")?); + + // Exact same multi-line schema. + assert!(is_same_schema( + "L3:0=f;1=f0\nL3:2=f", + "L3:0=f;1=f0\nL3:2=f" + )?); + + // Unknown line type is ignored. + assert!(is_same_schema( + "L3:0=f;1=f0\nL3:2=f\nBAR:foo", + "L3:0=f;1=f0\nL3:2=f" + )?); + + // Different multi-line schema. + assert!(!is_same_schema( + "L3:0=f;1=f0\nL3:2=f\nL3:3=f", + "L3:0=f;1=f0\nL3:2=f" + )?); + + // Different lines (two ways). + assert!(!is_same_schema( + "L3:0=f;1=f0\nL3:2=f\nL3:3=f", + "L3:0=f;1=f0\nL3:2=f" + )?); + assert!(!is_same_schema( + "L3:0=f;1=f0\nL3:2=f", + "L3:0=f;1=f0\nL3:2=f\nL3:3=f" + )?); + + // Same schema, different token order. + assert!(is_same_schema("L3:1=f0;0=0", "L3:0=0;1=f0")?); + + // Same schema, different whitespace and semicolons. + assert!(is_same_schema("L3:;; 0 = f; ; 1=f0", "L3:0=f;1 = f0;;")?); + + // Same schema, different leading zeros in masks. + assert!(is_same_schema("L3:0=000f", "L3:0=0f")?); + assert!(is_same_schema("L3:0=000f", "L3:0=0f")?); + assert!(is_same_schema("L3:0=f", "L3:0=0f")?); + assert!(is_same_schema("L3:0=0", "L3:0=0000")?); + + // Invalid schemas. + assert!(is_same_schema("L3:1=;0=f", "L3:1=;0=f").is_err()); + assert!(is_same_schema("L3:=0;0=f", "L3:=0;0=f").is_err()); + assert!(is_same_schema("L3:1=0=3;0=f", "L3:1=0=3;0=f").is_err()); + assert!(is_same_schema("L3:1=bar", "L3:1=bar").is_err()); + assert!(is_same_schema("MB:1=;0=f", "MB:1=;0=f").is_err()); + assert!(is_same_schema("MB:=0;0=f", "MB:=0;0=f").is_err()); + assert!(is_same_schema("MB:1=0=3;0=f", "MB:1=0=3;0=f").is_err()); + + Ok(()) + } + + #[test] + fn test_write_pid_to_resctrl_tasks() -> Result<()> { + let path = create_temp_dir("test_write_pid_to_resctrl_tasks").unwrap(); + + // Create the directory for id "foo". + let res = write_container_pid_to_resctrl_tasks(&path, "foo", Pid::from_raw(1000), false); + assert!(res.unwrap()); // new directory created + let res = fs::read_to_string(path.join("foo").join("tasks")); + assert!(res.unwrap() == "1000"); + + // Create the same directory the second time. + let res = write_container_pid_to_resctrl_tasks(&path, "foo", Pid::from_raw(1500), false); + assert!(!res.unwrap()); // no new directory created + + // If just clos_id then throw an error. + let res = write_container_pid_to_resctrl_tasks(&path, "foobar", Pid::from_raw(2000), true); + assert!(res.is_err()); + + // If the directory already exists then it's fine to have just clos_id. + let res = write_container_pid_to_resctrl_tasks(&path, "foo", Pid::from_raw(2500), true); + assert!(!res.unwrap()); // no new directory created + + Ok(()) + } + + #[test] + fn test_write_resctrl_schemata() -> Result<()> { + let path = create_temp_dir("test_write_resctrl_schemata").unwrap(); + + let res = write_container_pid_to_resctrl_tasks(&path, "foobar", Pid::from_raw(1000), false); + assert!(res.unwrap()); // new directory created + + // No schemes, clos_id was not set, directory created (with container id). + let res = write_resctrl_schemata(&path, "foobar", &None, &None, false, true); + assert!(res.is_ok()); + let res = fs::read_to_string(path.join("foobar").join("schemata")); + assert!(res.is_err()); // File not found because no schemes. + + let l3_1 = "L3:0=f;1=f0\nL3:2=f\nMB:0=20;1=70"; + let bw_1 = "MB:0=70;1=20"; + let res = write_resctrl_schemata( + &path, + "foobar", + &Some(l3_1.to_owned()), + &Some(bw_1.to_owned()), + false, + true, + ); + assert!(res.is_ok()); + + let res = fs::read_to_string(path.join("foobar").join("schemata")); + assert!(res.is_ok()); + assert!(is_same_schema( + "L3:0=f;1=f0\nL3:2=f\nMB:0=70;1=20\n", + &res.unwrap() + )?); + + // Try the verification case. If the directory existed (was not created + // by us) and the clos_id was set, it needs to contain the same data as + // we are trying to set. This is the same data: + let res = write_resctrl_schemata( + &path, + "foobar", + &Some(l3_1.to_owned()), + &Some(bw_1.to_owned()), + true, + false, + ); + assert!(res.is_ok()); + + // And this different data: + let l3_2 = "L3:0=f;1=f0\nMB:0=20;1=70"; + let bw_2 = "MB:0=70;1=20"; + let res = write_resctrl_schemata( + &path, + "foobar", + &Some(l3_2.to_owned()), + &Some(bw_2.to_owned()), + true, + false, + ); + assert!(res.is_err()); + + Ok(()) + } +} diff --git a/crates/libcontainer/src/process/mod.rs b/crates/libcontainer/src/process/mod.rs index 288993d72..9251ecaab 100644 --- a/crates/libcontainer/src/process/mod.rs +++ b/crates/libcontainer/src/process/mod.rs @@ -7,4 +7,5 @@ pub mod container_init_process; pub mod container_intermediate_process; pub mod container_main_process; pub mod fork; +pub mod intel_rdt; pub mod message; diff --git a/tests/rust-integration-tests/integration_test/src/main.rs b/tests/rust-integration-tests/integration_test/src/main.rs index f697d162e..a4a96b466 100644 --- a/tests/rust-integration-tests/integration_test/src/main.rs +++ b/tests/rust-integration-tests/integration_test/src/main.rs @@ -3,6 +3,7 @@ mod utils; use crate::tests::hooks::get_hooks_tests; use crate::tests::hostname::get_hostname_test; +use crate::tests::intel_rdt::get_intel_rdt_test; use crate::tests::lifecycle::{ContainerCreate, ContainerLifecycle}; use crate::tests::linux_ns_itype::get_ns_itype_tests; use crate::tests::mounts_recursive::get_mounts_recursive_test; @@ -92,6 +93,7 @@ fn main() -> Result<()> { let ro_paths = get_ro_paths_test(); let hostname = get_hostname_test(); let mounts_recursive = get_mounts_recursive_test(); + let intel_rdt = get_intel_rdt_test(); tm.add_test_group(Box::new(cl)); tm.add_test_group(Box::new(cc)); @@ -109,6 +111,7 @@ fn main() -> Result<()> { tm.add_test_group(Box::new(ro_paths)); tm.add_test_group(Box::new(hostname)); tm.add_test_group(Box::new(mounts_recursive)); + tm.add_test_group(Box::new(intel_rdt)); tm.add_cleanup(Box::new(cgroups::cleanup_v1)); tm.add_cleanup(Box::new(cgroups::cleanup_v2)); diff --git a/tests/rust-integration-tests/integration_test/src/tests/intel_rdt/intel_rdt_test.rs b/tests/rust-integration-tests/integration_test/src/tests/intel_rdt/intel_rdt_test.rs new file mode 100644 index 000000000..9e996dbfe --- /dev/null +++ b/tests/rust-integration-tests/integration_test/src/tests/intel_rdt/intel_rdt_test.rs @@ -0,0 +1,60 @@ +use anyhow::{Context, Result}; +use libcontainer::process::intel_rdt::find_resctrl_mount_point; + +use oci_spec::runtime::{LinuxBuilder, LinuxIntelRdt, Spec, SpecBuilder}; +use test_framework::{test_result, TestResult}; + +use crate::utils::{test_outside_container, test_utils::check_container_created}; + +fn create_spec( + maybe_l3_cache: Option<&str>, + maybe_mem_bw: Option<&str>, + maybe_clos_id: Option<&str>, +) -> Result { + let mut intel_rdt = LinuxIntelRdt::default(); + intel_rdt.set_l3_cache_schema(maybe_l3_cache.map(|x| x.to_owned())); + intel_rdt.set_mem_bw_schema(maybe_mem_bw.map(|x| x.to_owned())); + intel_rdt.set_clos_id(maybe_clos_id.map(|x| x.to_owned())); + + // Create the Linux Spec + let linux_spec = LinuxBuilder::default() + .intel_rdt(intel_rdt) + .build() + .context("failed to build linux spec")?; + + // Create the top level Spec + let spec = SpecBuilder::default() + .linux(linux_spec) + .build() + .context("failed to build spec")?; + + Ok(spec) +} + +pub fn test_intel_rdt() -> TestResult { + let cases = vec![ + test_result!(create_spec(Some("L3:0=fff"), Some("MB:0=70"), None)), + test_result!(create_spec(Some("L3:0=fff"), None, None)), + test_result!(create_spec(None, Some("MB:0=70"), None)), + test_result!(create_spec(None, None, None)), + ]; + + for spec in cases.into_iter() { + let test_result = test_outside_container(spec, &|data| { + test_result!(check_container_created(&data)); + + TestResult::Passed + }); + if let TestResult::Failed(_) = test_result { + return test_result; + } + } + + TestResult::Passed +} + +pub fn can_run() -> bool { + // Ensure the resctrl pseudo-filesystem is mounted. + let res = find_resctrl_mount_point(); + res.is_ok() +} diff --git a/tests/rust-integration-tests/integration_test/src/tests/intel_rdt/mod.rs b/tests/rust-integration-tests/integration_test/src/tests/intel_rdt/mod.rs new file mode 100644 index 000000000..9ba805b56 --- /dev/null +++ b/tests/rust-integration-tests/integration_test/src/tests/intel_rdt/mod.rs @@ -0,0 +1,14 @@ +use test_framework::{ConditionalTest, TestGroup}; + +use self::intel_rdt_test::{can_run, test_intel_rdt}; + +mod intel_rdt_test; + +pub fn get_intel_rdt_test() -> TestGroup { + let mut test_group = TestGroup::new("intel_rdt"); + let intel_rdt = ConditionalTest::new("intel_rdt", Box::new(can_run), Box::new(test_intel_rdt)); + + test_group.add(vec![Box::new(intel_rdt)]); + + test_group +} diff --git a/tests/rust-integration-tests/integration_test/src/tests/mod.rs b/tests/rust-integration-tests/integration_test/src/tests/mod.rs index e7ae89595..6328b5860 100644 --- a/tests/rust-integration-tests/integration_test/src/tests/mod.rs +++ b/tests/rust-integration-tests/integration_test/src/tests/mod.rs @@ -1,6 +1,7 @@ pub mod cgroups; pub mod hooks; pub mod hostname; +pub mod intel_rdt; pub mod lifecycle; pub mod linux_ns_itype; pub mod mounts_recursive;