Merge branch 'containers:main' into add-rdt

.github/workflows/fcos-podman-next-build.yml

@@ -28,7 +28,7 @@ jobs:
 
  - name: Set up wait-for-copr
  # Do not run on scheduled nightly builds
- if: ${{ github.event_name }} != 'schedule'
+ if: ${{ github.event_name != 'schedule' }}
  run: |
  pip3 install git+https://github.com/packit/wait-for-copr.git@main
 
@@ -41,7 +41,7 @@ jobs:
 
  - name: Wait for successful podman-next build with the latest commit
  # Do not run on scheduled nightly builds
- if: ${{ github.event_name }} != 'schedule'
+ if: ${{ github.event_name != 'schedule' }}
  run: |
  # TODO: add this in the Containerfile itself or as a --build-arg
  wait-for-copr --owner ${{ env.COPR_OWNER }} --project ${{ env.COPR_PROJECT }} podman ${{ env.SHORT_SHA }}

contrib/cirrus/lib.sh

@@ -214,11 +214,9 @@ use_cni() {
  die "Testing debian w/ CNI networking currently not supported"
  fi
 
- msg "Unsetting NETWORK_BACKEND for all subsequent environments."
- echo "export -n NETWORK_BACKEND" >> /etc/ci_environment
- echo "unset NETWORK_BACKEND" >> /etc/ci_environment
- export -n NETWORK_BACKEND
- unset NETWORK_BACKEND
+ msg "Forcing NETWORK_BACKEND=cni for all subsequent environments."
+ echo "NETWORK_BACKEND=cni" >> /etc/ci_environment
+ export NETWORK_BACKEND=cni
  # While it's possible a user may want both installed, for CNI CI testing
  # purposes we only care about backward-compatibility, not forward.
  # If both CNI & netavark are present, in some situations where --root
@@ -250,9 +248,11 @@ use_cni() {
 use_netavark() {
  req_env_vars OS_RELEASE_ID PRIOR_FEDORA_NAME DISTRO_NV
  local magickind repokind
- msg "Forcing NETWORK_BACKEND=netavark for all subsequent environments."
- echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
- export NETWORK_BACKEND=netavark # needed for install_test_configs()
+ msg "Unsetting NETWORK_BACKEND for all subsequent environments."
+ echo "export -n NETWORK_BACKEND" >> /etc/ci_environment
+ echo "unset NETWORK_BACKEND" >> /etc/ci_environment
+ export -n NETWORK_BACKEND
+ unset NETWORK_BACKEND
  msg "Removing any/all CNI configuration"
  showrun rm -rvf /etc/cni/net.d/*
  # N/B: The CNI packages are still installed and available. This is

go.mod

@@ -13,9 +13,9 @@ require (
  github.com/containernetworking/cni v1.1.2
  github.com/containernetworking/plugins v1.3.0
  github.com/containers/buildah v1.32.0
- github.com/containers/common v0.56.1-0.20230922104122-56ed984ea383
+ github.com/containers/common v0.56.1-0.20230927080007-46193148a72b
  github.com/containers/conmon v2.0.20+incompatible
- github.com/containers/gvisor-tap-vsock v0.7.1-0.20230922151156-97028a6a6d6a
+ github.com/containers/gvisor-tap-vsock v0.7.1
  github.com/containers/image/v5 v5.28.0
  github.com/containers/libhvee v0.4.1-0.20230920190832-6ab399cadb68
  github.com/containers/ocicrypt v1.1.8

go.sum

@@ -249,12 +249,12 @@ github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q
 github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0=
 github.com/containers/buildah v1.32.0 h1:uz5Rcf7lGeStj7iPTBgO4UdhQYZqMMzyt9suDf16k1k=
 github.com/containers/buildah v1.32.0/go.mod h1:sN3rA3DbnqekNz3bNdkqWduuirYDuMs54LUCOZOomBE=
-github.com/containers/common v0.56.1-0.20230922104122-56ed984ea383 h1:+SPOIY+DIO5nExB66n9aVWZi/yzcLpks6Ys4IwVxOLY=
-github.com/containers/common v0.56.1-0.20230922104122-56ed984ea383/go.mod h1:ABFEglmyt48WWWQv80kGhitfbVfR1Br35wk3gBQdrIk=
+github.com/containers/common v0.56.1-0.20230927080007-46193148a72b h1:ahyeJLCBiaNcyW0qKrAqZfy5Z2/ZKvAuTgM7pnxYijc=
+github.com/containers/common v0.56.1-0.20230927080007-46193148a72b/go.mod h1:ABFEglmyt48WWWQv80kGhitfbVfR1Br35wk3gBQdrIk=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
-github.com/containers/gvisor-tap-vsock v0.7.1-0.20230922151156-97028a6a6d6a h1:AytlbDLdlu6fZxulV3sHrXYQpQpkipNCZA6LGwcL37M=
-github.com/containers/gvisor-tap-vsock v0.7.1-0.20230922151156-97028a6a6d6a/go.mod h1:WSSsjcuYZkvP8i0J+Ht3LF8yvysn3krD5zxQ74wz7y0=
+github.com/containers/gvisor-tap-vsock v0.7.1 h1:+Rc+sOPplrkQb/BUXeN0ug8TxjgyrIqo/9P/eNS2A4c=
+github.com/containers/gvisor-tap-vsock v0.7.1/go.mod h1:WSSsjcuYZkvP8i0J+Ht3LF8yvysn3krD5zxQ74wz7y0=
 github.com/containers/image/v5 v5.28.0 h1:H4cWbdI88UA/mDb6SxMo3IxpmS1BSs/Kifvhwt9g048=
 github.com/containers/image/v5 v5.28.0/go.mod h1:9aPnNkwHNHgGl9VlQxXEshvmOJRbdRAc1rNDD6sP2eU=
 github.com/containers/libhvee v0.4.1-0.20230920190832-6ab399cadb68 h1:QIwOjkVpJp/onBOozw+MSr1mow9f5XQ8QG7Y8AP2Xp0=

libpod/oci_conmon_common.go

@@ -35,7 +35,6 @@ import (
  "github.com/containers/podman/v4/pkg/specgenutil"
  "github.com/containers/podman/v4/pkg/util"
  "github.com/containers/podman/v4/utils"
- "github.com/containers/storage/pkg/homedir"
  spec "github.com/opencontainers/runtime-spec/specs-go"
  "github.com/sirupsen/logrus"
  "golang.org/x/sys/unix"
@@ -1042,11 +1041,6 @@ func (r *ConmonOCIRuntime) getLogTag(ctr *Container) (string, error) {
 func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *ContainerCheckpointOptions) (int64, error) {
  var stderrBuf bytes.Buffer
 
- runtimeDir, err := util.GetRuntimeDir()
- if err != nil {
- return 0, err
- }
-
  parentSyncPipe, childSyncPipe, err := newPipe()
  if err != nil {
  return 0, fmt.Errorf("creating socket pair: %w", err)
@@ -1189,7 +1183,10 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
  }
 
  // 0, 1 and 2 are stdin, stdout and stderr
- conmonEnv := r.configureConmonEnv(runtimeDir)
+ conmonEnv, err := r.configureConmonEnv()
+ if err != nil {
+ return 0, fmt.Errorf("configuring conmon env: %w", err)
+ }
 
  var filesToClose []*os.File
  if preserveFDs > 0 {
@@ -1251,7 +1248,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
  if restoreOptions != nil {
  runtimeRestoreStarted = time.Now()
  }
- err = startCommand(cmd, ctr)
+ err = cmd.Start()
 
  // regardless of whether we errored or not, we no longer need the children pipes
  childSyncPipe.Close()
@@ -1311,38 +1308,23 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 }
 
 // configureConmonEnv gets the environment values to add to conmon's exec struct
-// TODO this may want to be less hardcoded/more configurable in the future
-func (r *ConmonOCIRuntime) configureConmonEnv(runtimeDir string) []string {
- var env []string
- for _, e := range os.Environ() {
- if strings.HasPrefix(e, "LC_") {
- env = append(env, e)
- }
- if strings.HasPrefix(e, "LANG=") {
- env = append(env, e)
+func (r *ConmonOCIRuntime) configureConmonEnv() ([]string, error) {
+ env := os.Environ()
+ res := make([]string, 0, len(env))
+ for _, v := range env {
+ if strings.HasPrefix(v, "NOTIFY_SOCKET=") {
+ // The NOTIFY_SOCKET must not leak into the environment.
+ continue
  }
+ res = append(res, v)
  }
- if path, ok := os.LookupEnv("PATH"); ok {
- env = append(env, fmt.Sprintf("PATH=%s", path))
- }
- if conf, ok := os.LookupEnv("CONTAINERS_CONF"); ok {
- env = append(env, fmt.Sprintf("CONTAINERS_CONF=%s", conf))
- }
- if conf, ok := os.LookupEnv("CONTAINERS_CONF_OVERRIDE"); ok {
- env = append(env, fmt.Sprintf("CONTAINERS_CONF_OVERRIDE=%s", conf))
- }
- if conf, ok := os.LookupEnv("CONTAINERS_HELPER_BINARY_DIR"); ok {
- env = append(env, fmt.Sprintf("CONTAINERS_HELPER_BINARY_DIR=%s", conf))
- }
- env = append(env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
- env = append(env, fmt.Sprintf("_CONTAINERS_USERNS_CONFIGURED=%s", os.Getenv("_CONTAINERS_USERNS_CONFIGURED")))
- env = append(env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%s", os.Getenv("_CONTAINERS_ROOTLESS_UID")))
- home := homedir.Get()
- if home != "" {
- env = append(env, fmt.Sprintf("HOME=%s", home))
+ runtimeDir, err := util.GetRuntimeDir()
+ if err != nil {
+ return nil, err
  }
 
- return env
+ res = append(res, "XDG_RUNTIME_DIR="+runtimeDir)
+ return res, nil
 }
 
 // sharedConmonArgs takes common arguments for exec and create/restore and formats them for the conmon CLI
@@ -1422,25 +1404,6 @@ func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, p
  return args
 }
 
-func startCommand(cmd *exec.Cmd, ctr *Container) error {
- // Make sure to unset the NOTIFY_SOCKET and reset it afterwards if needed.
- switch ctr.config.SdNotifyMode {
- case define.SdNotifyModeContainer, define.SdNotifyModeIgnore:
- if prev := os.Getenv("NOTIFY_SOCKET"); prev != "" {
- if err := os.Unsetenv("NOTIFY_SOCKET"); err != nil {
- logrus.Warnf("Error unsetting NOTIFY_SOCKET %v", err)
- }
- defer func() {
- if err := os.Setenv("NOTIFY_SOCKET", prev); err != nil {
- logrus.Errorf("Resetting NOTIFY_SOCKET=%s", prev)
- }
- }()
- }
- }
-
- return cmd.Start()
-}
-
 // newPipe creates a unix socket pair for communication.
 // Returns two files - first is parent, second is child.
 func newPipe() (*os.File, *os.File, error) {

libpod/oci_conmon_exec_common.go

@@ -17,7 +17,6 @@ import (
  "github.com/containers/podman/v4/libpod/define"
  "github.com/containers/podman/v4/pkg/errorhandling"
  "github.com/containers/podman/v4/pkg/lookup"
- "github.com/containers/podman/v4/pkg/util"
  spec "github.com/opencontainers/runtime-spec/specs-go"
  "github.com/sirupsen/logrus"
  "golang.org/x/sys/unix"
@@ -374,11 +373,6 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex
  }
  }()
 
- runtimeDir, err := util.GetRuntimeDir()
- if err != nil {
- return nil, nil, err
- }
-
  finalEnv := make([]string, 0, len(options.Env))
  for k, v := range options.Env {
  finalEnv = append(finalEnv, fmt.Sprintf("%s=%s", k, v))
@@ -438,7 +432,10 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex
  // }
  // }
 
- conmonEnv := r.configureConmonEnv(runtimeDir)
+ conmonEnv, err := r.configureConmonEnv()
+ if err != nil {
+ return nil, nil, fmt.Errorf("configuring conmon env: %w", err)
+ }
 
  var filesToClose []*os.File
  if options.PreserveFDs > 0 {
@@ -461,7 +458,7 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex
  Setpgid: true,
  }
 
- err = startCommand(execCmd, c)
+ err = execCmd.Start()
 
  // We don't need children pipes on the parent side
  errorhandling.CloseQuiet(childSyncPipe)

pkg/machine/applehv/machine.go

@@ -11,6 +11,7 @@ import (
  "io/fs"
  "net"
  "os"
+ "os/exec"
  "path/filepath"
  "strconv"
  "strings"
@@ -263,7 +264,7 @@ func (m *MacMachine) Init(opts machine.InitOptions) (bool, error) {
  return false, err
  }
 
- // Until the disk resize can be fixed, we ignore it
+ logrus.Debugf("resizing disk to %d GiB", opts.DiskSize)
  if err := m.resizeDisk(strongunits.GiB(opts.DiskSize)); err != nil {
  return false, err
  }
@@ -297,10 +298,16 @@ func (m *MacMachine) Inspect() (*machine.InspectInfo, error) {
  if err != nil {
  return nil, err
  }
+
+ podmanSocket, err := m.forwardSocketPath()
+ if err != nil {
+ return nil, err
+ }
+
  ii := machine.InspectInfo{
  ConfigPath: m.ConfigPath,
  ConnectionInfo: machine.ConnectionConfig{
- PodmanSocket: nil,
+ PodmanSocket: podmanSocket,
  PodmanPipe: nil,
  },
  Created: m.Created,
@@ -598,7 +605,6 @@ func (m *MacMachine) Start(name string, opts machine.StartOptions) error {
  }
 
  cmd.ExtraFiles = []*os.File{ioEater, ioEater, ioEater}
- fmt.Println(cmd.Args)
 
  readSocketBaseDir := filepath.Dir(m.ReadySocket.GetPath())
  if err := os.MkdirAll(readSocketBaseDir, 0755); err != nil {
@@ -680,7 +686,7 @@ func (m *MacMachine) Stop(name string, opts machine.StopOptions) error {
  }
 
  if vmState != machine.Running {
- return machine.ErrWrongState
+ return nil
  }
 
  defer func() {
@@ -952,7 +958,16 @@ func (m *MacMachine) resizeDisk(newSize strongunits.GiB) error {
  // error has not merged
  return fmt.Errorf("invalid disk size %d: new disk must be larger than %dGB", newSize, m.DiskSize)
  }
- return os.Truncate(m.ImagePath.GetPath(), int64(newSize.ToBytes()))
+ logrus.Debugf("resizing %s to %d bytes", m.ImagePath.GetPath(), newSize.ToBytes())
+ // seems like os.truncate() is not very performant with really large files
+ // so exec'ing out to the command truncate
+ size := fmt.Sprintf("%dG", newSize)
+ c := exec.Command("truncate", "-s", size, m.ImagePath.GetPath())
+ if logrus.IsLevelEnabled(logrus.DebugLevel) {
+ c.Stderr = os.Stderr
+ c.Stdout = os.Stdout
+ }
+ return c.Run()
 }
 
 // isFirstBoot returns a bool reflecting if the machine has been booted before

pkg/machine/applehv/rest.go

@@ -94,9 +94,10 @@ func (vf *VfkitHelper) stop(force, wait bool) error {
  if err := vf.stateChange(define.HardStop); err != nil {
  return err
  }
- }
- if err := vf.stateChange(define.Stop); err != nil {
- return err
+ } else {
+ if err := vf.stateChange(define.Stop); err != nil {
+ return err
+ }
  }
  if !wait {
  return nil

pkg/machine/e2e/README.md

@@ -13,14 +13,30 @@ Note: you must not have any machines defined before running tests
 
 1. Open a powershell as admin
 1. $env:CONTAINERS_MACHINE_PROVIDER="hyperv"
-1. $env:MACHINE_IMAGE="https://fedorapeople.org/groups/podman/testing/hyperv/fedora-coreos-38.20230830.dev.0-hyperv.x86_64.vhdx.zip"
-1. `./test/tools/build/ginkgo.exe -vv --tags "remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp remote" -timeout=90m --trace --no-color pkg/machine/e2e/. `
+1. `./winmake localmachine`
 
-Note: Add `--focus-file "basic_test.go" ` to only run basic test
+Note: To run specfic test files, add the test files to the end of the winmake command:
+
+`./winmake localmachine "basic_test.go start_test.go"`
 
 ### WSL
 1. Open a powershell as a regular user
 1. Build and copy win-sshproxy into bin/
-1. `./test/tools/build/ginkgo.exe -vv --tags "remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp remote" -timeout=90m --trace --no-color pkg/machine/e2e/. `
+1. `./winmake localmachine`
+
+Note: To run specfic test files, add the test files to the end of the winmake command:
+
+`./winmake localmachine "basic_test.go start_test.go"`
+
+## MacOS
+
+### Apple Hypervisor
+
+1. `make podman-remote`
+1. `make .install.ginkgo`
+1. `export TMPDIR=/Users/<yourname>`
+1. `export CONTAINERS_MACHINE_PROVIDER="applehv"`
+1. `export MACHINE_IMAGE="https://fedorapeople.org/groups/podman/testing/applehv/arm64/fedora-coreos-38.20230925.dev.0-applehv.aarch64.raw.gz"`
+1. `./test/tools/build/ginkgo -vv --tags "remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp remote" -timeout=90m --trace --no-color pkg/machine/e2e/.`
 
 Note: Add `--focus-file "basic_test.go" ` to only run basic test

pkg/machine/e2e/config_darwin_test.go

@@ -0,0 +1,3 @@
+package e2e_test
+
+const podmanBinary = "../../../bin/darwin/podman"

pkg/machine/e2e/config_linux_test.go

@@ -1,10 +1,3 @@
 package e2e_test
 
-import "os/exec"
-
 const podmanBinary = "../../../bin/podman-remote"
-
-func pgrep(n string) (string, error) {
- out, err := exec.Command("pgrep", "gvproxy").Output()
- return string(out), err
-}

pkg/machine/e2e/config_test.go

@@ -201,3 +201,19 @@ func (matcher *ValidJSONMatcher) FailureMessage(actual interface{}) (message str
 func (matcher *ValidJSONMatcher) NegatedFailureMessage(actual interface{}) (message string) {
  return format.Message(actual, "to _not_ be valid JSON")
 }
+
+func skipIfVmtype(vmType machine.VMType, message string) {
+ if testProvider.VMType() == vmType {
+ Skip(message)
+ }
+}
+
+func skipIfNotVmtype(vmType machine.VMType, message string) {
+ if testProvider.VMType() != vmType {
+ Skip(message)
+ }
+}
+
+func skipIfWSL(message string) {
+ skipIfVmtype(machine.WSLVirt, message)
+}