[PROD-17859] Properly handle opaque whiteout entries in Docker layer …

…flattener

Flattened version of upstream PR google#110

BUILD.bazel

@@ -101,3 +101,14 @@ sh_test(
  ":pusher.par",
  ],
 )
+
+py_test(
+ name = "client_v2_2_unit_tests",
+ size = "large",
+ srcs = [
+ "client_v2_2_unit_tests.py",
+ ":containerregistry",
+ ],
+ main = "client_v2_2_unit_tests.py",
+)
+

client/v2_2/docker_image_.py

@@ -697,20 +697,18 @@ def __exit__(self, unused_type, unused_value, unused_traceback):
  pass
 
 
-def _in_whiteout_dir(
- fs,
- name
-):
+def _in_whiteout_dir(fs, opaque_whiteouts, name):
  while name:
  dirname = os.path.dirname(name)
  if name == dirname:
  break
- if fs.get(dirname):
+ if fs.get(dirname) or dirname in opaque_whiteouts:
  return True
  name = dirname
  return False
 
 _WHITEOUT_PREFIX = '.wh.'
+_OPAQUE_WHITEOUT_FILENAME = '.wh..wh..opq'
 
 
 def extract(image, tar):
@@ -724,17 +722,27 @@ def extract(image, tar):
  # to whether they are a tombstone or not.
  fs = {}
 
+ opaque_whiteouts_in_higher_layers = set()
+
  # Walk the layers, topmost first and add files. If we've seen them in a
  # higher layer then we skip them
  for layer in image.diff_ids():
  buf = cStringIO.StringIO(image.uncompressed_layer(layer))
  with tarfile.open(mode='r:', fileobj=buf) as layer_tar:
+ opaque_whiteouts_in_this_layer = []
  for tarinfo in layer_tar:
  # If we see a whiteout file, then don't add anything to the tarball
  # but ensure that any lower layers don't add a file with the whited
  # out name.
  basename = os.path.basename(tarinfo.name)
  dirname = os.path.dirname(tarinfo.name)
+
+ # If we see an opaque whiteout file, then don't add anything to the
+ # tarball but ensure that any lower layers don't add files or
+ # directories which are siblings of the whiteout file.
+ if basename == _OPAQUE_WHITEOUT_FILENAME:
+ opaque_whiteouts_in_this_layer.append(dirname)
+
  tombstone = basename.startswith(_WHITEOUT_PREFIX)
  if tombstone:
  basename = basename[len(_WHITEOUT_PREFIX):]
@@ -746,7 +754,7 @@ def extract(image, tar):
  continue
 
  # Check for a whited out parent directory
- if _in_whiteout_dir(fs, name):
+ if _in_whiteout_dir(fs, opaque_whiteouts_in_higher_layers, name):
  continue
 
  # Mark this file as handled by adding its name.
@@ -758,3 +766,4 @@ def extract(image, tar):
  tar.addfile(tarinfo, fileobj=layer_tar.extractfile(tarinfo))
  else:
  tar.addfile(tarinfo, fileobj=None)
+ opaque_whiteouts_in_higher_layers.update(opaque_whiteouts_in_this_layer)

client_v2_2_unit_tests.py

@@ -0,0 +1,167 @@
+# Copyright 2018 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+import io
+from StringIO import StringIO
+import tarfile
+import unittest
+
+from containerregistry.client.v2_2 import docker_image as v2_2_image
+
+
+class MockImage(object):
+ """Mock of DockerImage, implementing only the methods called by extract()."""
+
+ def __init__(self):
+ self._fs_layers = OrderedDict()
+
+ def add_layer(self, filenames):
+ """Add a layer to the image.
+
+ Args:
+ filenames: a list of filenames or (filename, content) pairs. Filenames
+ with trailing slashes become directory entries in the generated tar
+ """
+ buf = io.BytesIO()
+ with tarfile.open(mode='w:', fileobj=buf) as tf:
+ for entry in filenames:
+ if (isinstance(entry, basestring)):
+ name = entry
+ content = ""
+ else:
+ (name, content) = entry
+ tarinfo = tarfile.TarInfo(name)
+ tarinfo.size = len(content)
+ if name.endswith("/"):
+ tarinfo.type = tarfile.DIRTYPE
+ tf.addfile(tarinfo, fileobj=(StringIO(content) if content else None))
+ buf.seek(0)
+ new_layer_id = str(len(self._fs_layers))
+ self._fs_layers[new_layer_id] = buf.getvalue()
+
+ def diff_ids(self):
+ return reversed(self._fs_layers.keys())
+
+ def uncompressed_layer(self, layer_id):
+ return self._fs_layers[layer_id]
+
+
+class TestExtract(unittest.TestCase):
+
+ def _test_flatten(self, layer_filenames, expected_flattened_output):
+ # Construct a mock DockerImage with the specified layers:
+ img = MockImage()
+ for filenames in layer_filenames:
+ img.add_layer(filenames)
+ buf = io.BytesIO()
+
+ # Run the actual extract logic:
+ with tarfile.open(mode='w:', fileobj=buf) as tar:
+ v2_2_image.extract(img, tar)
+
+ # Compare the extract() output to the expected results:
+ buf.seek(0)
+ flattened_output = []
+ with tarfile.open(mode='r', fileobj=buf) as tar:
+ for tarinfo in tar:
+ if tarinfo.isdir():
+ flattened_output.append(tarinfo.name + "/")
+ else:
+ contents = tar.extractfile(tarinfo).read()
+ if contents:
+ flattened_output.append((tarinfo.name, contents))
+ else:
+ flattened_output.append(tarinfo.name)
+ self.assertEqual(flattened_output, expected_flattened_output)
+
+ def test_single_layer(self):
+ self._test_flatten(
+ [["/directory/", "/file"]],
+ ["/directory/", "/file"]
+ )
+
+ def test_purely_additive_layers(self):
+ self._test_flatten(
+ [
+ ["dir/", "dir/file1", "file"],
+ ["dir/file2", "file2"]
+ ],
+ ["dir/file2", "file2", "dir/", "dir/file1", "file"]
+ )
+
+ def test_highest_layer_of_file_takes_precedence(self):
+ self._test_flatten(
+ [
+ [("file", "a")],
+ [("file", "b")]
+ ],
+ [("file", "b")]
+ )
+
+ def test_single_file_whiteout(self):
+ self._test_flatten(
+ [
+ ["/foo"],
+ ["/.wh.foo"]
+ ],
+ []
+ )
+
+ def test_parent_directory_whiteout(self):
+ self._test_flatten(
+ [
+ ["/x/a/", "/x/b/", "/x/b/1"],
+ ["/x/.wh.b"]
+ ],
+ ["/x/a/"]
+ )
+
+ def test_opaque_whiteout(self):
+ # Example from https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
+ self._test_flatten(
+ [
+ ["a/", "a/b/", "a/b/c/", "a/b/c/bar"],
+ ["a/", "a/.wh..wh..opq", "a/b/", "a/b/c/", "a/b/c/foo"],
+ ],
+ ["a/", "a/b/", "a/b/c/", "a/b/c/foo"],
+ )
+
+ self._test_flatten(
+ [
+ ["a/", "a/b/", "a/b/c/", "a/b/c/bar"],
+ ["a/", "a/b/", "a/b/c/", "a/b/c/foo", "a/.wh..wh..opq"],
+ ],
+ ["a/", "a/b/", "a/b/c/", "a/b/c/foo"],
+ )
+
+ def test_opaque_whiteout_preserves_parent_directory(self):
+ # Example from https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
+ self._test_flatten(
+ [
+ [
+ "bin/",
+ "bin/my-app-binary",
+ "bin/my-app-tools",
+ "bin/tools/",
+ "bin/tools/my-app-tool-one"
+ ],
+ ["bin/.wh..wh..opq"],
+ ],
+ ["bin/"],
+ )
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)