From 05d34ed1fbaa8233a4cf51a0f52b67aef99a9521 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 17:58:23 -0800 Subject: [PATCH 01/22] 3.1.0-dev Signed-off-by: Joffrey F --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index f141747ae..fc5684061 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.0.0" +version = "3.1.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 83d185d69533757eac4beef646f1e46737bdd67f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 16:02:09 -0800 Subject: [PATCH 02/22] Add login data to the right subdict in auth_configs Signed-off-by: Joffrey F --- docker/api/daemon.py | 2 +- tests/unit/api_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docker/api/daemon.py b/docker/api/daemon.py index 033dbf19e..0e1c75381 100644 --- a/docker/api/daemon.py +++ b/docker/api/daemon.py @@ -139,7 +139,7 @@ def login(self, username, password=None, email=None, registry=None, if response.status_code == 200: if 'auths' not in self._auth_configs: self._auth_configs['auths'] = {} - self._auth_configs[registry or auth.INDEX_NAME] = req_data + self._auth_configs['auths'][registry or auth.INDEX_NAME] = req_data return self._result(response, json=True) def ping(self): diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index c53a4be1f..f65e13ecb 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -212,6 +212,24 @@ def test_search(self): timeout=DEFAULT_TIMEOUT_SECONDS ) + def test_login(self): + self.client.login('sakuya', 'izayoi') + fake_request.assert_called_with( + 'POST', url_prefix + 'auth', + data=json.dumps({'username': 'sakuya', 'password': 'izayoi'}), + timeout=DEFAULT_TIMEOUT_SECONDS, + headers={'Content-Type': 'application/json'} + ) + + assert self.client._auth_configs['auths'] == { + 'docker.io': { + 'email': None, + 'password': 'izayoi', + 'username': 'sakuya', + 'serveraddress': None, + } + } + def test_events(self): self.client.events() From 04bf470f6e7e06615be453e1adfe7c656bc5f153 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 16:51:36 -0800 Subject: [PATCH 03/22] Add workaround for bpo-32713 Signed-off-by: Joffrey F --- docker/utils/utils.py | 4 ++++ tests/unit/utils_test.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e4e2c0dfc..b145f116b 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -107,6 +107,10 @@ def create_archive(root, files=None, fileobj=None, gzip=False): # ignore it and proceed. continue + # Workaround https://bugs.python.org/issue32713 + if i.mtime < 0 or i.mtime > 8**11 - 1: + i.mtime = int(i.mtime) + if constants.IS_WINDOWS_PLATFORM: # Windows doesn't keep track of the execute bit, so we make files # and directories executable by default. diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 1f9daf60a..155889181 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -995,6 +995,18 @@ def test_tar_socket_file(self): tar_data = tarfile.open(fileobj=archive) assert sorted(tar_data.getnames()) == ['bar', 'foo'] + def tar_test_negative_mtime_bug(self): + base = tempfile.mkdtemp() + filename = os.path.join(base, 'th.txt') + self.addCleanup(shutil.rmtree, base) + with open(filename, 'w') as f: + f.write('Invisible Full Moon') + os.utime(filename, (12345, -3600.0)) + with tar(base) as archive: + tar_data = tarfile.open(fileobj=archive) + assert tar_data.getnames() == ['th.txt'] + assert tar_data.getmember('th.txt').mtime == -3600 + class ShouldCheckDirectoryTest(unittest.TestCase): exclude_patterns = [ From 58639aecfa50e0bcfbd1415dc8bab2b4448f4d81 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Feb 2018 13:11:19 -0800 Subject: [PATCH 04/22] Rewrite access check in create_archive with EAFP Signed-off-by: Joffrey F --- docker/utils/utils.py | 8 +++----- tests/unit/utils_test.py | 8 ++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index e4e2c0dfc..b86a3f0ae 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -97,10 +97,6 @@ def create_archive(root, files=None, fileobj=None, gzip=False): for path in files: full_path = os.path.join(root, path) - if os.lstat(full_path).st_mode & os.R_OK == 0: - raise IOError( - 'Can not access file in context: {}'.format(full_path) - ) i = t.gettarinfo(full_path, arcname=path) if i is None: # This happens when we encounter a socket file. We can safely @@ -117,7 +113,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False): with open(full_path, 'rb') as f: t.addfile(i, f) except IOError: - t.addfile(i, None) + raise IOError( + 'Can not read file in context: {}'.format(full_path) + ) else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 1f9daf60a..3139a9709 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -933,7 +933,10 @@ def test_tar_with_empty_directory(self): tar_data = tarfile.open(fileobj=archive) assert sorted(tar_data.getnames()) == ['bar', 'foo'] - @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No chmod on Windows') + @pytest.mark.skipif( + IS_WINDOWS_PLATFORM or os.geteuid() == 0, + reason='root user always has access ; no chmod on Windows' + ) def test_tar_with_inaccessible_file(self): base = tempfile.mkdtemp() full_path = os.path.join(base, 'foo') @@ -944,8 +947,9 @@ def test_tar_with_inaccessible_file(self): with pytest.raises(IOError) as ei: tar(base) - assert 'Can not access file in context: {}'.format(full_path) in \ + assert 'Can not read file in context: {}'.format(full_path) in ( ei.exconly() + ) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows') def test_tar_with_file_symlinks(self): From 34d50483e20e86cb7ab22700e036a5c4d319268a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Feb 2018 14:59:41 -0800 Subject: [PATCH 05/22] Correctly support absolute paths in .dockerignore Signed-off-by: Joffrey F --- docker/utils/build.py | 20 +++++++++++++------- tests/unit/utils_test.py | 27 +++++++++++++++++---------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index d4223e749..a21887349 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -46,7 +46,7 @@ def exclude_paths(root, patterns, dockerfile=None): ) -def should_include(path, exclude_patterns, include_patterns): +def should_include(path, exclude_patterns, include_patterns, root): """ Given a path, a list of exclude patterns, and a list of inclusion patterns: @@ -61,11 +61,15 @@ def should_include(path, exclude_patterns, include_patterns): for pattern in include_patterns: if match_path(path, pattern): return True + if os.path.isabs(pattern) and match_path( + os.path.join(root, path), pattern): + return True return False return True -def should_check_directory(directory_path, exclude_patterns, include_patterns): +def should_check_directory(directory_path, exclude_patterns, include_patterns, + root): """ Given a directory path, a list of exclude patterns, and a list of inclusion patterns: @@ -91,7 +95,7 @@ def normalize_path(path): if (pattern + '/').startswith(path_with_slash) ] directory_included = should_include( - directory_path, exclude_patterns, include_patterns + directory_path, exclude_patterns, include_patterns, root ) return directory_included or len(possible_child_patterns) > 0 @@ -110,26 +114,28 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): # traversal. See https://docs.python.org/2/library/os.html#os.walk dirs[:] = [ d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns + os.path.join(parent, d), exclude_patterns, include_patterns, + root ) ] for path in dirs: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): + exclude_patterns, include_patterns, root): paths.append(os.path.join(parent, path)) for path in files: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): + exclude_patterns, include_patterns, root): paths.append(os.path.join(parent, path)) return paths def match_path(path, pattern): + pattern = pattern.rstrip('/' + os.path.sep) - if pattern: + if pattern and not os.path.isabs(pattern): pattern = os.path.relpath(pattern) pattern_components = pattern.split(os.path.sep) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index eedcf71a0..e144b7b15 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -876,6 +876,13 @@ def test_trailing_double_wildcard(self): ) ) + def test_exclude_include_absolute_path(self): + base = make_tree([], ['a.py', 'b.py']) + assert exclude_paths( + base, + ['/*', '!' + os.path.join(base, '*.py')] + ) == set(['a.py', 'b.py']) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): @@ -1026,52 +1033,52 @@ class ShouldCheckDirectoryTest(unittest.TestCase): def test_should_check_directory_not_excluded(self): assert should_check_directory( - 'not_excluded', self.exclude_patterns, self.include_patterns + 'not_excluded', self.exclude_patterns, self.include_patterns, '.' ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_shoud_check_parent_directories_of_excluded(self): assert should_check_directory( - 'dir', self.exclude_patterns, self.include_patterns + 'dir', self.exclude_patterns, self.include_patterns, '.' ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_should_not_check_excluded_directories_with_no_exceptions(self): assert not should_check_directory( 'exclude_rather_large_directory', self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) assert not should_check_directory( convert_path('dir/with/subdir_excluded'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_should_check_excluded_directory_with_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) assert should_check_directory( convert_path('dir/with/exceptions/in'), self.exclude_patterns, - self.include_patterns + self.include_patterns, '.' ) def test_should_not_check_siblings_of_exceptions(self): assert not should_check_directory( convert_path('dir/with/exceptions/but_not_here'), - self.exclude_patterns, self.include_patterns + self.exclude_patterns, self.include_patterns, '.' ) def test_should_check_subdirectories_of_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions/like_this_one/subdir'), - self.exclude_patterns, self.include_patterns + self.exclude_patterns, self.include_patterns, '.' ) From 48e45afe88f89a60401e3dfb7af69080204e6077 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Feb 2018 16:28:06 -0800 Subject: [PATCH 06/22] Add support for device_cgroup_rules parameter in host config Signed-off-by: Joffrey F --- docker/api/container.py | 2 ++ docker/models/containers.py | 3 +++ docker/types/containers.py | 12 +++++++++++- tests/integration/api_container_test.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/api/container.py b/docker/api/container.py index 962d8cb91..8994129a3 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -438,6 +438,8 @@ def create_host_config(self, *args, **kwargs): ``0,1``). cpuset_mems (str): Memory nodes (MEMs) in which to allow execution (``0-3``, ``0,1``). Only effective on NUMA systems. + device_cgroup_rules (:py:class:`list`): A list of cgroup rules to + apply to the container. device_read_bps: Limit read rate (bytes per second) from a device in the form of: `[{"Path": "device_path", "Rate": rate}]` device_read_iops: Limit read rate (IO per second) from a device. diff --git a/docker/models/containers.py b/docker/models/containers.py index 107a0204b..84dfa484e 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -515,6 +515,8 @@ def run(self, image, command=None, stdout=True, stderr=False, (``0-3``, ``0,1``). Only effective on NUMA systems. detach (bool): Run container in the background and return a :py:class:`Container` object. + device_cgroup_rules (:py:class:`list`): A list of cgroup rules to + apply to the container. device_read_bps: Limit read rate (bytes per second) from a device in the form of: `[{"Path": "device_path", "Rate": rate}]` device_read_iops: Limit read rate (IO per second) from a device. @@ -912,6 +914,7 @@ def prune(self, filters=None): 'cpuset_mems', 'cpu_rt_period', 'cpu_rt_runtime', + 'device_cgroup_rules', 'device_read_bps', 'device_read_iops', 'device_write_bps', diff --git a/docker/types/containers.py b/docker/types/containers.py index b4a329c22..252142073 100644 --- a/docker/types/containers.py +++ b/docker/types/containers.py @@ -120,7 +120,8 @@ def __init__(self, version, binds=None, port_bindings=None, init=None, init_path=None, volume_driver=None, cpu_count=None, cpu_percent=None, nano_cpus=None, cpuset_mems=None, runtime=None, mounts=None, - cpu_rt_period=None, cpu_rt_runtime=None): + cpu_rt_period=None, cpu_rt_runtime=None, + device_cgroup_rules=None): if mem_limit is not None: self['Memory'] = parse_bytes(mem_limit) @@ -466,6 +467,15 @@ def __init__(self, version, binds=None, port_bindings=None, raise host_config_version_error('mounts', '1.30') self['Mounts'] = mounts + if device_cgroup_rules is not None: + if version_lt(version, '1.28'): + raise host_config_version_error('device_cgroup_rules', '1.28') + if not isinstance(device_cgroup_rules, list): + raise host_config_type_error( + 'device_cgroup_rules', device_cgroup_rules, 'list' + ) + self['DeviceCgroupRules'] = device_cgroup_rules + def host_config_type_error(param, param_value, expected): error_msg = 'Invalid type for {0} param: expected {1} but found {2}' diff --git a/tests/integration/api_container_test.py b/tests/integration/api_container_test.py index 01780a771..8447aa5f0 100644 --- a/tests/integration/api_container_test.py +++ b/tests/integration/api_container_test.py @@ -474,6 +474,21 @@ def test_create_with_cpu_rt_options(self): assert config['HostConfig']['CpuRealtimeRuntime'] == 500 assert config['HostConfig']['CpuRealtimePeriod'] == 1000 + @requires_api_version('1.28') + def test_create_with_device_cgroup_rules(self): + rule = 'c 7:128 rwm' + ctnr = self.client.create_container( + BUSYBOX, 'cat /sys/fs/cgroup/devices/devices.list', + host_config=self.client.create_host_config( + device_cgroup_rules=[rule] + ) + ) + self.tmp_containers.append(ctnr) + config = self.client.inspect_container(ctnr) + assert config['HostConfig']['DeviceCgroupRules'] == [rule] + self.client.start(ctnr) + assert rule in self.client.logs(ctnr).decode('utf-8') + class VolumeBindTest(BaseAPIIntegrationTest): def setUp(self): From 3498b63fb0b9d2fd5a7f1f42e6c6dde772e055ce Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Feb 2018 18:55:56 -0800 Subject: [PATCH 07/22] Fix authconfig resolution when credStore is used combined with login() Signed-off-by: Joffrey F --- docker/auth.py | 5 ++++- tests/unit/auth_test.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docker/auth.py b/docker/auth.py index 91be2b850..48fcd8b50 100644 --- a/docker/auth.py +++ b/docker/auth.py @@ -90,9 +90,12 @@ def resolve_authconfig(authconfig, registry=None): log.debug( 'Using credentials store "{0}"'.format(store_name) ) - return _resolve_authconfig_credstore( + cfg = _resolve_authconfig_credstore( authconfig, registry, store_name ) + if cfg is not None: + return cfg + log.debug('No entry in credstore - fetching from auth dict') # Default to the public index server registry = resolve_index_name(registry) if registry else INDEX_NAME diff --git a/tests/unit/auth_test.py b/tests/unit/auth_test.py index d6981cd9d..ee32ca08a 100644 --- a/tests/unit/auth_test.py +++ b/tests/unit/auth_test.py @@ -210,6 +210,19 @@ def test_resolve_registry_and_auth_unauthenticated_registry(self): self.auth_config, auth.resolve_repository_name(image)[0] ) is None + def test_resolve_auth_with_empty_credstore_and_auth_dict(self): + auth_config = { + 'auths': auth.parse_auth({ + 'https://index.docker.io/v1/': self.index_config, + }), + 'credsStore': 'blackbox' + } + with mock.patch('docker.auth._resolve_authconfig_credstore') as m: + m.return_value = None + assert 'indexuser' == auth.resolve_authconfig( + auth_config, None + )['username'] + class CredStoreTest(unittest.TestCase): def test_get_credential_store(self): From cbbc37ac7b508d1a84ad68fa043f25f99d17b602 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 14 Feb 2018 13:01:16 +0000 Subject: [PATCH 08/22] Clean up created volume from test_run_with_named_volume This fix adds the volume id to the list so that it could be cleaned up on test teardown. The issue was originally from https://github.com/moby/moby/pull/36292 where an additional `somevolume` pre-exists in tests. Signed-off-by: Yong Tang --- tests/integration/models_containers_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/models_containers_test.py b/tests/integration/models_containers_test.py index a4d9f9c0d..f9f59c43b 100644 --- a/tests/integration/models_containers_test.py +++ b/tests/integration/models_containers_test.py @@ -55,7 +55,8 @@ def test_run_with_volume(self): def test_run_with_named_volume(self): client = docker.from_env(version=TEST_API_VERSION) - client.volumes.create(name="somevolume") + volume = client.volumes.create(name="somevolume") + self.tmp_volumes.append(volume.id) container = client.containers.run( "alpine", "sh -c 'echo \"hello\" > /insidecontainer/test'", From 15ae1f09f88b943aba35fb45f97e2d86de92a8ce Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Feb 2018 16:02:04 -0800 Subject: [PATCH 09/22] Bump docker-pycreds to 0.2.2 (pass support) Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1602750fd..2b281ae82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ asn1crypto==0.22.0 backports.ssl-match-hostname==3.5.0.1 cffi==1.10.0 cryptography==1.9 -docker-pycreds==0.2.1 +docker-pycreds==0.2.2 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 diff --git a/setup.py b/setup.py index b628f4a87..271d94f26 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ 'requests >= 2.14.2, != 2.18.0', 'six >= 1.4.0', 'websocket-client >= 0.32.0', - 'docker-pycreds >= 0.2.1' + 'docker-pycreds >= 0.2.2' ] extras_require = { From 581ccc9f7e8e189248054268c98561ca775bd3d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Feb 2018 15:17:03 -0800 Subject: [PATCH 10/22] Add chunk_size parameter to data downloading methods (export, get_archive, save) Signed-off-by: Joffrey F --- docker/api/client.py | 6 +++--- docker/api/container.py | 15 +++++++++++---- docker/api/image.py | 8 ++++++-- docker/constants.py | 1 + docker/models/containers.py | 17 +++++++++++++---- docker/models/images.py | 10 ++++++++-- tests/unit/models_containers_test.py | 9 +++++++-- tests/unit/models_images_test.py | 5 ++++- 8 files changed, 53 insertions(+), 18 deletions(-) diff --git a/docker/api/client.py b/docker/api/client.py index e69d143b2..bddab61f3 100644 --- a/docker/api/client.py +++ b/docker/api/client.py @@ -350,10 +350,10 @@ def _multiplexed_response_stream_helper(self, response): break yield data - def _stream_raw_result(self, response): - ''' Stream result for TTY-enabled container ''' + def _stream_raw_result(self, response, chunk_size=1, decode=True): + ''' Stream result for TTY-enabled container and raw binary data''' self._raise_for_status(response) - for out in response.iter_content(chunk_size=1, decode_unicode=True): + for out in response.iter_content(chunk_size, decode): yield out def _read_from_socket(self, response, stream, tty=False): diff --git a/docker/api/container.py b/docker/api/container.py index 962d8cb91..e986cf22c 100644 --- a/docker/api/container.py +++ b/docker/api/container.py @@ -3,6 +3,7 @@ from .. import errors from .. import utils +from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..types import ( ContainerConfig, EndpointConfig, HostConfig, NetworkingConfig ) @@ -643,12 +644,15 @@ def diff(self, container): ) @utils.check_resource('container') - def export(self, container): + def export(self, container, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Export the contents of a filesystem as a tar archive. Args: container (str): The container to export + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (generator): The archived filesystem data stream @@ -660,10 +664,10 @@ def export(self, container): res = self._get( self._url("/containers/{0}/export", container), stream=True ) - return self._stream_raw_result(res) + return self._stream_raw_result(res, chunk_size, False) @utils.check_resource('container') - def get_archive(self, container, path): + def get_archive(self, container, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Retrieve a file or folder from a container in the form of a tar archive. @@ -671,6 +675,9 @@ def get_archive(self, container, path): Args: container (str): The container where the file is located path (str): Path to the file or folder to retrieve + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (tuple): First element is a raw tar data stream. Second element is @@ -688,7 +695,7 @@ def get_archive(self, container, path): self._raise_for_status(res) encoded_stat = res.headers.get('x-docker-container-path-stat') return ( - self._stream_raw_result(res), + self._stream_raw_result(res, chunk_size, False), utils.decode_json_header(encoded_stat) if encoded_stat else None ) diff --git a/docker/api/image.py b/docker/api/image.py index fa832a389..3ebca32e5 100644 --- a/docker/api/image.py +++ b/docker/api/image.py @@ -4,6 +4,7 @@ import six from .. import auth, errors, utils +from ..constants import DEFAULT_DATA_CHUNK_SIZE log = logging.getLogger(__name__) @@ -11,12 +12,15 @@ class ImageApiMixin(object): @utils.check_resource('image') - def get_image(self, image): + def get_image(self, image, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Get a tarball of an image. Similar to the ``docker save`` command. Args: image (str): Image name to get + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (generator): A stream of raw archive data. @@ -34,7 +38,7 @@ def get_image(self, image): >>> f.close() """ res = self._get(self._url("/images/{0}/get", image), stream=True) - return self._stream_raw_result(res) + return self._stream_raw_result(res, chunk_size, False) @utils.check_resource('image') def history(self, image): diff --git a/docker/constants.py b/docker/constants.py index 9ab367325..7565a7688 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -17,3 +17,4 @@ DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version) DEFAULT_NUM_POOLS = 25 +DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048 diff --git a/docker/models/containers.py b/docker/models/containers.py index 107a0204b..b6e34dd24 100644 --- a/docker/models/containers.py +++ b/docker/models/containers.py @@ -3,6 +3,7 @@ from collections import namedtuple from ..api import APIClient +from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import (ContainerError, ImageNotFound, create_unexpected_kwargs_error) from ..types import HostConfig @@ -181,10 +182,15 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False, exec_output ) - def export(self): + def export(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Export the contents of the container's filesystem as a tar archive. + Args: + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB + Returns: (str): The filesystem tar archive @@ -192,15 +198,18 @@ def export(self): :py:class:`docker.errors.APIError` If the server returns an error. """ - return self.client.api.export(self.id) + return self.client.api.export(self.id, chunk_size) - def get_archive(self, path): + def get_archive(self, path, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Retrieve a file or folder from the container in the form of a tar archive. Args: path (str): Path to the file or folder to retrieve + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB Returns: (tuple): First element is a raw tar data stream. Second element is @@ -210,7 +219,7 @@ def get_archive(self, path): :py:class:`docker.errors.APIError` If the server returns an error. """ - return self.client.api.get_archive(self.id, path) + return self.client.api.get_archive(self.id, path, chunk_size) def kill(self, signal=None): """ diff --git a/docker/models/images.py b/docker/models/images.py index 0f3c71ab4..d604f7c7f 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -4,6 +4,7 @@ import six from ..api import APIClient +from ..constants import DEFAULT_DATA_CHUNK_SIZE from ..errors import BuildError, ImageLoadError from ..utils import parse_repository_tag from ..utils.json_stream import json_stream @@ -58,10 +59,15 @@ def history(self): """ return self.client.api.history(self.id) - def save(self): + def save(self, chunk_size=DEFAULT_DATA_CHUNK_SIZE): """ Get a tarball of an image. Similar to the ``docker save`` command. + Args: + chunk_size (int): The number of bytes returned by each iteration + of the generator. If ``None``, data will be streamed as it is + received. Default: 2 MB + Returns: (generator): A stream of raw archive data. @@ -77,7 +83,7 @@ def save(self): >>> f.write(chunk) >>> f.close() """ - return self.client.api.get_image(self.id) + return self.client.api.get_image(self.id, chunk_size) def tag(self, repository, tag=None, **kwargs): """ diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py index f79f5d5b3..2b0b499ef 100644 --- a/tests/unit/models_containers_test.py +++ b/tests/unit/models_containers_test.py @@ -1,4 +1,5 @@ import docker +from docker.constants import DEFAULT_DATA_CHUNK_SIZE from docker.models.containers import Container, _create_container_args from docker.models.images import Image import unittest @@ -422,13 +423,17 @@ def test_export(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) container.export() - client.api.export.assert_called_with(FAKE_CONTAINER_ID) + client.api.export.assert_called_with( + FAKE_CONTAINER_ID, DEFAULT_DATA_CHUNK_SIZE + ) def test_get_archive(self): client = make_fake_client() container = client.containers.get(FAKE_CONTAINER_ID) container.get_archive('foo') - client.api.get_archive.assert_called_with(FAKE_CONTAINER_ID, 'foo') + client.api.get_archive.assert_called_with( + FAKE_CONTAINER_ID, 'foo', DEFAULT_DATA_CHUNK_SIZE + ) def test_image(self): client = make_fake_client() diff --git a/tests/unit/models_images_test.py b/tests/unit/models_images_test.py index dacd72be0..67832795f 100644 --- a/tests/unit/models_images_test.py +++ b/tests/unit/models_images_test.py @@ -1,3 +1,4 @@ +from docker.constants import DEFAULT_DATA_CHUNK_SIZE from docker.models.images import Image import unittest @@ -116,7 +117,9 @@ def test_save(self): client = make_fake_client() image = client.images.get(FAKE_IMAGE_ID) image.save() - client.api.get_image.assert_called_with(FAKE_IMAGE_ID) + client.api.get_image.assert_called_with( + FAKE_IMAGE_ID, DEFAULT_DATA_CHUNK_SIZE + ) def test_tag(self): client = make_fake_client() From 4c708f568c55ede2396b01aebf607374f731b3f2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Feb 2018 16:22:33 -0800 Subject: [PATCH 11/22] Fix test_login flakes Signed-off-by: Joffrey F --- tests/unit/api_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/api_test.py b/tests/unit/api_test.py index f65e13ecb..61d24460e 100644 --- a/tests/unit/api_test.py +++ b/tests/unit/api_test.py @@ -214,12 +214,13 @@ def test_search(self): def test_login(self): self.client.login('sakuya', 'izayoi') - fake_request.assert_called_with( - 'POST', url_prefix + 'auth', - data=json.dumps({'username': 'sakuya', 'password': 'izayoi'}), - timeout=DEFAULT_TIMEOUT_SECONDS, - headers={'Content-Type': 'application/json'} - ) + args = fake_request.call_args + assert args[0][0] == 'POST' + assert args[0][1] == url_prefix + 'auth' + assert json.loads(args[1]['data']) == { + 'username': 'sakuya', 'password': 'izayoi' + } + assert args[1]['headers'] == {'Content-Type': 'application/json'} assert self.client._auth_configs['auths'] == { 'docker.io': { From 181c1c8eb970ae11a707b6b6c3d1e4d546504ccf Mon Sep 17 00:00:00 2001 From: mefyl Date: Fri, 16 Feb 2018 11:03:35 +0100 Subject: [PATCH 12/22] Revert "Correctly support absolute paths in .dockerignore" This reverts commit 34d50483e20e86cb7ab22700e036a5c4d319268a. Signed-off-by: mefyl --- docker/utils/build.py | 20 +++++++------------- tests/unit/utils_test.py | 27 ++++++++++----------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index a21887349..d4223e749 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -46,7 +46,7 @@ def exclude_paths(root, patterns, dockerfile=None): ) -def should_include(path, exclude_patterns, include_patterns, root): +def should_include(path, exclude_patterns, include_patterns): """ Given a path, a list of exclude patterns, and a list of inclusion patterns: @@ -61,15 +61,11 @@ def should_include(path, exclude_patterns, include_patterns, root): for pattern in include_patterns: if match_path(path, pattern): return True - if os.path.isabs(pattern) and match_path( - os.path.join(root, path), pattern): - return True return False return True -def should_check_directory(directory_path, exclude_patterns, include_patterns, - root): +def should_check_directory(directory_path, exclude_patterns, include_patterns): """ Given a directory path, a list of exclude patterns, and a list of inclusion patterns: @@ -95,7 +91,7 @@ def normalize_path(path): if (pattern + '/').startswith(path_with_slash) ] directory_included = should_include( - directory_path, exclude_patterns, include_patterns, root + directory_path, exclude_patterns, include_patterns ) return directory_included or len(possible_child_patterns) > 0 @@ -114,28 +110,26 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): # traversal. See https://docs.python.org/2/library/os.html#os.walk dirs[:] = [ d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns, - root + os.path.join(parent, d), exclude_patterns, include_patterns ) ] for path in dirs: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns, root): + exclude_patterns, include_patterns): paths.append(os.path.join(parent, path)) for path in files: if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns, root): + exclude_patterns, include_patterns): paths.append(os.path.join(parent, path)) return paths def match_path(path, pattern): - pattern = pattern.rstrip('/' + os.path.sep) - if pattern and not os.path.isabs(pattern): + if pattern: pattern = os.path.relpath(pattern) pattern_components = pattern.split(os.path.sep) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index e144b7b15..eedcf71a0 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -876,13 +876,6 @@ def test_trailing_double_wildcard(self): ) ) - def test_exclude_include_absolute_path(self): - base = make_tree([], ['a.py', 'b.py']) - assert exclude_paths( - base, - ['/*', '!' + os.path.join(base, '*.py')] - ) == set(['a.py', 'b.py']) - class TarTest(unittest.TestCase): def test_tar_with_excludes(self): @@ -1033,52 +1026,52 @@ class ShouldCheckDirectoryTest(unittest.TestCase): def test_should_check_directory_not_excluded(self): assert should_check_directory( - 'not_excluded', self.exclude_patterns, self.include_patterns, '.' + 'not_excluded', self.exclude_patterns, self.include_patterns ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_shoud_check_parent_directories_of_excluded(self): assert should_check_directory( - 'dir', self.exclude_patterns, self.include_patterns, '.' + 'dir', self.exclude_patterns, self.include_patterns ) assert should_check_directory( convert_path('dir/with'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_should_not_check_excluded_directories_with_no_exceptions(self): assert not should_check_directory( 'exclude_rather_large_directory', self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) assert not should_check_directory( convert_path('dir/with/subdir_excluded'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_should_check_excluded_directory_with_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) assert should_check_directory( convert_path('dir/with/exceptions/in'), self.exclude_patterns, - self.include_patterns, '.' + self.include_patterns ) def test_should_not_check_siblings_of_exceptions(self): assert not should_check_directory( convert_path('dir/with/exceptions/but_not_here'), - self.exclude_patterns, self.include_patterns, '.' + self.exclude_patterns, self.include_patterns ) def test_should_check_subdirectories_of_exceptions(self): assert should_check_directory( convert_path('dir/with/exceptions/like_this_one/subdir'), - self.exclude_patterns, self.include_patterns, '.' + self.exclude_patterns, self.include_patterns ) From cc455d7fd5ac8d192e64965f68ecea74dca011de Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Feb 2018 14:51:49 -0800 Subject: [PATCH 13/22] Fix DockerClient pull bug when pulling image by digest Signed-off-by: Joffrey F --- docker/models/images.py | 4 +++- tests/integration/models_images_test.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/models/images.py b/docker/models/images.py index d604f7c7f..58d5d93c4 100644 --- a/docker/models/images.py +++ b/docker/models/images.py @@ -314,7 +314,9 @@ def pull(self, repository, tag=None, **kwargs): self.client.api.pull(repository, tag=tag, **kwargs) if tag: - return self.get('{0}:{1}'.format(repository, tag)) + return self.get('{0}{2}{1}'.format( + repository, tag, '@' if tag.startswith('sha256:') else ':' + )) return self.list(repository) def push(self, repository, tag=None, **kwargs): diff --git a/tests/integration/models_images_test.py b/tests/integration/models_images_test.py index 2fa71a794..ae735baaf 100644 --- a/tests/integration/models_images_test.py +++ b/tests/integration/models_images_test.py @@ -74,6 +74,15 @@ def test_pull_with_tag(self): image = client.images.pull('alpine', tag='3.3') assert 'alpine:3.3' in image.attrs['RepoTags'] + def test_pull_with_sha(self): + image_ref = ( + 'hello-world@sha256:083de497cff944f969d8499ab94f07134c50bcf5e6b95' + '59b27182d3fa80ce3f7' + ) + client = docker.from_env(version=TEST_API_VERSION) + image = client.images.pull(image_ref) + assert image_ref in image.attrs['RepoDigests'] + def test_pull_multiple(self): client = docker.from_env(version=TEST_API_VERSION) images = client.images.pull('hello-world') From 820de848fa73f20cb80215ceb4b8cdafc855867e Mon Sep 17 00:00:00 2001 From: William Myers Date: Fri, 16 Feb 2018 22:29:21 -0700 Subject: [PATCH 14/22] Add support for generic resources to docker.types.Resources - Add support for dict and list generic_resources parameter - Add generic_resources integration test Signed-off-by: William Myers --- docker/types/services.py | 34 +++++++++++++++++- tests/integration/api_service_test.py | 51 +++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/docker/types/services.py b/docker/types/services.py index d530e61db..69e0e0240 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -306,9 +306,14 @@ class Resources(dict): mem_limit (int): Memory limit in Bytes. cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. mem_reservation (int): Memory reservation in Bytes. + generic_resources (dict:`list`): List of node level generic + resources, for example a GPU, in the form of + ``{ ResourceSpec: { 'Kind': kind, 'Value': value }}``, where + ResourceSpec is one of 'DiscreteResourceSpec' or 'NamedResourceSpec' + or in the form of ``{ resource_name: resource_value }``. """ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, - mem_reservation=None): + mem_reservation=None, generic_resources=None): limits = {} reservation = {} if cpu_limit is not None: @@ -319,6 +324,33 @@ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, reservation['NanoCPUs'] = cpu_reservation if mem_reservation is not None: reservation['MemoryBytes'] = mem_reservation + if generic_resources is not None: + # if isinstance(generic_resources, list): + # reservation['GenericResources'] = generic_resources + if isinstance(generic_resources, (list, tuple)): + reservation['GenericResources'] = list(generic_resources) + elif isinstance(generic_resources, dict): + resources = [] + for kind, value in six.iteritems(generic_resources): + resource_type = None + if isinstance(value, int): + resource_type = 'DiscreteResourceSpec' + elif isinstance(value, str): + resource_type = 'NamedResourceSpec' + else: + raise errors.InvalidArgument( + 'Unsupported generic resource reservation ' + 'type: {}'.format({kind: value}) + ) + resources.append({ + resource_type: {'Kind': kind, 'Value': value} + }) + reservation['GenericResources'] = resources + else: + raise errors.InvalidArgument( + 'Unsupported generic resources ' + 'type: {}'.format(generic_resources) + ) if limits: self['Limits'] = limits diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 5cc3fc190..07a34b95a 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -212,6 +212,57 @@ def test_create_service_with_resources_constraints(self): 'Reservations' ] + def _create_service_with_generic_resources(self, generic_resources): + container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) + + resources = docker.types.Resources( + generic_resources=generic_resources + ) + task_tmpl = docker.types.TaskTemplate( + container_spec, resources=resources + ) + name = self.get_service_name() + svc_id = self.client.create_service(task_tmpl, name=name) + return resources, self.client.inspect_service(svc_id) + + def test_create_service_with_generic_resources(self): + successful = [{ + 'input': [ + {'DiscreteResourceSpec': {'Kind': 'gpu', 'Value': 1}}, + {'NamedResourceSpec': {'Kind': 'gpu', 'Value': 'test'}} + ]}, { + 'input': {'gpu': 2, 'mpi': 'latest'}, + 'expected': [ + {'DiscreteResourceSpec': {'Kind': 'gpu', 'Value': 2}}, + {'NamedResourceSpec': {'Kind': 'mpi', 'Value': 'latest'}} + ]} + ] + + for test in successful: + t = test['input'] + resrcs, svc_info = self._create_service_with_generic_resources(t) + + assert 'TaskTemplate' in svc_info['Spec'] + res_template = svc_info['Spec']['TaskTemplate'] + assert 'Resources' in res_template + res_reservations = res_template['Resources']['Reservations'] + assert res_reservations == resrcs['Reservations'] + assert 'GenericResources' in res_reservations + + def _key(d, specs=('DiscreteResourceSpec', 'NamedResourceSpec')): + return [d.get(s, {}).get('Kind', '') for s in specs] + + actual = res_reservations['GenericResources'] + expected = test.get('expected', test['input']) + assert sorted(actual, key=_key) == sorted(expected, key=_key) + + for test_input in ['1', 1.0, lambda: '1', {1, 2}]: + try: + self._create_service_with_generic_resources(test_input) + self.fail('Should fail: {}'.format(test_input)) + except docker.errors.InvalidArgument: + pass + def test_create_service_with_update_config(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) task_tmpl = docker.types.TaskTemplate(container_spec) From 9b6b306e173d6b1f8a8fee781332f37735a12573 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Feb 2018 16:25:08 -0800 Subject: [PATCH 15/22] Code cleanup and version guards Signed-off-by: Joffrey F --- docker/api/service.py | 5 +++ docker/types/services.py | 65 ++++++++++++++------------- tests/integration/api_service_test.py | 8 ++-- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index ceae8fc9a..95fb07e47 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -73,6 +73,11 @@ def raise_version_error(param, min_version): if container_spec.get('Isolation') is not None: raise_version_error('ContainerSpec.isolation', '1.35') + if task_template.get('Resources'): + if utils.version_lt(version, '1.35'): + if task_template['Resources'].get('GenericResources'): + raise_version_error('Resources.generic_resources', '1.35') + def _merge_task_template(current, override): merged = current.copy() diff --git a/docker/types/services.py b/docker/types/services.py index 69e0e0240..09eb05edb 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -306,11 +306,10 @@ class Resources(dict): mem_limit (int): Memory limit in Bytes. cpu_reservation (int): CPU reservation in units of 10^9 CPU shares. mem_reservation (int): Memory reservation in Bytes. - generic_resources (dict:`list`): List of node level generic - resources, for example a GPU, in the form of - ``{ ResourceSpec: { 'Kind': kind, 'Value': value }}``, where - ResourceSpec is one of 'DiscreteResourceSpec' or 'NamedResourceSpec' - or in the form of ``{ resource_name: resource_value }``. + generic_resources (dict or :py:class:`list`): Node level generic + resources, for example a GPU, using the following format: + ``{ resource_name: resource_value }``. Alternatively, a list of + of resource specifications as defined by the Engine API. """ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, mem_reservation=None, generic_resources=None): @@ -325,39 +324,41 @@ def __init__(self, cpu_limit=None, mem_limit=None, cpu_reservation=None, if mem_reservation is not None: reservation['MemoryBytes'] = mem_reservation if generic_resources is not None: - # if isinstance(generic_resources, list): - # reservation['GenericResources'] = generic_resources - if isinstance(generic_resources, (list, tuple)): - reservation['GenericResources'] = list(generic_resources) - elif isinstance(generic_resources, dict): - resources = [] - for kind, value in six.iteritems(generic_resources): - resource_type = None - if isinstance(value, int): - resource_type = 'DiscreteResourceSpec' - elif isinstance(value, str): - resource_type = 'NamedResourceSpec' - else: - raise errors.InvalidArgument( - 'Unsupported generic resource reservation ' - 'type: {}'.format({kind: value}) - ) - resources.append({ - resource_type: {'Kind': kind, 'Value': value} - }) - reservation['GenericResources'] = resources - else: - raise errors.InvalidArgument( - 'Unsupported generic resources ' - 'type: {}'.format(generic_resources) - ) - + reservation['GenericResources'] = ( + _convert_generic_resources_dict(generic_resources) + ) if limits: self['Limits'] = limits if reservation: self['Reservations'] = reservation +def _convert_generic_resources_dict(generic_resources): + if isinstance(generic_resources, list): + return generic_resources + if not isinstance(generic_resources, dict): + raise errors.InvalidArgument( + 'generic_resources must be a dict or a list' + ' (found {})'.format(type(generic_resources)) + ) + resources = [] + for kind, value in six.iteritems(generic_resources): + resource_type = None + if isinstance(value, int): + resource_type = 'DiscreteResourceSpec' + elif isinstance(value, str): + resource_type = 'NamedResourceSpec' + else: + raise errors.InvalidArgument( + 'Unsupported generic resource reservation ' + 'type: {}'.format({kind: value}) + ) + resources.append({ + resource_type: {'Kind': kind, 'Value': value} + }) + return resources + + class UpdateConfig(dict): """ diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 07a34b95a..9d91f9e01 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -4,6 +4,7 @@ import time import docker +import pytest import six from ..helpers import ( @@ -225,6 +226,7 @@ def _create_service_with_generic_resources(self, generic_resources): svc_id = self.client.create_service(task_tmpl, name=name) return resources, self.client.inspect_service(svc_id) + @requires_api_version('1.35') def test_create_service_with_generic_resources(self): successful = [{ 'input': [ @@ -256,12 +258,10 @@ def _key(d, specs=('DiscreteResourceSpec', 'NamedResourceSpec')): expected = test.get('expected', test['input']) assert sorted(actual, key=_key) == sorted(expected, key=_key) + def test_create_service_with_invalid_generic_resources(self): for test_input in ['1', 1.0, lambda: '1', {1, 2}]: - try: + with pytest.raises(docker.errors.InvalidArgument): self._create_service_with_generic_resources(test_input) - self.fail('Should fail: {}'.format(test_input)) - except docker.errors.InvalidArgument: - pass def test_create_service_with_update_config(self): container_spec = docker.types.ContainerSpec(BUSYBOX, ['true']) From 8fd9d3c99e9314323228af4832054b22d2ac4966 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Feb 2018 17:11:27 -0800 Subject: [PATCH 16/22] GenericResources was introduced in 1.32 Signed-off-by: Joffrey F --- docker/api/service.py | 4 ++-- tests/integration/api_service_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/api/service.py b/docker/api/service.py index 95fb07e47..03b0ca6ea 100644 --- a/docker/api/service.py +++ b/docker/api/service.py @@ -74,9 +74,9 @@ def raise_version_error(param, min_version): raise_version_error('ContainerSpec.isolation', '1.35') if task_template.get('Resources'): - if utils.version_lt(version, '1.35'): + if utils.version_lt(version, '1.32'): if task_template['Resources'].get('GenericResources'): - raise_version_error('Resources.generic_resources', '1.35') + raise_version_error('Resources.generic_resources', '1.32') def _merge_task_template(current, override): diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 9d91f9e01..85f9dccf2 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -226,7 +226,7 @@ def _create_service_with_generic_resources(self, generic_resources): svc_id = self.client.create_service(task_tmpl, name=name) return resources, self.client.inspect_service(svc_id) - @requires_api_version('1.35') + @requires_api_version('1.32') def test_create_service_with_generic_resources(self): successful = [{ 'input': [ @@ -258,6 +258,7 @@ def _key(d, specs=('DiscreteResourceSpec', 'NamedResourceSpec')): expected = test.get('expected', test['input']) assert sorted(actual, key=_key) == sorted(expected, key=_key) + @requires_api_version('1.32') def test_create_service_with_invalid_generic_resources(self): for test_input in ['1', 1.0, lambda: '1', {1, 2}]: with pytest.raises(docker.errors.InvalidArgument): From c8f5a5ad4040560ce62d53002ecec12485b531f7 Mon Sep 17 00:00:00 2001 From: mefyl Date: Fri, 16 Feb 2018 11:22:29 +0100 Subject: [PATCH 17/22] Fix dockerignore handling of absolute path exceptions. Signed-off-by: mefyl --- docker/utils/build.py | 6 +++--- tests/unit/utils_test.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index d4223e749..e86a04e61 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -26,13 +26,13 @@ def exclude_paths(root, patterns, dockerfile=None): if dockerfile is None: dockerfile = 'Dockerfile' - patterns = [p.lstrip('/') for p in patterns] exceptions = [p for p in patterns if p.startswith('!')] - include_patterns = [p[1:] for p in exceptions] + include_patterns = [p[1:].lstrip('/') for p in exceptions] include_patterns += [dockerfile, '.dockerignore'] - exclude_patterns = list(set(patterns) - set(exceptions)) + exclude_patterns = [ + p.lstrip('/') for p in list(set(patterns) - set(exceptions))] paths = get_paths(root, exclude_patterns, include_patterns, has_exceptions=len(exceptions) > 0) diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index eedcf71a0..0ee041ae9 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -758,6 +758,13 @@ def test_single_subdir_single_filename_leading_slash(self): self.all_paths - set(['foo/a.py']) ) + def test_exclude_include_absolute_path(self): + base = make_tree([], ['a.py', 'b.py']) + assert exclude_paths( + base, + ['/*', '!/*.py'] + ) == set(['a.py', 'b.py']) + def test_single_subdir_with_path_traversal(self): assert self.exclude(['foo/whoops/../a.py']) == convert_paths( self.all_paths - set(['foo/a.py']) From bb3ad64060b1f1136a6a42ed4d2018f71ebd371d Mon Sep 17 00:00:00 2001 From: mefyl Date: Fri, 16 Feb 2018 16:08:11 +0100 Subject: [PATCH 18/22] Fix .dockerignore: accept wildcard in inclusion pattern, honor last line precedence. Signed-off-by: mefyl --- docker/utils/build.py | 188 +++++++++++++++------------------------ tests/unit/utils_test.py | 84 +++++------------ 2 files changed, 94 insertions(+), 178 deletions(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index e86a04e61..bfdb87b2f 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,20 +1,24 @@ import os +import re from ..constants import IS_WINDOWS_PLATFORM -from .fnmatch import fnmatch +from fnmatch import fnmatch +from itertools import chain from .utils import create_archive def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): root = os.path.abspath(path) exclude = exclude or [] - return create_archive( files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), root=root, fileobj=fileobj, gzip=gzip ) +_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') + + def exclude_paths(root, patterns, dockerfile=None): """ Given a root directory path and a list of .dockerignore patterns, return @@ -23,121 +27,77 @@ def exclude_paths(root, patterns, dockerfile=None): All paths returned are relative to the root. """ + if dockerfile is None: dockerfile = 'Dockerfile' - exceptions = [p for p in patterns if p.startswith('!')] - - include_patterns = [p[1:].lstrip('/') for p in exceptions] - include_patterns += [dockerfile, '.dockerignore'] - - exclude_patterns = [ - p.lstrip('/') for p in list(set(patterns) - set(exceptions))] - - paths = get_paths(root, exclude_patterns, include_patterns, - has_exceptions=len(exceptions) > 0) - - return set(paths).union( - # If the Dockerfile is in a subdirectory that is excluded, get_paths - # will not descend into it and the file will be skipped. This ensures - # it doesn't happen. - set([dockerfile.replace('/', os.path.sep)]) - if os.path.exists(os.path.join(root, dockerfile)) else set() - ) - - -def should_include(path, exclude_patterns, include_patterns): - """ - Given a path, a list of exclude patterns, and a list of inclusion patterns: - - 1. Returns True if the path doesn't match any exclusion pattern - 2. Returns False if the path matches an exclusion pattern and doesn't match - an inclusion pattern - 3. Returns true if the path matches an exclusion pattern and matches an - inclusion pattern + def normalize(p): + # Leading and trailing slashes are not relevant. Yes, + # "foo.py/" must exclude the "foo.py" regular file. "." + # components are not relevant either, even if the whole + # pattern is only ".", as the Docker reference states: "For + # historical reasons, the pattern . is ignored." + split = [pt for pt in re.split(_SEP, p) if pt and pt != '.'] + # ".." component must be cleared with the potential previous + # component, regardless of whether it exists: "A preprocessing + # step [...] eliminates . and .. elements using Go's + # filepath.". + i = 0 + while i < len(split): + if split[i] == '..': + del split[i] + if i > 0: + del split[i - 1] + i -= 1 + else: + i += 1 + return split + + patterns = ( + (True, normalize(p[1:])) + if p.startswith('!') else + (False, normalize(p)) + for p in patterns) + patterns = list(reversed(list(chain( + # Exclude empty patterns such as "." or the empty string. + filter(lambda p: p[1], patterns), + # Always include the Dockerfile and .dockerignore + [(True, dockerfile.split('/')), (True, ['.dockerignore'])])))) + return set(walk(root, patterns)) + + +def walk(root, patterns, default=True): """ - for pattern in exclude_patterns: - if match_path(path, pattern): - for pattern in include_patterns: - if match_path(path, pattern): - return True - return False - return True - - -def should_check_directory(directory_path, exclude_patterns, include_patterns): + A collection of file lying below root that should be included according to + patterns. """ - Given a directory path, a list of exclude patterns, and a list of inclusion - patterns: - - 1. Returns True if the directory path should be included according to - should_include. - 2. Returns True if the directory path is the prefix for an inclusion - pattern - 3. Returns False otherwise - """ - - # To account for exception rules, check directories if their path is a - # a prefix to an inclusion pattern. This logic conforms with the current - # docker logic (2016-10-27): - # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 - - def normalize_path(path): - return path.replace(os.path.sep, '/') - - path_with_slash = normalize_path(directory_path) + '/' - possible_child_patterns = [ - pattern for pattern in map(normalize_path, include_patterns) - if (pattern + '/').startswith(path_with_slash) - ] - directory_included = should_include( - directory_path, exclude_patterns, include_patterns - ) - return directory_included or len(possible_child_patterns) > 0 - - -def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): - paths = [] - - for parent, dirs, files in os.walk(root, topdown=True, followlinks=False): - parent = os.path.relpath(parent, root) - if parent == '.': - parent = '' - - # Remove excluded patterns from the list of directories to traverse - # by mutating the dirs we're iterating over. - # This looks strange, but is considered the correct way to skip - # traversal. See https://docs.python.org/2/library/os.html#os.walk - dirs[:] = [ - d for d in dirs if should_check_directory( - os.path.join(parent, d), exclude_patterns, include_patterns - ) - ] - - for path in dirs: - if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): - paths.append(os.path.join(parent, path)) - - for path in files: - if should_include(os.path.join(parent, path), - exclude_patterns, include_patterns): - paths.append(os.path.join(parent, path)) - - return paths - - -def match_path(path, pattern): - pattern = pattern.rstrip('/' + os.path.sep) - if pattern: - pattern = os.path.relpath(pattern) - - pattern_components = pattern.split(os.path.sep) - if len(pattern_components) == 1 and IS_WINDOWS_PLATFORM: - pattern_components = pattern.split('/') - if '**' not in pattern: - path_components = path.split(os.path.sep)[:len(pattern_components)] - else: - path_components = path.split(os.path.sep) - return fnmatch('/'.join(path_components), '/'.join(pattern_components)) + def match(p): + if p[1][0] == '**': + rec = (p[0], p[1][1:]) + return [p] + (match(rec) if rec[1] else [rec]) + elif fnmatch(f, p[1][0]): + return [(p[0], p[1][1:])] + else: + return [] + + for f in os.listdir(root): + cur = os.path.join(root, f) + # The patterns if recursing in that directory. + sub = list(chain(*(match(p) for p in patterns))) + # Whether this file is explicitely included / excluded. + hit = next((p[0] for p in sub if not p[1]), None) + # Whether this file is implicitely included / excluded. + matched = default if hit is None else hit + sub = list(filter(lambda p: p[1], sub)) + if os.path.isdir(cur): + children = False + for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): + yield r + children = True + # The current unit tests expect directories only under those + # conditions. It might be simplifiable though. + if (not sub or not children) and hit or hit is None and default: + yield f + elif matched: + yield f diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 0ee041ae9..8a4b1937e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -23,7 +23,6 @@ decode_json_header, tar, split_command, parse_devices, update_headers, ) -from docker.utils.build import should_check_directory from docker.utils.ports import build_port_bindings, split_port from docker.utils.utils import format_environment @@ -883,6 +882,26 @@ def test_trailing_double_wildcard(self): ) ) + def test_include_wildcard(self): + base = make_tree(['a'], ['a/b.py']) + assert exclude_paths( + base, + ['*', '!*/b.py'] + ) == convert_paths(['a/b.py']) + + def test_last_line_precedence(self): + base = make_tree( + [], + ['garbage.md', + 'thrash.md', + 'README.md', + 'README-bis.md', + 'README-secret.md']) + assert exclude_paths( + base, + ['*.md', '!README*.md', 'README-secret.md'] + ) == set(['README.md', 'README-bis.md']) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self): @@ -1019,69 +1038,6 @@ def tar_test_negative_mtime_bug(self): assert tar_data.getmember('th.txt').mtime == -3600 -class ShouldCheckDirectoryTest(unittest.TestCase): - exclude_patterns = [ - 'exclude_rather_large_directory', - 'dir/with/subdir_excluded', - 'dir/with/exceptions' - ] - - include_patterns = [ - 'dir/with/exceptions/like_this_one', - 'dir/with/exceptions/in/descendents' - ] - - def test_should_check_directory_not_excluded(self): - assert should_check_directory( - 'not_excluded', self.exclude_patterns, self.include_patterns - ) - assert should_check_directory( - convert_path('dir/with'), self.exclude_patterns, - self.include_patterns - ) - - def test_shoud_check_parent_directories_of_excluded(self): - assert should_check_directory( - 'dir', self.exclude_patterns, self.include_patterns - ) - assert should_check_directory( - convert_path('dir/with'), self.exclude_patterns, - self.include_patterns - ) - - def test_should_not_check_excluded_directories_with_no_exceptions(self): - assert not should_check_directory( - 'exclude_rather_large_directory', self.exclude_patterns, - self.include_patterns - ) - assert not should_check_directory( - convert_path('dir/with/subdir_excluded'), self.exclude_patterns, - self.include_patterns - ) - - def test_should_check_excluded_directory_with_exceptions(self): - assert should_check_directory( - convert_path('dir/with/exceptions'), self.exclude_patterns, - self.include_patterns - ) - assert should_check_directory( - convert_path('dir/with/exceptions/in'), self.exclude_patterns, - self.include_patterns - ) - - def test_should_not_check_siblings_of_exceptions(self): - assert not should_check_directory( - convert_path('dir/with/exceptions/but_not_here'), - self.exclude_patterns, self.include_patterns - ) - - def test_should_check_subdirectories_of_exceptions(self): - assert should_check_directory( - convert_path('dir/with/exceptions/like_this_one/subdir'), - self.exclude_patterns, self.include_patterns - ) - - class FormatEnvironmentTest(unittest.TestCase): def test_format_env_binary_unicode_value(self): env_dict = { From 3b464f983e77cf85d9bc91e6f725cd7773bc0871 Mon Sep 17 00:00:00 2001 From: mefyl Date: Mon, 19 Feb 2018 12:48:17 +0100 Subject: [PATCH 19/22] Skip entirely excluded directories when handling dockerignore. This is pure optimization to not recurse into directories when there are no chances any file will be included. Signed-off-by: mefyl --- docker/utils/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index bfdb87b2f..09b207170 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -91,6 +91,10 @@ def match(p): matched = default if hit is None else hit sub = list(filter(lambda p: p[1], sub)) if os.path.isdir(cur): + # Entirely skip directories if there are no chance any subfile will + # be included. + if all(not p[0] for p in sub) and not matched: + continue children = False for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): yield r From 0c948c7df65b0d7378c3c0c8d966c38171f1ef21 Mon Sep 17 00:00:00 2001 From: mefyl Date: Mon, 19 Feb 2018 12:54:04 +0100 Subject: [PATCH 20/22] Add note about potential dockerignore optimization. Signed-off-by: mefyl --- docker/utils/build.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/utils/build.py b/docker/utils/build.py index 09b207170..1da56fbcc 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -95,6 +95,15 @@ def match(p): # be included. if all(not p[0] for p in sub) and not matched: continue + # I think this would greatly speed up dockerignore handling by not + # recursing into directories we are sure would be entirely + # included, and only yielding the directory itself, which will be + # recursively archived anyway. However the current unit test expect + # the full list of subfiles and I'm not 100% sure it would make no + # difference yet. + # if all(p[0] for p in sub) and matched: + # yield f + # continue children = False for r in (os.path.join(f, p) for p in walk(cur, sub, matched)): yield r From e54e8f41993e5fd6378b15c5ab9a3d43615b8618 Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Wed, 21 Feb 2018 19:55:17 +0000 Subject: [PATCH 21/22] Shorthand method for service.force_update() Signed-off-by: Viktor Adam --- docker/models/services.py | 15 +++++++++ tests/integration/models_services_test.py | 41 +++++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docker/models/services.py b/docker/models/services.py index 8a633dfa0..125896bab 100644 --- a/docker/models/services.py +++ b/docker/models/services.py @@ -69,6 +69,11 @@ def update(self, **kwargs): spec = self.attrs['Spec']['TaskTemplate']['ContainerSpec'] kwargs['image'] = spec['Image'] + if kwargs.get('force_update') is True: + task_template = self.attrs['Spec']['TaskTemplate'] + current_value = int(task_template.get('ForceUpdate', 0)) + kwargs['force_update'] = current_value + 1 + create_kwargs = _get_create_service_kwargs('update', kwargs) return self.client.api.update_service( @@ -124,6 +129,16 @@ def scale(self, replicas): service_mode, fetch_current_spec=True) + def force_update(self): + """ + Force update the service even if no changes require it. + + Returns: + ``True``if successful. + """ + + return self.update(force_update=True, fetch_current_spec=True) + class ServiceCollection(Collection): """Services on the Docker server.""" diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index cb8eca29b..36caa8513 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -276,7 +276,7 @@ def test_scale_method_global_service(self): assert spec.get('Command') == ['sleep', '300'] @helpers.requires_api_version('1.25') - def test_restart_service(self): + def test_force_update_service(self): client = docker.from_env(version=TEST_API_VERSION) service = client.services.create( # create arguments @@ -286,7 +286,7 @@ def test_restart_service(self): command="sleep 300" ) initial_version = service.version - service.update( + assert service.update( # create argument name=service.name, # task template argument @@ -296,3 +296,40 @@ def test_restart_service(self): ) service.reload() assert service.version > initial_version + + @helpers.requires_api_version('1.25') + def test_force_update_service_using_bool(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + initial_version = service.version + assert service.update( + # create argument + name=service.name, + # task template argument + force_update=True, + # ContainerSpec argument + command="sleep 600" + ) + service.reload() + assert service.version > initial_version + + @helpers.requires_api_version('1.25') + def test_force_update_service_using_shorthand_method(self): + client = docker.from_env(version=TEST_API_VERSION) + service = client.services.create( + # create arguments + name=helpers.random_name(), + # ContainerSpec arguments + image="alpine", + command="sleep 300" + ) + initial_version = service.version + assert service.force_update() + service.reload() + assert service.version > initial_version From 1d85818f4caeba437d4200a6b8f870beb28ca5a2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Feb 2018 13:52:44 -0800 Subject: [PATCH 22/22] Bump 3.1.0 Signed-off-by: Joffrey F --- docker/version.py | 2 +- docs/change-log.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index fc5684061..c79cf93da 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "3.1.0-dev" +version = "3.1.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index 8ae88ef2a..ceab083ea 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,36 @@ Change log ========== +3.1.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/44?closed=1) + +### Features + +* Added support for `device_cgroup_rules` in host config +* Added support for `generic_resources` when creating a `Resources` + object. +* Added support for a configurable `chunk_size` parameter in `export`, + `get_archive` and `get_image` (`Image.save`) +* Added a `force_update` method to the `Service` class. +* In `Service.update`, when the `force_update` parameter is set to `True`, + the current `force_update` counter is incremented by one in the update + request. + +### Bugfixes + +* Fixed a bug where authentication through `login()` was being ignored if the + SDK was configured to use a credential store. +* Fixed a bug where download methods would use an absurdly small chunk size, + leading to slow data retrieval +* Fixed a bug where using `DockerClient.images.pull` to pull an image by digest + would lead to an exception being raised. +* `.dockerignore` rules should now be respected as defined by the spec, + including respect for last-line precedence and proper handling of absolute + paths +* The `pass` credential store is now properly supported. + 3.0.1 -----