From 16772692335400e958668ac482bfa56064646db1 Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Fri, 9 Sep 2016 14:04:07 -0400 Subject: [PATCH] Cleaned containers.py by moving some functionality to container and dockercontainer Additional changes: - unit tests for containers.py - some nits in emitter for pep8 Signed-off-by: Ricardo Koller --- crawler/container.py | 44 ++++- crawler/containers.py | 76 ++------- crawler/crawler_exceptions.py | 1 + crawler/dockercontainer.py | 14 +- crawler/emitter.py | 7 +- tests/unit/test_containers.py | 222 +++++++++++++++++++++++++ tests/unit/test_dockercontainer.py | 255 ++++++++++++++++++++++++++++- 7 files changed, 544 insertions(+), 75 deletions(-) create mode 100644 tests/unit/test_containers.py diff --git a/crawler/container.py b/crawler/container.py index 2511c83a..4ffe95d6 100644 --- a/crawler/container.py +++ b/crawler/container.py @@ -3,23 +3,59 @@ import os import logging import defaults +import psutil +import misc +import namespace logger = logging.getLogger('crawlutils') +def list_raw_containers(user_list='ALL'): + """ + A running container is defined as a group of processes with the + `pid` namespace different to the `init` process `pid` namespace. + """ + init_ns = namespace.get_pid_namespace(1) + for p in psutil.process_iter(): + pid = (p.pid() if hasattr(p.pid, '__call__') else p.pid) + if pid == 1 or pid == '1': + + # don't confuse the init process as a container + + continue + if user_list not in ['ALL', 'all', 'All']: + if str(pid) not in user_list: + + # skip containers not in the list + + continue + if misc.process_is_crawler(pid): + + # don't confuse the crawler process with a container + + continue + curr_ns = namespace.get_pid_namespace(pid) + if not curr_ns: + + # invalid container + + continue + if curr_ns == init_ns: + continue + yield Container(pid, curr_ns) + + class Container(object): """ This class abstracts a running Linux container. - - A running container is defined as a process subtree with the `pid` - namespace different to the `init` process `pid` namespace. """ def __init__( self, pid, container_opts={}, + process_namespace=None, ): self.pid = str(pid) self.short_id = str(hash(pid)) @@ -30,6 +66,8 @@ def __init__( self.root_fs = None self.log_prefix = None self.log_file_list = None + self.process_namespace = (process_namespace or + namespace.get_pid_namespace(pid)) # XXX(kollerr): when running in alchemy environment, non-alchemy # containres should be ignored diff --git a/crawler/containers.py b/crawler/containers.py index 280527cc..c1014ea9 100644 --- a/crawler/containers.py +++ b/crawler/containers.py @@ -4,14 +4,10 @@ # External dependencies that must be pip install'ed separately -import psutil - import defaults -from container import Container +import container import misc -import namespace from dockercontainer import list_docker_containers -from crawler_exceptions import ContainerInvalidEnvironment logger = logging.getLogger('crawlutils') @@ -25,59 +21,19 @@ def list_all_containers(user_list='ALL', A running container is defined as a process subtree with the `pid` namespace different to the `init` process `pid` namespace. """ - all_docker_containers = list_docker_containers(container_opts) - - if user_list in ['ALL', 'all', 'All']: - init_ns = namespace.get_pid_namespace(1) - - visited_ns = set() # visited PID namespaces - - # Start with all docker containers - - for container in all_docker_containers: - curr_ns = namespace.get_pid_namespace(container.pid) - if not curr_ns: - continue - if curr_ns not in visited_ns and curr_ns != init_ns: - visited_ns.add(curr_ns) - try: - yield container - except ContainerInvalidEnvironment as e: - logger.exception(e) - - # Continue with all other containers not known to docker - - for p in psutil.process_iter(): - pid = (p.pid() if hasattr(p.pid, '__call__') else p.pid) - if pid == 1 or pid == '1': - - # don't confuse the init process as a container - - continue - if misc.process_is_crawler(pid): - - # don't confuse the crawler process with a container - - continue - curr_ns = namespace.get_pid_namespace(pid) - if not curr_ns: - - # invalid container - - continue - if curr_ns not in visited_ns and curr_ns != init_ns: - visited_ns.add(curr_ns) - yield Container(pid) - else: + visited_ns = set() # visited PID namespaces - # User provided a list of containers + for _container in list_docker_containers(container_opts, user_list): + curr_ns = _container.process_namespace + if curr_ns not in visited_ns: + visited_ns.add(curr_ns) + yield _container - user_containers = user_list.split(',') - for container in all_docker_containers: - short_id_match = container.short_id in user_containers - long_id_match = container.long_id in user_containers - if short_id_match or long_id_match: - yield container + for _container in container.list_raw_containers(user_list): + curr_ns = _container.process_namespace + if curr_ns not in visited_ns: + visited_ns.add(curr_ns) + yield _container def get_filtered_list_of_containers( @@ -108,7 +64,7 @@ def get_filtered_list_of_containers( filtered_list = [] containers_list = list_all_containers(user_list, container_opts) - for container in containers_list: + for _container in containers_list: """ There are docker and non-docker containers in this list. An example of @@ -118,7 +74,7 @@ def get_filtered_list_of_containers( """ if (environment != defaults.DEFAULT_ENVIRONMENT and - not container.is_docker_container()): + not _container.is_docker_container()): continue """ @@ -126,9 +82,9 @@ def get_filtered_list_of_containers( process pid. We do it by hashing the long_id of the container. """ - _hash = container.long_id + _hash = _container.long_id num = int(_hash, 16) % int(num_processes) if num == process_id: - filtered_list.append(container) + filtered_list.append(_container) return filtered_list diff --git a/crawler/crawler_exceptions.py b/crawler/crawler_exceptions.py index 462db995..689fa03f 100644 --- a/crawler/crawler_exceptions.py +++ b/crawler/crawler_exceptions.py @@ -43,6 +43,7 @@ class ContainerWithoutCgroups(Exception): pass + class DockerutilsException(Exception): """Exception from the dockerutils module.""" diff --git a/crawler/dockercontainer.py b/crawler/dockercontainer.py index 00f73bca..7d780dd1 100644 --- a/crawler/dockercontainer.py +++ b/crawler/dockercontainer.py @@ -20,11 +20,12 @@ DockerutilsException, ContainerWithoutCgroups) from requests.exceptions import HTTPError +import namespace logger = logging.getLogger('crawlutils') -def list_docker_containers(container_opts={}): +def list_docker_containers(container_opts={}, user_list='ALL'): """ Get the list of running Docker containers, as `DockerContainer` objects. @@ -34,6 +35,13 @@ def list_docker_containers(container_opts={}): """ for inspect in exec_dockerps(): long_id = inspect['Id'] + + if user_list not in ['ALL', 'all', 'All']: + user_ctrs = [cid[:12] for cid in user_list.split(',')] + short_id = long_id[:12] + if short_id not in user_ctrs: + continue + try: c = DockerContainer(long_id, inspect, container_opts) if c.namespace: @@ -73,6 +81,7 @@ def __init__( long_id, inspect=None, container_opts={}, + process_namespace=None, ): # Some quick sanity checks @@ -105,6 +114,9 @@ def __init__( self.volumes = inspect.get('Volumes') self.inspect = inspect + self.process_namespace = (process_namespace or + namespace.get_pid_namespace(self.pid)) + # This short ID is mainly used for logging purposes self.short_id = long_id[:12] diff --git a/crawler/emitter.py b/crawler/emitter.py index 66933c07..46dd16af 100644 --- a/crawler/emitter.py +++ b/crawler/emitter.py @@ -29,6 +29,7 @@ # Kafka logs too much logging.getLogger('kafka').addHandler(NullHandler()) + def kafka_send(kurl, temp_fpath, format, topic, queue=None): try: kafka_python_client = kafka_python.KafkaClient(kurl) @@ -59,6 +60,7 @@ def kafka_send(kurl, temp_fpath, format, topic, queue=None): finally: queue and queue.close() + class Emitter: """Class that abstracts the outputs supported by the crawler, like @@ -321,7 +323,6 @@ def _publish_to_kafka_no_retries(self, url): kurl, self.temp_fpath, self.format, topic, queue)) child_process.start() except OSError: - #queue.close() # closing queue in finally clause raise try: @@ -335,7 +336,6 @@ def _publish_to_kafka_no_retries(self, url): if child_process.is_alive(): errmsg = ('Timed out waiting for process %d to exit.' % child_process.pid) - #queue.close() # closing queue in finally clause os.kill(child_process.pid, 9) logger.error(errmsg) raise EmitterEmitTimeout(errmsg) @@ -427,7 +427,8 @@ def __exit__( try: self._publish(url) except EmitterEmitTimeout as exc: - logger.warning("Failed to emit due to timout, continuing anyway...") + logger.warning('Failed to emit due to timeout, ' + 'continuing anyway...') finally: if os.path.exists(self.temp_fpath): os.remove(self.temp_fpath) diff --git a/tests/unit/test_containers.py b/tests/unit/test_containers.py new file mode 100644 index 00000000..2c3ca184 --- /dev/null +++ b/tests/unit/test_containers.py @@ -0,0 +1,222 @@ +import mock +import unittest + +from crawler.containers import (list_all_containers, + get_filtered_list_of_containers) + + +def mocked_exists(pid): + return True + + +class DockerContainer(): + + def __init__(self, pid): + self.pid = pid + self.short_id = pid + self.long_id = pid + self.process_namespace = pid + + def __str__(self): + return 'container %s' % self.pid + + def is_docker_container(self): + return True + +DOCKER_IDS = ['101', '102', '103', '104', '105', '106'] + + +def mocked_list_docker_containers(container_opts={}, user_list='ALL'): + for long_id in DOCKER_IDS: + + if user_list not in ['ALL', 'all', 'All']: + user_ctrs = [cid[:12] for cid in user_list.split(',')] + short_id = long_id[:12] + if short_id not in user_ctrs: + continue + + c = DockerContainer(long_id) + yield c + + +class PsUtilProcess(): + + def __init__(self, pid): + self.pid = pid + + +class ContainersTests(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + @mock.patch('crawler.containers.list_docker_containers', + side_effect=lambda container_opts, user_list='ALL': + mocked_list_docker_containers(container_opts, user_list)) + @mock.patch('crawler.containers.container.namespace.get_pid_namespace', + side_effect=lambda pid: pid) + @mock.patch('crawler.containers.container.psutil.process_iter', + side_effect=lambda: [PsUtilProcess('4'), # container + PsUtilProcess('1'), # init + PsUtilProcess('5')]) # crawler + @mock.patch('crawler.containers.container.misc.process_is_crawler', + side_effect=lambda pid: True if pid == '5' else False) + def test_list_all_containers(self, *args): + pids = [c.pid for c in list_all_containers()] + # pid 1 is the init process, which is not a container + # according to the definition in container.py + assert set(pids) == set(DOCKER_IDS + ['4']) + assert '1' not in pids # init process + assert '5' not in pids # crawler process + assert args[0].call_count == 2 + assert args[1].call_count == 1 + assert args[2].call_count == 3 + assert args[3].call_count == 1 + + @mock.patch('crawler.containers.list_docker_containers', + side_effect=lambda container_opts, user_list='ALL': + mocked_list_docker_containers(container_opts, user_list)) + @mock.patch('crawler.containers.container.namespace.get_pid_namespace', + side_effect=lambda pid: pid) + @mock.patch('crawler.containers.container.psutil.process_iter', + side_effect=lambda: [PsUtilProcess('4'), # container + PsUtilProcess('1'), # init + PsUtilProcess('5')]) # crawler + @mock.patch('crawler.containers.container.misc.process_is_crawler', + side_effect=lambda pid: True if pid == '5' else False) + def test_list_all_containers_input_list(self, *args): + pids = [c.pid for c in list_all_containers(user_list='102')] + # pid 1 is the init process, which is not a container + # according to the definition in container.py + assert set(pids) == set(['102']) + assert '3' not in pids # filtered container + assert '4' not in pids # filtered container + assert '1' not in pids # init process + assert '5' not in pids # crawler process + + @mock.patch('crawler.containers.list_docker_containers', + side_effect=lambda container_opts, user_list='ALL': + mocked_list_docker_containers(container_opts, user_list)) + @mock.patch('crawler.containers.container.namespace.get_pid_namespace', + side_effect=lambda pid: pid) + @mock.patch('crawler.containers.container.psutil.process_iter', + side_effect=lambda: [PsUtilProcess('4'), # container + PsUtilProcess('1'), # init + PsUtilProcess('5')]) # crawler + @mock.patch('crawler.containers.container.misc.process_is_crawler', + side_effect=lambda pid: True if pid == '5' else False) + def test_get_filtered_list(self, *args): + pids = [c.pid for c in get_filtered_list_of_containers()] + # pid 1 is the init process, which is not a container + # according to the definition in container.py + assert set(pids) == set(DOCKER_IDS + ['4']) + assert '1' not in pids # init process + assert '5' not in pids # crawler process + + @mock.patch('crawler.containers.list_docker_containers', + side_effect=lambda container_opts, user_list='ALL': + mocked_list_docker_containers(container_opts, user_list)) + @mock.patch('crawler.containers.container.namespace.get_pid_namespace', + side_effect=lambda pid: pid) + @mock.patch('crawler.containers.container.psutil.process_iter', + side_effect=lambda: [PsUtilProcess('4'), # container + PsUtilProcess('1'), # init + PsUtilProcess('5')]) # crawler + @mock.patch('crawler.containers.container.misc.process_is_crawler', + side_effect=lambda pid: True if pid == '5' else False) + def test_get_filtered_list_with_input_list(self, *args): + opts = {'docker_containers_list': '102', + 'partition_strategy': {'name': 'equally_by_pid', + 'args': {'process_id': 0, + 'num_processes': 1}}} + pids = [c.pid for c in get_filtered_list_of_containers(opts)] + # pid 1 is the init process, which is not a container + # according to the definition in container.py + assert set(pids) == set(['102']) + assert '3' not in pids # filtered container + assert '4' not in pids # filtered container + assert '1' not in pids # init process + assert '5' not in pids # crawler process + + @mock.patch('crawler.containers.list_docker_containers', + side_effect=lambda container_opts, user_list='ALL': + mocked_list_docker_containers(container_opts, user_list)) + @mock.patch('crawler.containers.container.namespace.get_pid_namespace', + side_effect=lambda pid: pid) + @mock.patch('crawler.containers.container.psutil.process_iter', + side_effect=lambda: [PsUtilProcess('4'), # container + PsUtilProcess('1'), # init + PsUtilProcess('5')]) # crawler + @mock.patch('crawler.containers.container.misc.process_is_crawler', + side_effect=lambda pid: True if pid == '5' else False) + def test_get_filtered_list_with_input_list_ALL(self, *args): + opts = {'docker_containers_list': 'ALL', + 'partition_strategy': {'name': 'equally_by_pid', + 'args': {'process_id': 0, + 'num_processes': 1}}} + pids = [c.pid for c in get_filtered_list_of_containers(opts)] + # pid 1 is the init process, which is not a container + # according to the definition in container.py + assert set(pids) == set(DOCKER_IDS + ['4']) + + @mock.patch('crawler.containers.list_docker_containers', + side_effect=lambda container_opts, user_list='ALL': + mocked_list_docker_containers(container_opts, user_list)) + @mock.patch('crawler.containers.container.namespace.get_pid_namespace', + side_effect=lambda pid: pid) + @mock.patch('crawler.containers.container.psutil.process_iter', + side_effect=lambda: [PsUtilProcess('4'), # container + PsUtilProcess('1'), # init + PsUtilProcess('9')]) # crawler + @mock.patch('crawler.containers.container.misc.process_is_crawler', + side_effect=lambda pid: True if pid == '9' else False) + def test_get_filtered_list_with_3_processes(self, *args): + + # Let's divide the lists for three procs: pids1, pids2, pids3 + opts = {'docker_containers_list': 'ALL', + 'partition_strategy': {'name': 'equally_by_pid', + 'args': {'process_id': 0, # ", + "Created": "2016-07-06T16:38:05.479090842Z", + "State": { + "Status": "running", + "Running": True, + "Pid": 11186 + }, + "Image": "sha256:07c86167cdc4264926fa5d2894e34a339ad27", + "Name": "/pensive_rosalind", + "Mounts": [], + "Config": { + "Cmd": [ + "bash" + ], + "Image": "ubuntu:trusty" + }, + "NetworkSettings": { + } + } + for i in range(10): + _inspect = inspect + _inspect['Id'] = str(i) + yield _inspect + + def mocked_get_rootfs(long_id): if long_id == 'valid_rootfs_id': return '/tmp/something/docker/' + long_id @@ -164,14 +191,6 @@ def mocked_get_container_json_logs_path(id, inspect): return '/var/lib/docker/abc/container/log.json' -@mock.patch('crawler.dockercontainer.exec_dockerps', - side_effect=mocked_exec_dockerps) -@mock.patch('crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', - side_effect=mocked_get_runtime_env) -@mock.patch('crawler.dockercontainer.exec_dockerinspect', - side_effect=mocked_docker_inspect) -@mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', - side_effect=mocked_get_rootfs) class DockerDockerContainerTests(unittest.TestCase): def setUp(self): @@ -180,6 +199,15 @@ def setUp(self): def tearDown(self): pass + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_list_docker_containers(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): n = 0 @@ -189,6 +217,37 @@ def test_list_docker_containers(self, mock_get_rootfs, mock_inspect, assert mocked_get_runtime_env.call_count == 3 assert n == 2 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps_long) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) + def test_list_docker_containers_with_input( + self, + mock_get_rootfs, + mock_inspect, + mocked_get_runtime_env, + mocked_dockerps): + n = 0 + ids = [c.short_id for c in list_docker_containers(user_list='1,2,8')] + assert set(ids) == set(['1', '2', '8']) + assert mocked_get_runtime_env.call_count == 3 + ids = [c.long_id for c in list_docker_containers(user_list='5,3')] + assert set(ids) == set(['3', '5']) + + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_list_docker_containers_with_opts(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, @@ -204,6 +263,15 @@ def test_list_docker_containers_with_opts(self, mock_get_rootfs, assert mocked_get_runtime_env.call_count == 1 assert n == 2 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_init( self, mock_get_rootfs, @@ -215,6 +283,15 @@ def test_init( assert not c.root_fs assert mocked_get_runtime_env.call_count == 1 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_init_from_inspect(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): inspect = { @@ -242,6 +319,15 @@ def test_init_from_inspect(self, mock_get_rootfs, mock_inspect, assert not c.root_fs assert mocked_get_runtime_env.call_count == 1 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_init_from_inspect_w_repotags(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): @@ -276,6 +362,15 @@ def test_init_from_inspect_w_repotags(self, mock_get_rootfs, mock_inspect, assert c.docker_image_registry == 'registry.com:123' assert c.owner_namespace == 'ric' + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_init_from_inspect_w_repotags2(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): @@ -310,12 +405,30 @@ def test_init_from_inspect_w_repotags2(self, mock_get_rootfs, mock_inspect, assert c.docker_image_registry == 'registry.com:123' assert c.owner_namespace == '' + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_init_failed(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): with self.assertRaises(crawler_exceptions.ContainerNonExistent): DockerContainer("no_container_id") assert mocked_get_runtime_env.call_count == 0 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_init_wrong_environment( self, mock_get_rootfs, @@ -331,6 +444,15 @@ def test_init_wrong_environment( with self.assertRaises(crawler_exceptions.ContainerInvalidEnvironment): DockerContainer("throw_value_error_id") + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def test_is_docker( self, mock_get_rootfs, @@ -341,6 +463,15 @@ def test_is_docker( assert c.is_docker_container() print(c) + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.path.ismount', side_effect=lambda x: True if x == '/cgroup/memory' else False) def test_memory_cgroup( @@ -354,6 +485,15 @@ def test_memory_cgroup( assert c.get_memory_cgroup_path( 'abc') == '/cgroup/memory/docker/good_id/abc' + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch( 'crawler.dockercontainer.os.path.ismount', side_effect=lambda x: True if x == '/cgroup/cpuacct' else False) @@ -368,6 +508,15 @@ def test_cpu_cgroup( assert c.get_cpu_cgroup_path( 'abc') == '/cgroup/cpuacct/docker/good_id/abc' + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') def test_link_logfiles( @@ -385,6 +534,15 @@ def test_link_logfiles( '/var/log/crawler_container_logs/random_prefix/var/log/2') assert mock_symlink.call_count == 4 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') @mock.patch( @@ -407,6 +565,15 @@ def test_link_logfiles_env_variable( '/var/log/crawler_container_logs/random_prefix/var/log/2') assert mock_symlink.call_count == 6 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink', side_effect=mocked_symlink_oserror) @@ -422,6 +589,15 @@ def test_link_logfiles_symlink_oserror( c.link_logfiles() # no exceptoin should be thrown + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink', side_effect=mocked_symlink_exception) @@ -437,6 +613,15 @@ def test_link_logfiles_symlink_exception( c.link_logfiles() # no exceptoin should be thrown + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') @mock.patch('crawler.dockercontainer.shutil.rmtree') @@ -458,6 +643,15 @@ def test_link_and_unlink_logfiles( assert mock_symlink.call_count == 4 assert mock_rmtree.call_count == 1 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') @mock.patch('crawler.dockercontainer.shutil.rmtree') @@ -482,6 +676,15 @@ def test_link_and_unlink_docker_json_logfile( assert mock_symlink.call_count == 5 assert mock_rmtree.call_count == 1 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') @mock.patch('crawler.dockercontainer.shutil.rmtree', @@ -504,6 +707,15 @@ def test_link_and_unlink_logfiles_failed_rmtree( assert mock_symlink.call_count == 4 assert mock_rmtree.call_count == 1 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') @mock.patch('crawler.dockercontainer.shutil.rmtree', @@ -548,6 +760,15 @@ def test_links_with_mounts( c.unlink_logfiles() assert mock_symlink.call_count == 4 + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.dockercontainer.os.makedirs') @mock.patch('crawler.dockercontainer.os.symlink') @mock.patch('crawler.dockercontainer.shutil.rmtree', @@ -594,6 +815,15 @@ def test_links_with_volumes( # TODO test _get_cgroup_dir when ismount fails + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) def _test_non_implemented_methods(self): c = DockerContainer("some_id") with self.assertRaises(NotImplementedError): @@ -605,6 +835,15 @@ def _test_non_implemented_methods(self): with self.assertRaises(NotImplementedError): c.unlink_logfiles() + @mock.patch('crawler.dockercontainer.exec_dockerps', + side_effect=mocked_exec_dockerps) + @mock.patch( + 'crawler.dockercontainer.plugins_manager.get_runtime_env_plugin', + side_effect=mocked_get_runtime_env) + @mock.patch('crawler.dockercontainer.exec_dockerinspect', + side_effect=mocked_docker_inspect) + @mock.patch('crawler.dockercontainer.get_docker_container_rootfs_path', + side_effect=mocked_get_rootfs) @mock.patch('crawler.emitter.os.path.exists', side_effect=mocked_exists) def _test_is_running(self, mock_exists): c = DockerContainer("good_id")