diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d6ba02..2042ecd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,8 +44,13 @@ repos: language_version: python3.11 repo: https://github.com/psf/black rev: 23.3.0 - - hooks: - - args: ["--strict", "--non-interactive", "--install-types", "--ignore-missing-imports"] - id: mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + # - hooks: + # - args: ["--strict", "--non-interactive", "--install-types", "--ignore-missing-imports"] + # id: mypy + # repo: https://github.com/pre-commit/mirrors-mypy + # rev: v1.3.0 + - repo: https://github.com/netromdk/vermin + rev: v1.5.1 + hooks: + - id: vermin + args: ['-t=3.6-', '--violations'] diff --git a/CHANGELOG.md b/CHANGELOG.md index 353e76b..306765f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,36 @@ --- +# [v0.8.0](https://github.com/DaemonDude23/container-image-replicator/releases/tag/v0.8.0) - June 1 2023 + +**Enhancements** + +- Logging + - Added support for `success` logs with the `verboselogs` library. + - Added exception catching for _input file not found_ and _failed to parse scenarios_. +- Added example RegEx named capture group example for those using log aggregation tools. + +**Bugfixes** + +- Fixed issue where if the destination image didn't already exist, CIR wouldn't attempt a _push_. +- _Hopefully_ fixed the broken PyInstaller binaries. + +**Housekeeping** + +- pre-commit + - Removed `mypy` pre-commit hook. + - Added [`vermin`](https://github.com/netromdk/vermin) to test minimum Python version required, which is apparently lower than I thought at `v3.6` +- Docs + - Added image of prettily-colored screenshot of command output. +- Added TODO list/musings for future plans to expand functionality of this script at the bottom of [README.md](README.md). + # [v0.7.0](https://github.com/DaemonDude23/container-image-replicator/releases/tag/v0.7.0) - May 24 2023 **Enhancements** -logging -- Added default coloration of logs (turn it off with argument `--no-colors`). - - Add much improved error detail instead of `a silent error has occurred`, replacing it with (example) `denied: Your authorization token has expired. Reauthenticate and try again.` +- Logging + - Added default coloration of logs (turn it off with argument `--no-colors`). + - Add much improved error detail instead of `a silent error has occurred`, replacing it with (example) `denied: Your authorization token has expired. Reauthenticate and try again.` **Bugfixes** diff --git a/README.md b/README.md index 970b68c..8574b87 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -**container-image-replicator** +**container-image-replicator** (_CIR_ for short) - [About](#about) - [Usage](#usage) @@ -17,6 +17,8 @@ - [Dev](#dev) - [`mypy` for type hinting](#mypy-for-type-hinting) - [Code Validation](#code-validation) + - [Miscellaneous Info](#miscellaneous-info) + - [The Future](#the-future) --- @@ -110,7 +112,7 @@ images: # required ## Requirements: -- Python `3.7+` (or manually adjust [./src/requirements.txt](./src/requirements.txt) with more broad constraints) +- Python `3.6+` (or manually adjust [./src/requirements.txt](./src/requirements.txt) with more broad constraints) - `docker` installed and running on the system where this script executed, and sufficient permissions for the user executing `container-image-replicator` ## Installation @@ -119,7 +121,7 @@ images: # required - For local installation/use of the raw script, I use a local virtual environment to isolate dependencies: ```bash -git clone https://github.com/DaemonDude23/container-image-replicator.git -b v0.7.0 +git clone https://github.com/DaemonDude23/container-image-replicator.git -b v0.8.0 cd container-image-replicator ``` @@ -152,21 +154,12 @@ pip3 install -U -r ./src/requirements.txt ### Example ```bash -./src/main.py ./tests/yamls/test.yaml -``` -```bash -2022-10-22T15:19:24+0000 INFO input file successfully validated -2022-10-22T15:19:24+0000 INFO preparing threads. Maximum threads: 2 -2022-10-22T15:19:24+0000 INFO nginx:1.23.2-alpine - source image exists locally -2022-10-22T15:19:29+0000 INFO 000000000000.dkr.ecr.us-east-1.amazonaws.com/nginx:1.23.2-alpine - already present in destination. Skipping push -2022-10-22T15:19:29+0000 INFO httpd:2.4.54-alpine - source image exists locally -2022-10-22T15:19:29+0000 WARNING httpd:2.4.54-alpine - image not found locally -2022-10-22T15:19:29+0000 INFO httpd:2.4.54-alpine - pulling image -2022-10-22T15:19:33+0000 INFO httpd:2.4.54-alpine - image pulled successfully -2022-10-22T15:19:36+0000 INFO 000000000000.dkr.ecr.us-east-1.amazonaws.com/apache:2.4.54 - pushing image -2022-10-22T15:19:45+0000 INFO 000000000000.dkr.ecr.us-east-1.amazonaws.com/apache:2.4.54 - image pushed successfully +# this file doesn't exist in git since it contains my account IDs, but just point it to ./tests/yamls/test1.yaml after updating it +./src/main.py ./tests/yamls/test2.yaml ``` +![output-example](docs/images/output-example.png) + ### kubectl to list all of your container images ```bash @@ -196,3 +189,24 @@ mypy ./src/main.py --check-untyped-defs ```bash mypy --install-types --non-interactive --ignore-missing-imports src/main.py ``` + +## Miscellaneous Info + +If you need a named capture group to capture logs in a semi-structed way, this should work: + +``` +(?^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}-\d{4})\s(?\w+)\s(?.+) +``` + +## The Future + +Any help with these things would be appreciated. + +- I'm considering adding support for + - [PodMan](https://github.com/containers/podman-py) to push images. This would allow a non-`root` user to run this which is always good. + - Building and pushing images, not _just_ pulling them from somewhere else. + - This one is probably pretty easy. `-f` equivalent field in the config file for the Dockerfile, the build context, build-args, etc. + - Scan **Kubernetes** and generate a file containing all images, allowing the user to customize it further for their specific destination repositories. + - Equivalent of `kubectl get` for Pods with `annotations` that are watched by CIR and periodically + - Can be run inside of Kubernetes or outside of it. + - Would require building and maintaining container images and a Helm Chart. diff --git a/docs/images/output-example.png b/docs/images/output-example.png new file mode 100644 index 0000000..4e3e668 Binary files /dev/null and b/docs/images/output-example.png differ diff --git a/src/main.py b/src/main.py index 40781b4..e865e74 100755 --- a/src/main.py +++ b/src/main.py @@ -8,17 +8,18 @@ from sys import stdout from time import sleep from typing import Any -from typing import Dict -from typing import List -from typing import LiteralString -from typing import Optional from typing import Tuple import coloredlogs import docker +import verboselogs import yaml +verboselogs.install() +logger = logging.getLogger(__name__) + + def init_docker() -> Any | Any: docker_client = docker.from_env() docker_api = docker.APIClient() @@ -36,7 +37,7 @@ def init_arg_parser() -> Any: args_optional = parser.add_argument_group("optional") args_required = parser.add_argument_group("required") - args_optional.add_argument("--version", "-v", action="version", version="v0.7.0") + args_optional.add_argument("--version", "-v", action="version", version="v0.8.0") args_optional.add_argument( "--max-workers", @@ -81,7 +82,8 @@ def init_arg_parser() -> Any: exit(1) -def parse_image_list_yaml(image_list: Dict[LiteralString, Any]) -> bool: +# def parse_image_list_yaml(image_list: Dict[LiteralString, Any]) -> bool: +def parse_image_list_yaml(image_list) -> bool: """searches for each element in the list for all expected keys/values Args: @@ -98,12 +100,12 @@ def parse_image_list_yaml(image_list: Dict[LiteralString, Any]) -> bool: try: str(image["destination"]["tag"]) except KeyError: - logging.debug("no destination tag provided - using source tag as a fallback") + logger.debug("no destination tag provided - using source tag as a fallback") str(image["source"]["tag"]) except KeyError: - logging.fatal("syntax error in list file provided") + logger.critical("syntax error in list file provided") exit(1) - logging.info("input file successfully validated") + logger.success("input file successfully validated") return True @@ -118,12 +120,12 @@ def pull_image(repository: str, tag: str) -> bool: bool: success or failure """ try: - logging.info(f"{repository}:{tag} - pulling image") + logger.info(f"{repository}:{tag} - pulling image") docker_client.images.pull(repository, tag=tag) - logging.info(f"{repository}:{tag} - image pulled successfully") + logger.success(f"{repository}:{tag} - image pulled successfully") return True except docker.errors.APIError or docker.errors.ImageNotFound as e: - logging.warning(e) + logger.warning(e) return False @@ -140,11 +142,11 @@ def push_image(repository: str, tag: str) -> bool: bool: success or failure """ try: - logging.info(f"{repository}:{tag} - pushing image") + logger.info(f"{repository}:{tag} - pushing image") docker_client.images.push(repository, tag=tag) return True except docker.errors.APIError as e: - logging.error(f"{repository}:{tag} - failed to push image") + logger.error(f"{repository}:{tag} - failed to push image") return False @@ -176,17 +178,17 @@ def verify_local_image( if final_sha256 != "": source_endpoint_and_sha256: str = str(f"{source_endpoint}@{final_sha256}") docker_client.images.list(filters={"reference": f"{source_endpoint_and_sha256}"}) - logging.info(f"{source_endpoint_and_sha256} - source image exists locally") + logger.info(f"{source_endpoint_and_sha256} - source image exists locally") else: docker_client.images.list(filters={"reference": f"{source_endpoint}"}) - logging.info(f"{source_endpoint} - source image exists locally") + logger.info(f"{source_endpoint} - source image exists locally") # replace docker.io as its implicit and not returned by the API when doing lookups docker_api.tag(source_endpoint, destination_repository, destination_tag) # type: ignore return True except docker.errors.ImageNotFound as e: - logging.warning(f"{source_endpoint} - image not found locally") - logging.debug(e) + logger.warning(f"{source_endpoint} - image not found locally") + logger.debug(e) pull_image(source_repository, source_tag) return False @@ -202,7 +204,7 @@ def verify_destination_image(docker_client: Any, uri: str) -> Tuple[str, int]: """ try: docker_client.images.get_registry_data(uri) - logging.debug(f"{uri} - verified this exists in destination") + logger.debug(f"{uri} - verified this exists in destination") return "exists", 200 except docker.errors.ImageNotFound as e: # reasonable error return "does not exist", int(e.status_code) @@ -253,7 +255,6 @@ def check_remote( force_pull (bool): force pull, used for immutable tags force_push (bool): force push, used for immutable tags """ - # image_already_pushed: bool = False if arguments.force_pull_push or force_pull: pull_image(source_repository, source_tag) if arguments.force_pull_push or force_push: @@ -261,19 +262,18 @@ def check_remote( sleep(1) verify_destination, status_code = verify_destination_image(docker_client, destination_endpoint) - if status_code == 404: - logging.error(f"{destination_endpoint} - {verify_destination}") - elif verify_destination == "exists": - logging.info(f"{destination_endpoint} - image pushed successfully") + if verify_destination == "exists": + logger.info(f"{destination_endpoint} - destination image exists in registry") # image_already_pushed = True - elif verify_destination == "does not exist": + elif (verify_destination == "does not exist") or (status_code == 404): + logging.warning(f"{destination_endpoint} - destination image not found in registry") # see if image exists locally and pull from the source registry if it doesn't verify_local_image( docker_api, source_endpoint, source_repository, source_tag, destination_repository, destination_tag, final_sha256 ) push_image(destination_repository, destination_tag) else: - logging.error(f"{destination_endpoint} - {verify_destination}") + logger.error(f"{destination_endpoint} - {verify_destination}") return True @@ -289,7 +289,7 @@ def actions(arguments: Any, docker_api: Any, image_list: Any) -> bool: Returns: bool: True if no show-stopping exceptions """ - logging.info(f"preparing threads. Maximum threads: {arguments.max_workers}") + logger.info(f"preparing threads. Maximum threads: {arguments.max_workers}") thread_pool = ThreadPoolExecutor(max_workers=arguments.max_workers) for image in list(image_list["images"]): # remove docker.io registry prefix as its implicit and not returned by the API when doing lookups @@ -312,7 +312,7 @@ def actions(arguments: Any, docker_api: Any, image_list: Any) -> bool: try: destination_tag = str(image["destination"]["tag"]) except KeyError: - logging.debug("no destination tag provided - using source tag as a fallback") + logger.debug("no destination tag provided - using source tag as a fallback") destination_tag = str(image["source"]["tag"]) # use []source.sha256 if its valid @@ -321,12 +321,12 @@ def actions(arguments: Any, docker_api: Any, image_list: Any) -> bool: source_sha256: str = str(image["source"]["sha256"]) if valid_sha256(source_sha256): final_sha256 = source_sha256 - logging.debug(f"{final_sha256} - using this valid sha256") + logger.debug(f"{final_sha256} - using this valid sha256") else: - logging.warning(f"{source_repository}:@sha256:{source_sha256} - skipping image because sha256 is not valid") + logger.warning(f"{source_repository}:@sha256:{source_sha256} - skipping image because sha256 is not valid") break except KeyError: - logging.debug("no valid source sha256 provided, not using sha256 suffix on image URI") + logger.debug("no valid source sha256 provided, not using sha256 suffix on image URI") # combine repositories and tags source_endpoint: str = str(f"{source_repository}:{source_tag}") @@ -373,8 +373,8 @@ def main(docker_api: Any) -> None: print("failed to determine the specified value for --log-level") if arguments.no_colors: - logging.basicConfig( - level=eval(f"logging.{log_level}"), + logger.basicConfig( + level=eval(f"logger.{log_level}"), datefmt="%Y-%m-%dT%H:%M:%S%z", stream=stdout, format="%(asctime)s %(levelname)s %(message)s", @@ -382,9 +382,15 @@ def main(docker_api: Any) -> None: else: coloredlogs.install(level=log_level, fmt="%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%S%z") - image_list: Dict[LiteralString, List[Any]] = yaml.safe_load(Path(arguments.input_file).read_text()) - parse_image_list_yaml(image_list) - actions(arguments, docker_api, image_list) + try: + # image_list: Dict[LiteralString, List[Any]] = yaml.safe_load(Path(arguments.input_file).read_text()) + image_list = yaml.safe_load(Path(arguments.input_file).read_text()) + parse_image_list_yaml(image_list) + actions(arguments, docker_api, image_list) + except FileNotFoundError: + logger.critical(f"input file not found. I cannot continue: {Path(arguments.input_file)}") + except yaml.parser.ParserError as e: + logger.critical(f"failed to parse input file: {Path(arguments.input_file)} with error: {e}") if __name__ == "__main__": diff --git a/src/requirements.txt b/src/requirements.txt index 21817cd..5df9b4c 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,5 @@ coloredlogs==15.0.1 -docker==6.1.2 +docker==6.1.3 PyYAML==6.0 requests<=2.29.0 # https://github.com/docker/docker-py/issues/3113 -# types-PyYAML==6.0.12.9 +verboselogs==1.7