feat: allow overriding templates in gen and validate

Author: AFriemann

.envrc

@@ -1,6 +1,6 @@
 source_up
 
-which use_nix &>/dev/null && use_nix
+which nix-shell &>/dev/null && use_nix
 
 layout python
 

k8t/cli.py

@@ -57,13 +57,14 @@ def print_license():
 @root.command(name="validate", help="Validate template files for given context.")
 @click.option("-m", "--method", type=click.Choice(MERGE_METHODS), default="ltr", show_default=True, help="Value file merge method.")
 @click.option("--value-file", "value_files", multiple=True, type=click.Path(dir_okay=False, exists=True), help="Additional value file to include.")
-@click.option("--value", "cli_values", type=(str, str), multiple=True, metavar="<KEY VALUE>", help="Additional value(s) to include.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
-@click.option("--environment", "-e", "ename", help="Deployment environment to use.")
+@click.option("--value", "cli_values", type=(str, str), multiple=True, metavar="KEY VALUE", help="Additional value(s) to include.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
+@click.option("--environment", "-e", "ename", metavar="NAME", help="Deployment environment to use.")
 @click.option("--suffix", "-s", "suffixes", default=[".yaml", ".j2", ".jinja2"], help="Filter template files by suffix. Can be used multiple times.", show_default=True)
+@click.option("--template-file", "-t", "template_overrides", metavar="KEY PATH", type=click.Tuple([str, str]), multiple=True, help="Restrict validation to single template file (the key is needed for references in templates).")
 @click.argument("directory", type=click.Path(dir_okay=True, file_okay=False, exists=True), default=os.getcwd())
 @requires_project_directory
-def cli_validate(method, value_files, cli_values, cname, ename, suffixes, directory):
+def cli_validate(method, value_files, cli_values, cname, ename, suffixes, template_overrides, directory):
     vals = deep_merge(  # pylint: disable=redefined-outer-name
         values.load_all(directory, cname, ename, method),
         *(load_yaml(p) for p in value_files),
@@ -73,12 +74,12 @@ def cli_validate(method, value_files, cli_values, cname, ename, suffixes, direct
     )
     config.CONFIG = config.load_all(directory, cname, ename, method)
 
-    eng = build(directory, cname, ename)
+    eng = build(directory, cname, ename, template_overrides)
 
     templates = eng.list_templates()  # pylint: disable=redefined-outer-name
 
     if suffixes:
-        templates = [ name for name in templates if os.path.splitext(name)[1] in suffixes ]
+        templates = [name for name in templates if os.path.splitext(name)[1] in suffixes]
 
     all_validated = True
 
@@ -114,14 +115,15 @@ def cli_validate(method, value_files, cli_values, cname, ename, suffixes, direct
 @root.command(name="gen", help="Create manifest files using stored templates.")
 @click.option("-m", "--method", type=click.Choice(MERGE_METHODS), default="ltr", show_default=True, help="Value file merge method.")
 @click.option("--value-file", "value_files", multiple=True, type=click.Path(dir_okay=False, exists=True), help="Additional value file to include.")
-@click.option("--value", "cli_values", type=(str, str), multiple=True, metavar="<KEY VALUE>", help="Additional value(s) to include.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
-@click.option("--environment", "-e", "ename", help="Deployment environment to use.")
+@click.option("--value", "cli_values", type=(str, str), multiple=True, metavar="KEY VALUE", help="Additional value(s) to include.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
+@click.option("--environment", "-e", "ename", metavar="NAME", help="Deployment environment to use.")
 @click.option("--suffix", "-s", "suffixes", default=[".yaml", ".j2", ".jinja2"], help="Filter template files by suffix. Can be used multiple times.", show_default=True)
-@click.option("--secret-provider", help="Secret provider override.")
+@click.option("--secret-provider", help="Secret provider override.", type=click.Choice(['ssm', 'random', 'hash']))
+@click.option("--template-file", "-t", "template_overrides", metavar="KEY PATH", type=click.Tuple([str, str]), multiple=True, help="Restrict validation to single template file (the key is needed for references in templates).")
 @click.argument("directory", type=click.Path(dir_okay=True, file_okay=False, exists=True), default=os.getcwd())
 @requires_project_directory
-def cli_gen(method, value_files, cli_values, cname, ename, suffixes, secret_provider, directory):  # pylint: disable=redefined-outer-name,too-many-arguments
+def cli_gen(method, value_files, cli_values, cname, ename, suffixes, secret_provider, template_overrides, directory):  # pylint: disable=redefined-outer-name,too-many-arguments
     vals = deep_merge(  # pylint: disable=redefined-outer-name
         values.load_all(directory, cname, ename, method),
         *(load_yaml(p) for p in value_files),
@@ -138,12 +140,12 @@ def cli_gen(method, value_files, cli_values, cname, ename, suffixes, secret_prov
 
         config.CONFIG['secrets']['provider'] = secret_provider
 
-    eng = build(directory, cname, ename)
+    eng = build(directory, cname, ename, template_overrides)
 
     templates = eng.list_templates()  # pylint: disable=redefined-outer-name
 
     if suffixes:
-        templates = [ name for name in templates if os.path.splitext(name)[1] in suffixes ]
+        templates = [name for name in templates if os.path.splitext(name)[1] in suffixes]
 
     validated = True
 
@@ -188,7 +190,7 @@ def new_cluster(name, directory):
 
 
 @new.command(name="environment", help="Create a new environment context.")
-@click.option("--cluster", "-c", "cname")
+@click.option("--cluster", "-c", "cname", metavar="NAME")
 @click.argument("name")
 @click.argument("directory", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
 @requires_project_directory
@@ -199,8 +201,8 @@ def new_environment(cname, name, directory):
 
 
 @new.command(name="template", help="Create specified kubernetes manifest template.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
-@click.option("--environment", "-e", "ename", help="Deployment environment to use.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
+@click.option("--environment", "-e", "ename", metavar="NAME", help="Deployment environment to use.")
 @click.option("--name", "-n", help="Template filename.")
 @click.option("--prefix", "-p", help="Prefix for filename.")
 @click.argument("kind", type=click.Choice(sorted(list(scaffolding.list_available_templates()))))
@@ -233,7 +235,7 @@ def get_clusters(directory):
 
 
 @get.command(name="environments", help="Get configured environments.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
 @click.argument("directory", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
 @requires_project_directory
 def get_environments(cname, directory):  # pylint: disable=redefined-outer-name
@@ -244,8 +246,8 @@ def get_environments(cname, directory):  # pylint: disable=redefined-outer-name
 
 
 @get.command(name="templates", help="Get stored templates.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
-@click.option("--environment", "-e", "ename", help="Deployment environment to use.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
+@click.option("--environment", "-e", "ename", metavar="NAME", help="Deployment environment to use.")
 @click.argument("directory", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
 @requires_project_directory
 def get_templates(directory, cname, ename):  # pylint: disable=redefined-outer-name
@@ -259,8 +261,8 @@ def edit():
 
 
 @edit.command(name="config", help="Edit config files in chosen context.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
-@click.option("--environment", "-e", "ename", help="Deployment environment to use.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
+@click.option("--environment", "-e", "ename", metavar="NAME", help="Deployment environment to use.")
 @click.argument("directory", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
 @requires_project_directory
 def edit_config(directory, cname, ename):  # pylint: disable=redefined-outer-name
@@ -281,8 +283,8 @@ def edit_config(directory, cname, ename):  # pylint: disable=redefined-outer-nam
 
 
 @edit.command(name="values", help="Edit value files in chosen context.")
-@click.option("--cluster", "-c", "cname", help="Cluster context to use.")
-@click.option("--environment", "-e", "ename", help="Deployment environment to use.")
+@click.option("--cluster", "-c", "cname", metavar="NAME", help="Cluster context to use.")
+@click.option("--environment", "-e", "ename", metavar="NAME", help="Deployment environment to use.")
 @click.argument("directory", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
 @requires_project_directory
 def edit_values(directory, cname, ename):  # pylint: disable=redefined-outer-name

k8t/engine.py

@@ -7,24 +7,35 @@
 #
 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+import os
 import logging
 
-from jinja2 import Environment, FileSystemLoader, StrictUndefined
+from typing import List
+from jinja2 import Environment, DictLoader, FileSystemLoader, StrictUndefined
 
 from k8t.filters import (b64decode, b64encode, envvar, get_secret, hashf,
                          random_password, sanitize_label, to_bool)
 from k8t.project import find_files
+from k8t.util import read_file
 
 LOGGER = logging.getLogger(__name__)
 
 
-def build(path: str, cluster: str, environment: str) -> Environment:
-    template_paths = find_template_paths(path, cluster, environment)
+def build(path: str, cluster: str, environment: str, template_overrides: List[str] = None) -> Environment:
+    env = None
+    template_paths = []
 
     LOGGER.debug(
         "building template environment")
 
-    env = Environment(undefined=StrictUndefined, loader=FileSystemLoader(template_paths))
+    if template_overrides is not None and len(template_overrides) > 0:
+        template_paths = {key: read_file(os.path.abspath(path)) for key, path in template_overrides}
+
+        env = Environment(undefined=StrictUndefined, loader=DictLoader(template_paths))
+    else:
+        template_paths = find_template_paths(path, cluster, environment)
+
+        env = Environment(undefined=StrictUndefined, loader=FileSystemLoader(template_paths))
 
     # Filter functions
     env.filters["b64decode"] = b64decode

k8t/util.py

@@ -145,3 +145,7 @@ def list_files(
         break
 
     return result
+
+def read_file(path: str) -> str:
+    with open(path, 'rb') as stream:
+        return stream.read().decode()