Add support for optional custom keys

README.md

@@ -144,6 +144,9 @@ To use this tool you need:
       [ECS Cluster](https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/numerai-submission-ecs-cluster/tasks)
       and [other resources](https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups)
 
+### Using Custom Keys
+During `numerai setup` you can optionally set up to two custom keys, or leave blank if not required. These keys can be used for any constants that you don't want to hardcode in your app or Dockerfile (for example usernames, passwords or secrets for other services that your app uses, such as Numerbay). These custom keys will be available in the environment as *CUSTOM_KEY_1* and *CUSTOM_KEY_2*.
+
 ## List of Commands
 Use the `--help` option on any command or sub-command to get a full description of it:
 ```

numerai/cli/setup.py

@@ -7,7 +7,8 @@
     copy_files
 from numerai.cli.util.keys import \
     config_numerai_keys, \
-    config_provider_keys
+    config_provider_keys, \
+    config_custom_keys
 
 
 @click.command()
@@ -36,6 +37,11 @@ def setup(provider, verbose):
                 f"(press enter to keep value in brackets)...", fg='yellow')
     config_provider_keys(provider)
 
+    # setup custom keys
+    click.secho("\nInitializing custom keys (optional) "
+                "(press enter to keep value in brackets or skip custom keys)...", fg='yellow')
+    config_custom_keys()
+
     # copy tf files
     click.secho("copying terraform files...")
     copy_files(TERRAFORM_PATH, CONFIG_PATH, force=True, verbose=True)
@@ -46,5 +52,6 @@ def setup(provider, verbose):
 
     click.secho("Numerai API Keys setup and working", fg='green')
     click.secho(f"{provider} API Keys setup and working", fg='green')
+    click.secho("Custom Keys setup", fg='green')
     click.secho(f"Terraform files copied to {CONFIG_PATH}", fg='green')
     click.echo('succesfully initialized numerai-cli')

numerai/cli/upgrade.py

@@ -11,7 +11,8 @@
     load_or_init_keys, \
     load_or_init_nodes, \
     config_numerai_keys, \
-    config_provider_keys
+    config_provider_keys, \
+    config_custom_keys
 
 
 @click.command()
@@ -51,10 +52,11 @@ def upgrade(verbose):
 
     # INIT KEYS AND NODES
     keys_config = load_or_init_keys()
-    if not os.path.exists(KEYS_PATH) or 'aws' not in keys_config or 'numerai' not in keys_config:
+    if not os.path.exists(KEYS_PATH) or 'aws' not in keys_config or 'numerai' not in keys_config or 'custom' not in keys_config:
         click.secho(f"Keys missing from {KEYS_PATH}, you must re-initialize your keys:")
         config_numerai_keys()
         config_provider_keys(PROVIDER_AWS)
+        config_custom_keys()
     nodes_config = load_or_init_nodes()
 
     # DELETE OLD CONFIG FILES

numerai/cli/util/docker.py

@@ -130,6 +130,7 @@ def terraform(tf_cmd, verbose, env_vars=None, inputs=None, version='0.14.3'):
 
 def build(node_config, node, verbose):
     numerai_keys = load_or_init_keys()['numerai']
+    custom_keys = load_or_init_keys()['custom']
 
     node_path = node_config["path"]
     curr_path = os.path.abspath('.')
@@ -145,6 +146,8 @@ def build(node_config, node, verbose):
     build_arg_str = ''
     for arg in numerai_keys:
         build_arg_str += f' --build-arg {arg}={numerai_keys[arg]}'
+    for arg in custom_keys:
+        build_arg_str += f' --build-arg {arg}={custom_keys[arg]}'
     build_arg_str += f' --build-arg MODEL_ID={node_config["model_id"]}'
     build_arg_str += f' --build-arg SRC_PATH={path}'
     build_arg_str += f' --build-arg NODE={node}'

numerai/cli/util/keys.py

@@ -32,6 +32,10 @@ def reformat_keys():
             'numerai': {
                 'NUMERAI_PUBLIC_ID': config['default']['NUMERAI_PUBLIC_ID'],
                 'NUMERAI_SECRET_KEY': config['default']['NUMERAI_SECRET_KEY']
+            },
+            'custom': {
+                'CUSTOM_KEY_1': config['default']['CUSTOM_KEY_1'],
+                'CUSTOM_KEY_2': config['default']['CUSTOM_KEY_2']
             }
         }
 
@@ -66,8 +70,11 @@ def get_numerai_keys():
         return None, None
 
 
-def prompt_for_key(name, default):
+def prompt_for_key(name, default, allow_blank=False):
     hidden = sanitize_message(default)
+    if hidden is None:
+        if allow_blank:
+            hidden='' # if allow_blank=True, pass an empty string as the default value
     new = click.prompt(name, default=hidden).strip()
     if new == hidden:
         return default
@@ -144,6 +151,28 @@ def check_aws_validity(key_id, secret):
         )
 
 
+def get_custom_keys():
+    keys = load_or_init_keys()
+    try:
+        return keys['custom']['CUSTOM_KEY_1'],\
+               keys['custom']['CUSTOM_KEY_2']
+    except KeyError:
+        return None, None
+
+
+def config_custom_keys():
+    custom_key_1, custom_key_2 = get_custom_keys()
+    # custom keys are optional, so are allowed to be blank, unlike numerai and provider keys
+    custom_key_1 = prompt_for_key('CUSTOM_KEY_1', custom_key_1, allow_blank=True)
+    custom_key_2 = prompt_for_key('CUSTOM_KEY_2', custom_key_2, allow_blank=True)
+
+    keys_config = load_or_init_keys()
+    keys_config.setdefault('custom', {})
+    keys_config['custom']['CUSTOM_KEY_1'] = custom_key_1
+    keys_config['custom']['CUSTOM_KEY_2'] = custom_key_2
+    store_config(KEYS_PATH, keys_config)
+
+
 def config_provider_keys(cloud_provider):
     if cloud_provider == PROVIDER_AWS:
         config_aws_keys()
@@ -152,7 +181,7 @@ def config_provider_keys(cloud_provider):
 def sanitize_message(message, censor_substr=None):
     if message is None:
         return None
-    all_keys = get_aws_keys() + get_numerai_keys()
+    all_keys = get_aws_keys() + get_numerai_keys() + get_custom_keys()
     for key in all_keys:
         if key:
             try: