From 42f965d80ef17ec0b09f6f73005c45c87c7349ef Mon Sep 17 00:00:00 2001 From: Mike Jones <73889157+mikedbjones@users.noreply.github.com> Date: Wed, 19 Apr 2023 14:46:21 +0100 Subject: [PATCH] Add support for optional custom keys --- README.md | 3 +++ numerai/cli/setup.py | 9 ++++++++- numerai/cli/upgrade.py | 6 ++++-- numerai/cli/util/docker.py | 3 +++ numerai/cli/util/keys.py | 33 +++++++++++++++++++++++++++++++-- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 58e57dd..2c34ba6 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,9 @@ NOTES: NEXT: [read the Prediction Nodes wiki](https://github.com/numerai/numerai-cli/wiki/Prediction-Nodes) to learn about Numerai Examples and how to customize Prediction Nodes. +### Using Custom Keys +During `numerai setup` you can optionally set up to two custom keys, or leave blank if not required. These keys can be used for any constants that you don't want to hardcode in your app or Dockerfile (for example usernames, passwords or secrets for other services that your app uses, such as Numerbay). These custom keys will be available in the environment as *CUSTOM_KEY_1* and *CUSTOM_KEY_2*. + ## List of Commands Use the `--help` option on any command or sub-command to get a full description of it: ``` diff --git a/numerai/cli/setup.py b/numerai/cli/setup.py index 3cde82c..a720fa4 100644 --- a/numerai/cli/setup.py +++ b/numerai/cli/setup.py @@ -7,7 +7,8 @@ copy_files from numerai.cli.util.keys import \ config_numerai_keys, \ - config_provider_keys + config_provider_keys, \ + config_custom_keys @click.command() @@ -36,6 +37,11 @@ def setup(provider, verbose): f"(press enter to keep value in brackets)...", fg='yellow') config_provider_keys(provider) + # setup custom keys + click.secho("\nInitializing custom keys (optional) " + "(press enter to keep value in brackets or skip custom keys)...", fg='yellow') + config_custom_keys() + # copy tf files click.secho("copying terraform files...") copy_files(TERRAFORM_PATH, CONFIG_PATH, force=True, verbose=True) @@ -46,5 +52,6 @@ def setup(provider, verbose): click.secho("Numerai API Keys setup and working", fg='green') click.secho(f"{provider} API Keys setup and working", fg='green') + click.secho("Custom Keys setup", fg='green') click.secho(f"Terraform files copied to {CONFIG_PATH}", fg='green') click.echo('succesfully initialized numerai-cli') \ No newline at end of file diff --git a/numerai/cli/upgrade.py b/numerai/cli/upgrade.py index 7884e0d..b6dbc77 100644 --- a/numerai/cli/upgrade.py +++ b/numerai/cli/upgrade.py @@ -11,7 +11,8 @@ load_or_init_keys, \ load_or_init_nodes, \ config_numerai_keys, \ - config_provider_keys + config_provider_keys, \ + config_custom_keys @click.command() @@ -51,10 +52,11 @@ def upgrade(verbose): # INIT KEYS AND NODES keys_config = load_or_init_keys() - if not os.path.exists(KEYS_PATH) or 'aws' not in keys_config or 'numerai' not in keys_config: + if not os.path.exists(KEYS_PATH) or 'aws' not in keys_config or 'numerai' not in keys_config or 'custom' not in keys_config: click.secho(f"Keys missing from {KEYS_PATH}, you must re-initialize your keys:") config_numerai_keys() config_provider_keys(PROVIDER_AWS) + config_custom_keys() nodes_config = load_or_init_nodes() # DELETE OLD CONFIG FILES diff --git a/numerai/cli/util/docker.py b/numerai/cli/util/docker.py index ed2b1d8..4df4bee 100644 --- a/numerai/cli/util/docker.py +++ b/numerai/cli/util/docker.py @@ -130,6 +130,7 @@ def terraform(tf_cmd, verbose, env_vars=None, inputs=None, version='0.14.3'): def build(node_config, node, verbose): numerai_keys = load_or_init_keys()['numerai'] + custom_keys = load_or_init_keys()['custom'] node_path = node_config["path"] curr_path = os.path.abspath('.') @@ -145,6 +146,8 @@ def build(node_config, node, verbose): build_arg_str = '' for arg in numerai_keys: build_arg_str += f' --build-arg {arg}={numerai_keys[arg]}' + for arg in custom_keys: + build_arg_str += f' --build-arg {arg}={custom_keys[arg]}' build_arg_str += f' --build-arg MODEL_ID={node_config["model_id"]}' build_arg_str += f' --build-arg SRC_PATH={path}' build_arg_str += f' --build-arg NODE={node}' diff --git a/numerai/cli/util/keys.py b/numerai/cli/util/keys.py index 8aebe22..e960e92 100644 --- a/numerai/cli/util/keys.py +++ b/numerai/cli/util/keys.py @@ -32,6 +32,10 @@ def reformat_keys(): 'numerai': { 'NUMERAI_PUBLIC_ID': config['default']['NUMERAI_PUBLIC_ID'], 'NUMERAI_SECRET_KEY': config['default']['NUMERAI_SECRET_KEY'] + }, + 'custom': { + 'CUSTOM_KEY_1': config['default']['CUSTOM_KEY_1'], + 'CUSTOM_KEY_2': config['default']['CUSTOM_KEY_2'] } } @@ -66,8 +70,11 @@ def get_numerai_keys(): return None, None -def prompt_for_key(name, default): +def prompt_for_key(name, default, allow_blank=False): hidden = sanitize_message(default) + if hidden is None: + if allow_blank: + hidden='' # if allow_blank=True, pass an empty string as the default value new = click.prompt(name, default=hidden).strip() if new == hidden: return default @@ -144,6 +151,28 @@ def check_aws_validity(key_id, secret): ) +def get_custom_keys(): + keys = load_or_init_keys() + try: + return keys['custom']['CUSTOM_KEY_1'],\ + keys['custom']['CUSTOM_KEY_2'] + except KeyError: + return None, None + + +def config_custom_keys(): + custom_key_1, custom_key_2 = get_custom_keys() + # custom keys are optional, so are allowed to be blank, unlike numerai and provider keys + custom_key_1 = prompt_for_key('CUSTOM_KEY_1', custom_key_1, allow_blank=True) + custom_key_2 = prompt_for_key('CUSTOM_KEY_2', custom_key_2, allow_blank=True) + + keys_config = load_or_init_keys() + keys_config.setdefault('custom', {}) + keys_config['custom']['CUSTOM_KEY_1'] = custom_key_1 + keys_config['custom']['CUSTOM_KEY_2'] = custom_key_2 + store_config(KEYS_PATH, keys_config) + + def config_provider_keys(cloud_provider): if cloud_provider == PROVIDER_AWS: config_aws_keys() @@ -152,7 +181,7 @@ def config_provider_keys(cloud_provider): def sanitize_message(message, censor_substr=None): if message is None: return None - all_keys = get_aws_keys() + get_numerai_keys() + all_keys = get_aws_keys() + get_numerai_keys() + get_custom_keys() for key in all_keys: if key: try: