diff --git a/.env b/.env index cc88a7f..47fac1d 100644 --- a/.env +++ b/.env @@ -1,2 +1,9 @@ # .env -TIINGO_API_KEY=your_tiingo_api_key +TIINGO_API_KEY=5da9e86c26b6f16540af23883eea67d71216abf4 +TOKEN= +TRAINING_DAYS= +TIMEFRAME= +MODEL= +REGION= +DATA_PROVIDER= +CG_API_KEY= \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..96cde77 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim AS project_env + +# Install system dependencies including build essentials +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + gcc \ + g++ + +# Set the working directory in the container +WORKDIR /app + +# Install Python build dependencies first +COPY requirements.txt requirements.txt +RUN pip install --upgrade pip setuptools wheel \ + && pip install Cython==3.0.11 numpy==1.24.3 \ + && pip install -r requirements.txt + +FROM project_env + +COPY . /app/ + +# Set the entrypoint command +CMD ["gunicorn", "--conf", "/app/gunicorn_conf.py", "main:app"] diff --git a/config.example.json b/config.example.json new file mode 100644 index 0000000..c586a90 --- /dev/null +++ b/config.example.json @@ -0,0 +1,29 @@ +{ + "wallet": { + "addressKeyName": "test", + "addressRestoreMnemonic": "", + "alloraHomeDir": "", + "gas": "auto", + "gasAdjustment": 1.5, + "gasPrices": 10, + "maxFees": 25000000, + "nodeRpc": "https://allora-rpc.testnet.allora.network", + "maxRetries": 5, + "retryDelay": 3, + "accountSequenceRetryDelay": 5, + "submitTx": true, + "blockDurationEstimated": 5, + "windowCorrectionFactor": 0.8 + }, + "worker": [ + { + "topicId": 1, + "inferenceEntrypointName": "api-worker-reputer", + "loopSeconds": 5, + "parameters": { + "InferenceEndpoint": "http://inference:8000/inference/{Token}", + "Token": "ETH" + } + } + ] +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..a70b156 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "wallet": { + "addressKeyName": "forge-wallet", + "addressRestoreMnemonic": "degree shoe you side trigger gadget cricket estate essay bronze silent glass ranch luggage fat basic penalty garbage crazy genre weapon memory good thank", + "alloraHomeDir": "", + "gas": "auto", + "gasAdjustment": 1.5, + "gasPrices": 10, + "maxFees": 25000000, + "nodeRpc": "https://allora-rpc.testnet.allora.network", + "maxRetries": 5, + "retryDelay": 3, + "accountSequenceRetryDelay": 5, + "submitTx": true, + "blockDurationEstimated": 5, + "windowCorrectionFactor": 0.8 + }, + "worker": [ + { + "topicId": 15, + "inferenceEntrypointName": "api-worker-reputer", + "loopSeconds": 5, + "parameters": { + "InferenceEndpoint": "http://inference:8000/inference/{Token}", + "Token": "ETH" + } + } + ] +} \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..c7bf78c --- /dev/null +++ b/config.py @@ -0,0 +1,21 @@ +import os +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +app_base_path = os.getenv("APP_BASE_PATH", default=os.getcwd()) +data_base_path = os.path.join(app_base_path, "data") +model_file_path = os.path.join(data_base_path, "model.pkl") + +TOKEN = os.getenv("TOKEN").upper() +TRAINING_DAYS = os.getenv("TRAINING_DAYS") +TIMEFRAME = os.getenv("TIMEFRAME") +MODEL = os.getenv("MODEL") +REGION = os.getenv("REGION").lower() +if REGION in ["us", "com", "usa"]: + REGION = "us" +else: + REGION = "com" +DATA_PROVIDER = os.getenv("DATA_PROVIDER").lower() +CG_API_KEY = os.getenv("CG_API_KEY", default=None) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e523331 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +services: + inference: + container_name: inference + env_file: + - .env + build: . + command: python -u /app/app.py + ports: + - "8000:8000" + healthcheck: + test: ["CMD", "curl", "-f", "http://inference:8000/inference/${TOKEN}"] + interval: 10s + timeout: 5s + retries: 12 + volumes: + - ./inference-data:/app/data + + updater: + container_name: updater + build: . + environment: + - INFERENCE_API_ADDRESS=http://inference:8000 + command: > + sh -c " + while true; do + python -u /app/update_app.py; + sleep 24h; + done + " + depends_on: + inference: + condition: service_healthy + + worker: + container_name: worker + image: alloranetwork/allora-offchain-node:v0.6.0 + volumes: + - ./worker-data:/data + depends_on: + inference: + condition: service_healthy + env_file: + - ./worker-data/env_file + +volumes: + inference-data: + worker-data: \ No newline at end of file diff --git a/gunicorn_conf.py b/gunicorn_conf.py new file mode 100644 index 0000000..759df0b --- /dev/null +++ b/gunicorn_conf.py @@ -0,0 +1,12 @@ +# Gunicorn config variables +loglevel = "info" +errorlog = "-" # stderr +accesslog = "-" # stdout +worker_tmp_dir = "/dev/shm" +graceful_timeout = 120 +timeout = 30 +keepalive = 5 +worker_class = "gthread" +workers = 1 +threads = 8 +bind = "0.0.0.0:9000" diff --git a/init.config b/init.config new file mode 100755 index 0000000..353d0e2 --- /dev/null +++ b/init.config @@ -0,0 +1,61 @@ +#!/bin/bash + +set -e + +if [ ! -f config.json ]; then + echo "Error: config.json file not found, please provide one" + exit 1 +fi + +# Install system dependencies required for building Python packages +if command -v apt-get &> /dev/null; then + apt-get update && apt-get install -y \ + build-essential \ + gcc \ + gfortran \ + python3-dev \ + libopenblas-dev \ + python3-pip +fi + +# Install critical Python dependencies first +pip install --no-cache-dir \ + setuptools==72.1.0 \ + wheel==0.44.0 \ + Cython==3.0.11 \ + numpy==1.24.3 + +nodeName=$(jq -r '.wallet.addressKeyName' config.json) +if [ -z "$nodeName" ]; then + echo "No wallet name provided for the node, please provide your preferred wallet name. config.json >> wallet.addressKeyName" + exit 1 +fi + +# Ensure the worker-data directory exists +mkdir -p ./worker-data + +json_content=$(cat ./config.json) +stringified_json=$(echo "$json_content" | jq -c .) + +mnemonic=$(jq -r '.wallet.addressRestoreMnemonic' config.json) +if [ -n "$mnemonic" ]; then + echo "ALLORA_OFFCHAIN_NODE_CONFIG_JSON='$stringified_json'" > ./worker-data/env_file + echo "NAME=$nodeName" >> ./worker-data/env_file + echo "ENV_LOADED=true" >> ./worker-data/env_file + echo "wallet mnemonic already provided by you, loading config.json . Please proceed to run docker compose" + exit 1 +fi + +if [ ! -f ./worker-data/env_file ]; then + echo "ENV_LOADED=false" > ./worker-data/env_file +fi + +ENV_LOADED=$(grep '^ENV_LOADED=' ./worker-data/env_file | cut -d '=' -f 2) +if [ "$ENV_LOADED" = "false" ]; then + json_content=$(cat ./config.json) + stringified_json=$(echo "$json_content" | jq -c .) + docker run -it --entrypoint=bash -v $(pwd)/worker-data:/data -v $(pwd)/scripts:/scripts -e NAME="${nodeName}" -e ALLORA_OFFCHAIN_NODE_CONFIG_JSON="${stringified_json}" alloranetwork/allora-chain:v0.8.0 -c "bash /scripts/init.sh" + echo "config.json saved to ./worker-data/env_file" +else + echo "config.json is already loaded, skipping the operation. You can set ENV_LOADED variable to false in ./worker-data/env_file to reload the config.json" +fi diff --git a/scripts/init.sh b/scripts/init.sh new file mode 100644 index 0000000..0e6af84 --- /dev/null +++ b/scripts/init.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +if allorad keys --home=/data/.allorad --keyring-backend test show $NAME > /dev/null 2>&1 ; then + echo "allora account: $NAME already imported" +else + echo "creating allora account: $NAME" + output=$(allorad keys add $NAME --home=/data/.allorad --keyring-backend test 2>&1) + address=$(echo "$output" | grep 'address:' | sed 's/.*address: //') + mnemonic=$(echo "$output" | tail -n 1) + + # Parse and update the JSON string + updated_json=$(echo "$ALLORA_OFFCHAIN_NODE_CONFIG_JSON" | jq --arg name "$NAME" --arg mnemonic "$mnemonic" ' + .wallet.addressKeyName = $name | + .wallet.addressRestoreMnemonic = $mnemonic + ') + + stringified_json=$(echo "$updated_json" | jq -c .) + + echo "ALLORA_OFFCHAIN_NODE_CONFIG_JSON='$stringified_json'" > /data/env_file + echo ALLORA_OFFCHAIN_ACCOUNT_ADDRESS=$address >> /data/env_file + echo "NAME=$NAME" >> /data/env_file + + echo "Updated ALLORA_OFFCHAIN_NODE_CONFIG_JSON saved to /data/env_file" +fi + + +if grep -q "ENV_LOADED=false" /data/env_file; then + sed -i 's/ENV_LOADED=false/ENV_LOADED=true/' /data/env_file +else + echo "ENV_LOADED=true" >> /data/env_file +fi diff --git a/updater.py b/updater.py new file mode 100644 index 0000000..6ae4cf1 --- /dev/null +++ b/updater.py @@ -0,0 +1,175 @@ +import os +from datetime import date, timedelta +import pathlib +import time +import requests +from requests.adapters import HTTPAdapter +from urllib3.util import Retry +from concurrent.futures import ThreadPoolExecutor +import pandas as pd +import json + + +# Define the retry strategy +retry_strategy = Retry( + total=4, # Maximum number of retries + backoff_factor=2, # Exponential backoff factor (e.g., 2 means 1, 2, 4, 8 seconds, ...) + status_forcelist=[429, 500, 502, 503, 504], # HTTP status codes to retry on +) + +# Create an HTTP adapter with the retry strategy and mount it to session +adapter = HTTPAdapter(max_retries=retry_strategy) + +# Create a new session object +session = requests.Session() +session.mount('http://', adapter) +session.mount('https://', adapter) + + +files = [] + + +# Function to download the URL, called asynchronously by several child processes +def download_url(url, download_path, name=None): + try: + global files + if name: + file_name = os.path.join(download_path, name) + else: + file_name = os.path.join(download_path, os.path.basename(url)) + dir_path = os.path.dirname(file_name) + pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True) + if os.path.isfile(file_name): + # print(f"{file_name} already exists") + return + # Make a request using the session object + response = session.get(url) + if response.status_code == 404: + print(f"File does not exist: {url}") + elif response.status_code == 200: + with open(file_name, 'wb') as f: + f.write(response.content) + # print(f"Downloaded: {url} to {file_name}") + files.append(file_name) + return + else: + print(f"Failed to download {url}") + return + except Exception as e: + print(str(e)) + + +# Function to generate a range of dates +def daterange(start_date, end_date): + for n in range(int((end_date - start_date).days)): + yield start_date + timedelta(n) + + +# Function to download daily data from Binance +def download_binance_daily_data(pair, training_days, region, download_path): + base_url = f"https://data.binance.vision/data/spot/daily/klines" + + end_date = date.today() + start_date = end_date - timedelta(days=int(training_days)) + + global files + files = [] + + with ThreadPoolExecutor() as executor: + print(f"Downloading data for {pair}") + for single_date in daterange(start_date, end_date): + url = f"{base_url}/{pair}/1m/{pair}-1m-{single_date}.zip" + executor.submit(download_url, url, download_path) + + return files + + +def download_binance_current_day_data(pair, region): + limit = 1000 + base_url = f'https://api.binance.{region}/api/v3/klines?symbol={pair}&interval=1m&limit={limit}' + + # Make a request using the session object + response = session.get(base_url) + response.raise_for_status() + resp = str(response.content, 'utf-8').rstrip() + + columns = ['start_time','open','high','low','close','volume','end_time','volume_usd','n_trades','taker_volume','taker_volume_usd','ignore'] + + df = pd.DataFrame(json.loads(resp),columns=columns) + df['date'] = [pd.to_datetime(x+1,unit='ms') for x in df['end_time']] + df['date'] = df['date'].apply(pd.to_datetime) + df[["volume", "taker_volume", "open", "high", "low", "close"]] = df[["volume", "taker_volume", "open", "high", "low", "close"]].apply(pd.to_numeric) + + return df.sort_index() + + +def get_coingecko_coin_id(token): + token_map = { + 'ETH': 'ethereum', + 'SOL': 'solana', + 'BTC': 'bitcoin', + 'BNB': 'binancecoin', + 'ARB': 'arbitrum', + # Add more tokens here + } + + token = token.upper() + if token in token_map: + return token_map[token] + else: + raise ValueError("Unsupported token") + + +def download_coingecko_data(token, training_days, download_path, CG_API_KEY): + if training_days <= 7: + days = 7 + elif training_days <= 14: + days = 14 + elif training_days <= 30: + days = 30 + elif training_days <= 90: + days = 90 + elif training_days <= 180: + days = 180 + elif training_days <= 365: + days = 365 + else: + days = "max" + print(f"Days: {days}") + + coin_id = get_coingecko_coin_id(token) + print(f"Coin ID: {coin_id}") + + # Get OHLC data from Coingecko + url = f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days={days}&api_key={CG_API_KEY}' + + global files + files = [] + + with ThreadPoolExecutor() as executor: + print(f"Downloading data for {coin_id}") + name = os.path.basename(url).split("?")[0].replace("/", "_") + ".json" + executor.submit(download_url, url, download_path, name) + + return files + + +def download_coingecko_current_day_data(token, CG_API_KEY): + coin_id = get_coingecko_coin_id(token) + print(f"Coin ID: {coin_id}") + + url = f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days=1&api_key={CG_API_KEY}' + + # Make a request using the session object + response = session.get(url) + response.raise_for_status() + resp = str(response.content, 'utf-8').rstrip() + + columns = ['timestamp','open','high','low','close'] + + df = pd.DataFrame(json.loads(resp), columns=columns) + df['date'] = [pd.to_datetime(x,unit='ms') for x in df['timestamp']] + df['date'] = df['date'].apply(pd.to_datetime) + df[["open", "high", "low", "close"]] = df[["open", "high", "low", "close"]].apply(pd.to_numeric) + + return df.sort_index() diff --git a/utils/scripts/init.sh b/utils/scripts/init.sh new file mode 100644 index 0000000..0e6af84 --- /dev/null +++ b/utils/scripts/init.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +if allorad keys --home=/data/.allorad --keyring-backend test show $NAME > /dev/null 2>&1 ; then + echo "allora account: $NAME already imported" +else + echo "creating allora account: $NAME" + output=$(allorad keys add $NAME --home=/data/.allorad --keyring-backend test 2>&1) + address=$(echo "$output" | grep 'address:' | sed 's/.*address: //') + mnemonic=$(echo "$output" | tail -n 1) + + # Parse and update the JSON string + updated_json=$(echo "$ALLORA_OFFCHAIN_NODE_CONFIG_JSON" | jq --arg name "$NAME" --arg mnemonic "$mnemonic" ' + .wallet.addressKeyName = $name | + .wallet.addressRestoreMnemonic = $mnemonic + ') + + stringified_json=$(echo "$updated_json" | jq -c .) + + echo "ALLORA_OFFCHAIN_NODE_CONFIG_JSON='$stringified_json'" > /data/env_file + echo ALLORA_OFFCHAIN_ACCOUNT_ADDRESS=$address >> /data/env_file + echo "NAME=$NAME" >> /data/env_file + + echo "Updated ALLORA_OFFCHAIN_NODE_CONFIG_JSON saved to /data/env_file" +fi + + +if grep -q "ENV_LOADED=false" /data/env_file; then + sed -i 's/ENV_LOADED=false/ENV_LOADED=true/' /data/env_file +else + echo "ENV_LOADED=true" >> /data/env_file +fi diff --git a/worker-data/env_file b/worker-data/env_file new file mode 100644 index 0000000..3dbbb28 --- /dev/null +++ b/worker-data/env_file @@ -0,0 +1,3 @@ +ALLORA_OFFCHAIN_NODE_CONFIG_JSON='{"wallet":{"addressKeyName":"forge-wallet","addressRestoreMnemonic":"degree shoe you side trigger gadget cricket estate essay bronze silent glass ranch luggage fat basic penalty garbage crazy genre weapon memory good thank","alloraHomeDir":"","gas":"auto","gasAdjustment":1.5,"gasPrices":10,"maxFees":25000000,"nodeRpc":"https://allora-rpc.testnet.allora.network","maxRetries":5,"retryDelay":3,"accountSequenceRetryDelay":5,"submitTx":true,"blockDurationEstimated":5,"windowCorrectionFactor":0.8},"worker":[{"topicId":15,"inferenceEntrypointName":"api-worker-reputer","loopSeconds":5,"parameters":{"InferenceEndpoint":"http://inference:8000/inference/{Token}","Token":"ETH"}}]}' +NAME=forge-wallet +ENV_LOADED=true