From 37e828021b43e4c1a57cf3e465296f056c99ba33 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 14:50:11 +0100 Subject: [PATCH 01/43] Add upload script for crypto risk dashboard project This script uploads files to an existing GitHub repository, creating necessary directories and files for a crypto risk dashboard project. --- upload_to_existing_repo.py | 545 +++++++++++++++++++++++++++++++++++++ 1 file changed, 545 insertions(+) create mode 100644 upload_to_existing_repo.py diff --git a/upload_to_existing_repo.py b/upload_to_existing_repo.py new file mode 100644 index 00000000..a067c115 --- /dev/null +++ b/upload_to_existing_repo.py @@ -0,0 +1,545 @@ +#!/usr/bin/env python3 +import os, sys, json, textwrap, base64, shutil +from pathlib import Path +import urllib.request, urllib.parse + +def gh_request(method, url, token, data=None, headers=None): + hdr = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github+json", + "User-Agent": "crypto-risk-dashboard-uploader" + } + if headers: hdr.update(headers) + req = urllib.request.Request(url, data=data, headers=hdr, method=method) + try: + with urllib.request.urlopen(req) as resp: + return resp.getcode(), resp.read() + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8","ignore") + raise SystemExit(f"HTTP {e.code} Error at {url}\n{body}") + +def gh_put_file(user, repo, token, path, content_bytes, message, branch="main"): + url = f"https://api.github.com/repos/{user}/{repo}/contents/{urllib.parse.quote(path)}" + payload = { + "message": message, + "content": base64.b64encode(content_bytes).decode("utf-8"), + "branch": branch + } + code, body = gh_request("PUT", url, token, data=json.dumps(payload).encode("utf-8")) + return json.loads(body.decode("utf-8")) + +print("=== Upload crypto-risk-dashboard to an EXISTING GitHub repo ===") +gh_user = input("GitHub username: ").strip() +token = input("GitHub Personal Access Token (classic, scope: repo; SSO authorized if needed): ").strip() +repo = input("Existing repo name (exact): ").strip() +print("NOTE: The repo must already exist and have a README so branch 'main' exists.") + +root = Path.cwd() / "crypto-risk-dashboard-starter" +if root.exists(): shutil.rmtree(root) +root.mkdir(parents=True, exist_ok=True) + +def w(relpath: str, content: str): + p = root / relpath + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(textwrap.dedent(content).lstrip("\n"), encoding="utf-8") + +# ------- Minimal starter (same structure you need) ------- +w("README.md", """ +# Crypto Risk Dashboard — Self-Hosted Starter +Dockerized stack: FastAPI API, worker (ingest+signals), Postgres (Timescale-ready), Streamlit UI. +Local: `cp .env.example .env` → `docker compose up --build` +Render deploy: use `render.yaml` (free Postgres + API + UI + worker). +""") + +w(".env.example", """ +POSTGRES_USER=cryptouser +POSTGRES_PASSWORD=cryptopass +POSTGRES_DB=cryptodb +POSTGRES_HOST=db +POSTGRES_PORT=5432 +REDIS_URL=redis://redis:6379/0 +SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT +SCHEDULE_MINUTES=5 +REDDIT_CLIENT_ID= +REDDIT_CLIENT_SECRET= +REDDIT_USER_AGENT=crypto-risk-app/0.1 by you +X_BEARER_TOKEN= +CRYPTOPANIC_API_KEY= +NEWSAPI_KEY= +COINGLASS_API_KEY= +""") + +w("docker-compose.yml", """ +services: + db: + image: postgres:16 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: [ "db_data:/var/lib/postgresql/data" ] + ports: [ "5432:5432" ] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 20 + redis: + image: redis:7 + ports: [ "6379:6379" ] + api: + build: ./services/api + env_file: .env + depends_on: { db: { condition: service_healthy } } + ports: [ "8000:8000" ] + worker: + build: ./services/worker + env_file: .env + depends_on: + db: { condition: service_healthy } + redis: { condition: service_started } + ui: + build: ./services/ui + env_file: .env + depends_on: { api: { condition: service_started } } + ports: [ "8501:8501" ] +volumes: { db_data: {} } +""") + +w("render.yaml", """ +databases: + - name: cryptodb + databaseName: cryptodb + plan: free +services: + - type: web + name: crypto-risk-api + runtime: docker + rootDir: services/api + plan: free + envVars: + - key: DATABASE_URL + fromDatabase: { name: cryptodb, property: connectionString } + - key: SYMBOLS + value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + - key: SCHEDULE_MINUTES + value: "5" + - type: web + name: crypto-risk-ui + runtime: docker + rootDir: services/ui + plan: free + envVars: + - key: API_CANDIDATES + value: https://crypto-risk-api.onrender.com,http://api:8000,http://localhost:8000 + - type: worker + name: crypto-risk-worker + runtime: docker + rootDir: services/worker + plan: free + envVars: + - key: DATABASE_URL + fromDatabase: { name: cryptodb, property: connectionString } + - key: SYMBOLS + value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + - key: SCHEDULE_MINUTES + value: "5" +""") + +common_reqs = """ +fastapi==0.115.2 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +python-dotenv==1.0.1 +psycopg2-binary==2.9.9 +SQLAlchemy==2.0.36 +alembic==1.13.2 +redis==5.0.8 +celery==5.4.0 +pandas==2.2.3 +numpy==2.1.2 +plotly==5.24.1 +matplotlib==3.9.2 +ccxt==4.4.6 +scikit-learn==1.5.2 +textblob==0.18.0.post0 +nltk==3.9.1 +requests==2.32.3 +beautifulsoup4==4.12.3 +""" + +# API +(api := root / "services/api"); api.mkdir(parents=True, exist_ok=True) +(api / "Dockerfile").write_text("""FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["uvicorn","main:app","--host","0.0.0.0","--port","8000"]""", encoding="utf-8") +(api / "requirements.txt").write_text(common_reqs, encoding="utf-8") +(api / "main.py").write_text(""" +import os +from fastapi import FastAPI, Query +from services_common.db import fetch_df +from services_common.signals import latest_signals_for_pairs, signal_explanations +from services_common.config import load_config + +app = FastAPI(title="Crypto Risk API", version="0.1.0") +cfg = load_config() + +@app.get("/health") +def health(): return {"ok": True} + +@app.get("/pairs") +def pairs(): return {"pairs": cfg.symbols} + +@app.get("/signals") +def get_signals(pairs: str = Query(None)): + pairs_list = [p.strip() for p in pairs.split(",")] if pairs else cfg.symbols + data = latest_signals_for_pairs(pairs_list) + return {"signals": data, "explanations": signal_explanations()} + +@app.get("/timeseries/{metric}") +def timeseries(metric: str, pair: str, limit: int = 500): + table_map = {"candles":"candles","funding":"funding_rates","oi":"open_interest","vol":"volatility","sentiment":"sentiment"} + table = table_map.get(metric) + if not table: return {"error":"unknown metric"} + q = f"SELECT * FROM {table} WHERE pair=%(pair)s ORDER BY ts DESC LIMIT %(limit)s" + df = fetch_df(q, {"pair": pair, "limit": limit}) + return {"columns": list(df.columns), "rows": df.to_dict(orient='records')} +""", encoding="utf-8") + +# Worker +(worker := root / "services/worker"); worker.mkdir(parents=True, exist_ok=True) +(worker / "Dockerfile").write_text("""FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python","run_worker.py"]""", encoding="utf-8") +(worker / "requirements.txt").write_text(common_reqs, encoding="utf-8") +(worker / "run_worker.py").write_text(""" +import os, time +from services_common.db import ensure_schema +from services_common.ingest import run_ingest_cycle +from services_common.signals import compute_all_signals +def main(): + ensure_schema() + interval = int(os.getenv("SCHEDULE_MINUTES","5")) + print(f"[worker] schedule {interval}m") + while True: + print("[worker] ingest..."); run_ingest_cycle() + print("[worker] signals..."); compute_all_signals() + print("[worker] sleep..."); time.sleep(interval*60) +if __name__=="__main__": main() +""", encoding="utf-8") + +# UI +(ui := root / "services/ui"); ui.mkdir(parents=True, exist_ok=True) +(ui / "Dockerfile").write_text("""FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["streamlit","run","app.py","--server.port=8501","--server.address=0.0.0.0"]""", encoding="utf-8") +(ui / "requirements.txt").write_text(common_reqs + "\nstreamlit==1.39.0\nstreamlit_echarts==0.4.0\n", encoding="utf-8") +(ui / "app.py").write_text(""" +import os, requests, pandas as pd +import streamlit as st +from urllib.parse import urlencode + +DEFAULT_CANDIDATES = [ + "https://crypto-risk-api.onrender.com", + "http://api:8000", + "http://localhost:8000", +] + +def probe_api(base: str, timeout=2.0) -> bool: + try: + r = requests.get(f"{base}/health", timeout=timeout) + return r.ok and r.json().get("ok") is True + except Exception: return False + +def resolve_api_base(): + env_base = os.getenv("API_BASE","").strip() + if env_base and probe_api(env_base): return env_base + env_candidates = os.getenv("API_CANDIDATES","") + candidates = [c.strip() for c in env_candidates.split(",") if c.strip()] if env_candidates else [] + merged, seen = [], set() + for x in (candidates + DEFAULT_CANDIDATES): + if x not in seen: merged.append(x); seen.add(x) + for base in merged: + if probe_api(base): return base + return None + +API_BASE = resolve_api_base() + +st.set_page_config(page_title="Crypto Risk Dashboard", layout="wide") +st.title("🧭 Crypto Risk Dashboard") +st.caption("Self-hosted. Graphs • Meters • Hot Signals") + +if not API_BASE: + st.error("Could not locate the API automatically.") + manual = st.text_input("Enter your API base URL") + if manual and probe_api(manual): + API_BASE = manual; st.success("Connected!") +if not API_BASE: st.stop() + +resp = requests.get(f"{API_BASE}/pairs", timeout=30).json() +pairs = resp.get("pairs", ["binance:BTC/USDT"]) + +col1, col2 = st.columns([2,1]) +with col1: pair = st.selectbox("Pair", pairs, index=0) +with col2: refresh = st.button("Refresh") + +def fetch(metric, pair, limit=500): + qs = urlencode({"pair": pair, "limit": limit}) + return requests.get(f"{API_BASE}/timeseries/{metric}?{qs}", timeout=30).json() + +def get_signals(pairs=None): + qs = "?pairs=" + ",".join(pairs) if pairs else "" + return requests.get(f"{API_BASE}/signals{qs}", timeout=30).json() + +sig = get_signals([pair]); signals = sig.get("signals", {}) + +st.subheader("Hot Signals") +if signals.get(pair): + s = signals[pair] + c1,c2,c3,c4 = st.columns(4) + with c1: st.metric("Market Regime", s.get("regime","—")) + with c2: st.metric("Bias", s.get("bias","—")) + with c3: st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) + with c4: st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) + st.info(s.get("summary","")) +else: st.warning("No signals yet. The worker may still be seeding data.") +st.divider(); st.subheader("Charts") +tabs = st.tabs(["Candles","Funding","Open Interest","Volatility","Sentiment"]) +with tabs[0]: + rows = fetch("candles", pair, 300).get("rows", []) + if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["close"]) + else: st.write("No data yet.") +with tabs[1]: + rows = fetch("funding", pair, 500).get("rows", []) + if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["rate"]) + else: st.write("No data yet.") +with tabs[2]: + rows = fetch("oi", pair, 500).get("rows", []) + if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["value_usd"]) + else: st.write("No data yet.") +with tabs[3]: + rows = fetch("vol", pair, 500).get("rows", []) + if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["atr"]) + else: st.write("No data yet.") +with tabs[4]: + rows = fetch("sentiment", pair, 200).get("rows", []) + if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["score_norm"]) + else: st.write("No data yet.") +st.divider(); st.caption("Set API_BASE or API_CANDIDATES to skip auto-detect.") +""", encoding="utf-8") + +# Shared lib +(common := root / "services/common"); common.mkdir(parents=True, exist_ok=True) +(common / "__init__.py").write_text("", encoding="utf-8") +(common / "config.py").write_text(""" +import os +from dataclasses import dataclass +@dataclass +class Config: + pg_user: str; pg_pass: str; pg_db: str; pg_host: str; pg_port: int; redis_url: str; symbols: list[str] +def load_config() -> Config: + symbols = [s.strip() for s in os.getenv("SYMBOLS","binance:BTC/USDT").split(",") if s.strip()] + return Config( + pg_user=os.getenv("POSTGRES_USER","cryptouser"), + pg_pass=os.getenv("POSTGRES_PASSWORD","cryptopass"), + pg_db=os.getenv("POSTGRES_DB","cryptodb"), + pg_host=os.getenv("POSTGRES_HOST","db"), + pg_port=int(os.getenv("POSTGRES_PORT","5432")), + redis_url=os.getenv("REDIS_URL","redis://redis:6379/0"), + symbols=symbols + ) +""", encoding="utf-8") +(common / "db.py").write_text(""" +import os, psycopg2, pandas as pd +from psycopg2.extras import execute_values +def _conn(): + url = os.getenv("DATABASE_URL") + if url: return psycopg2.connect(url) + return psycopg2.connect( + host=os.getenv("POSTGRES_HOST","db"), + port=int(os.getenv("POSTGRES_PORT","5432")), + dbname=os.getenv("POSTGRES_DB","cryptodb"), + user=os.getenv("POSTGRES_USER","cryptouser"), + password=os.getenv("POSTGRES_PASSWORD","cryptopass") + ) +def execute(sql, params=None): + with _conn() as conn, conn.cursor() as cur: cur.execute(sql, params or {}) +def fetch_df(sql, params=None)->pd.DataFrame: + with _conn() as conn: return pd.read_sql(sql, conn, params=params) +def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_cols: list[str]): + if not rows: return + cols=list(rows[0].keys()); vals=[[r[c] for c in cols] for r in rows] + on_conflict=", ".join(conflict_cols); updates=", ".join([f"{c}=EXCLUDED.{c}" for c in update_cols]) + sql=f"INSERT INTO {table} ({','.join(cols)}) VALUES %s ON CONFLICT ({on_conflict}) DO UPDATE SET {updates}" + with _conn() as conn, conn.cursor() as cur: execute_values(cur, sql, vals) +def ensure_schema(): + from services_common.schema import SCHEMA_SQL + execute(SCHEMA_SQL) +""", encoding="utf-8") +(common / "schema.py").write_text(""" +SCHEMA_SQL = ''' +CREATE TABLE IF NOT EXISTS candles (pair text NOT NULL, ts timestamptz NOT NULL, + open double precision, high double precision, low double precision, close double precision, volume double precision, + PRIMARY KEY (pair, ts)); +CREATE TABLE IF NOT EXISTS funding_rates (pair text NOT NULL, ts timestamptz NOT NULL, rate double precision, PRIMARY KEY (pair, ts)); +CREATE TABLE IF NOT EXISTS open_interest (pair text NOT NULL, ts timestamptz NOT NULL, value_usd double precision, PRIMARY KEY (pair, ts)); +CREATE TABLE IF NOT EXISTS volatility (pair text NOT NULL, ts timestamptz NOT NULL, atr double precision, PRIMARY KEY (pair, ts)); +CREATE TABLE IF NOT EXISTS sentiment (pair text NOT NULL, ts timestamptz NOT NULL, mentions integer, score_norm double precision, keywords jsonb, PRIMARY KEY (pair, ts)); +CREATE TABLE IF NOT EXISTS headlines (id bigserial PRIMARY KEY, ts timestamptz NOT NULL, source text, title text, url text, keywords jsonb); +CREATE TABLE IF NOT EXISTS signals (id bigserial PRIMARY KEY, ts timestamptz NOT NULL, pair text NOT NULL, regime text, bias text, long_prob double precision, short_prob double precision, summary text); +CREATE TABLE IF NOT EXISTS kv_store (k text PRIMARY KEY, v jsonb, updated_at timestamptz DEFAULT now()); +'; +""", encoding="utf-8") + +# Adapters & ingest & signals +(ad := common / "adapters"); ad.mkdir(parents=True, exist_ok=True) +(ad / "__init__.py").write_text("", encoding="utf-8") +(ad / "exchanges.py").write_text(""" +import datetime as dt, ccxt +def fetch_candles(exchange_pair: str, timeframe="1h", limit=200): + ex_name, pair = exchange_pair.split(":", 1) + ex = getattr(ccxt, ex_name)() + ohlcv = ex.fetch_ohlcv(pair, timeframe=timeframe, limit=limit) + rows=[] + for t,o,h,l,c,v in ohlcv: + ts=dt.datetime.utcfromtimestamp(t/1000).replace(tzinfo=dt.timezone.utc) + rows.append({"pair": exchange_pair,"ts": ts,"open": o,"high": h,"low": l,"close": c,"volume": v}) + return rows +def mock_funding(exchange_pair: str, candles_df): + rate=(candles_df["close"].pct_change().fillna(0).tail(1).iloc[0])/10 if len(candles_df) else 0.0 + ts=candles_df["ts"].tail(1).iloc[0] if len(candles_df) else dt.datetime.now(dt.timezone.utc) + return [{"pair": exchange_pair, "ts": ts, "rate": float(rate)}] +""", encoding="utf-8") +(ad / "open_interest.py").write_text(""" +import datetime as dt, random +def fetch_open_interest(exchange_pair: str): + now=dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + base=1_000_000 + random.randint(-50_000, 50_000) + return [{"pair": exchange_pair, "ts": now, "value_usd": float(base)}] +""", encoding="utf-8") +(ad / "volatility.py").write_text(""" +def compute_atr_like(candles_df, window=14): + if len(candles_df)==0: return None + df=candles_df.copy().sort_values("ts") + hl=df["high"]-df["low"] + hc=(df["high"]-df["close"].shift()).abs() + lc=(df["low"]-df["close"].shift()).abs() + tr=(hl.to_frame("hl").join(hc.to_frame("hc")).join(lc.to_frame("lc"))).max(axis=1) + atr=tr.rolling(window).mean() + last=df["ts"].iloc[-1] + return [{"pair": df["pair"].iloc[-1], "ts": last, "atr": float(atr.iloc[-1])}] +""", encoding="utf-8") +(ad / "sentiment.py").write_text(""" +import datetime as dt, random +KEYWORDS=["liquidation","margin call","rekt","funding","open interest"] +def fetch_sentiment_mock(exchange_pair: str): + now=dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + mentions=random.randint(5,50); score=random.uniform(-1,1) + kw_counts={k: random.randint(0, mentions//2) for k in KEYWORDS} + return [{"pair": exchange_pair, "ts": now, "mentions": mentions, "score_norm": score, "keywords": kw_counts}] +""", encoding="utf-8") +(ad / "headlines.py").write_text(""" +import datetime as dt +def fetch_headlines_mock(): + now=dt.datetime.now(dt.timezone.utc) + return [{"ts": now,"source":"mock","title":"Market wobbles as OI surges; funding flips negative","url":"https://example.com","keywords":["open interest","funding"]}] +""", encoding="utf-8") + +(common / "ingest.py").write_text(""" +import pandas as pd +from services_common.config import load_config +from services_common.db import upsert_many +from services_common.adapters.exchanges import fetch_candles, mock_funding +from services_common.adapters.open_interest import fetch_open_interest +from services_common.adapters.volatility import compute_atr_like +from services_common.adapters.sentiment import fetch_sentiment_mock +from services_common.adapters.headlines import fetch_headlines_mock +cfg = load_config() +def run_ingest_cycle(): + for pair in cfg.symbols: + candle_rows = fetch_candles(pair, timeframe="1h", limit=200) + upsert_many("candles", candle_rows, ["pair","ts"], ["open","high","low","close","volume"]) + df = pd.DataFrame(candle_rows) + upsert_many("funding_rates", mock_funding(pair, df), ["pair","ts"], ["rate"]) + upsert_many("open_interest", fetch_open_interest(pair), ["pair","ts"], ["value_usd"]) + vol = compute_atr_like(df) + if vol: upsert_many("volatility", vol, ["pair","ts"], ["atr"]) + upsert_many("sentiment", fetch_sentiment_mock(pair), ["pair","ts"], ["mentions","score_norm","keywords"]) + upsert_many("headlines", fetch_headlines_mock(), ["id"], []) +""", encoding="utf-8") + +(common / "signals.py").write_text(""" +import pandas as pd +from services_common.db import fetch_df, upsert_many +from datetime import datetime, timezone +def _percentile(series: pd.Series, value: float): + if series.empty: return None + return (series < value).mean() * 100.0 +def compute_market_stress(pair: str): + oi = fetch_df("SELECT ts, value_usd FROM open_interest WHERE pair=%(pair)s AND ts > now() - interval '30 days' ORDER BY ts", {"pair": pair}) + fr = fetch_df("SELECT ts, rate FROM funding_rates WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts", {"pair": pair}) + sent = fetch_df("SELECT ts, mentions, score_norm, keywords FROM sentiment WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts", {"pair": pair}) + regime, bias = "Unknown", "Neutral"; long_prob = short_prob = 0.5; summary = "Insufficient data." + if not oi.empty and not fr.empty and not sent.empty: + latest_oi = oi["value_usd"].iloc[-1]; oi_pct = _percentile(oi["value_usd"], latest_oi); latest_funding = fr["rate"].iloc[-1] + sent["liq_kw"] = sent["keywords"].apply(lambda d: (d.get("liquidation",0) if isinstance(d, dict) else 0) + (d.get("margin call",0) if isinstance(d, dict) else 0)) + last_week = sent.tail(max(1, len(sent)//2)); first_week = sent.head(len(sent)-len(last_week)) if len(sent)>1 else sent.head(1) + base = max(1, first_week["liq_kw"].sum()); spike_ratio = (last_week["liq_kw"].sum()) / base + if (oi_pct is not None) and oi_pct >= 90 and latest_funding < 0 and spike_ratio >= 2.0: + regime, bias, long_prob, short_prob = "Risky / High Liquidation Risk","Short",0.25,0.75 + summary = "OI in 90th pct+, funding negative, and liquidation mentions up ≥200%. Consider caution or short bias." + else: + candles = fetch_df("SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 50", {"pair": pair}).sort_values("ts") + slope = candles["close"].pct_change().fillna(0).tail(10).mean() if not candles.empty else 0.0 + if slope > 0 and latest_funding >= 0: + regime, bias, long_prob, short_prob = "Constructive","Long",0.65,0.35 + summary = "Upward momentum + non-negative funding → modest long bias." + elif slope < 0 and latest_funding <= 0: + regime, bias, long_prob, short_prob = "Weak","Short",0.35,0.65 + summary = "Downward momentum + non-positive funding → modest short bias." + else: + regime, bias, long_prob, short_prob = "Balanced / Choppy","Flat",0.5,0.5 + summary = "Mixed signals; consider mean-reversion or wait." + return {"pair": pair, "regime": regime, "bias": bias, "long_prob": float(long_prob), "short_prob": float(short_prob), "summary": summary} +def compute_all_signals(): + dfpairs = fetch_df("SELECT DISTINCT pair FROM candles"); out=[] + for p in dfpairs.get("pair", []): out.append(compute_market_stress(p)) + now = datetime.now(timezone.utc) + rows = [{"ts": now,"pair": s["pair"],"regime": s["regime"],"bias": s["bias"],"long_prob": s["long_prob"],"short_prob": s["short_prob"],"summary": s["summary"]} for s in out] + if rows: upsert_many("signals", rows, ["id"], []) +def latest_signals_for_pairs(pairs: list[str]): + placeholders = ",".join(["%s"]*len(pairs)) + q = f"SELECT DISTINCT ON (pair) pair, ts, regime, bias, long_prob, short_prob, summary FROM signals WHERE pair IN ({placeholders}) ORDER BY pair, ts DESC" + df = fetch_df(q, pairs); result={} + for _, r in df.iterrows(): result[r["pair"]] = dict(r) + return result +def signal_explanations(): + return {"market_stress": "OI ≥ 90th pct + negative funding + liquidation keyword spike ≥ 200% → bearish risk."} +""", encoding="utf-8") + +# Make services_common importable in Docker builds +(api / "services_common").mkdir(exist_ok=True) +(worker / "services_common").mkdir(exist_ok=True) +(api / "services_common/__init__.py").write_text("from pathlib import Path as _P; import sys as _s; _s.path.append(str(_P(__file__).resolve().parents[2]))\n", encoding="utf-8") +(worker / "services_common/__init__.py").write_text("from pathlib import Path as _P; import sys as _s; _s.path.append(str(_P(__file__).resolve().parents[2]))\n", encoding="utf-8") + +# Upload all files +print("Uploading files to GitHub (branch 'main')...") +for p in root.rglob("*"): + if p.is_dir(): continue + rel = str(p.relative_to(root)).replace("\\", "/") + content = p.read_bytes() + gh_put_file(gh_user, repo, token, rel, content, f"Add {rel}", branch="main") + +print("\n✅ Done!") +print(f"GitHub repository: https://github.com/{gh_user}/{repo}") +print("Next: Render → Blueprints → New Blueprint → pick this repo → Create.") From b26451ec1dc41e6b088895e904ddd947be652450 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 15:02:51 +0100 Subject: [PATCH 02/43] . --- .env.example | 15 ++ .gitignore | 7 + README.md | 16 ++ docker-compose.yml | 51 +++++++ render.yaml | 44 ++++++ services/api/Dockerfile | 6 + services/api/main.py | 35 +++++ services/api/requirements.txt | 19 +++ services/api/services_common/__init__.py | 1 + services/common/__init__.py | 0 services/common/adapters/__init__.py | 0 services/common/adapters/exchanges.py | 16 ++ services/common/adapters/headlines.py | 8 + services/common/adapters/open_interest.py | 6 + services/common/adapters/sentiment.py | 9 ++ services/common/adapters/volatility.py | 10 ++ services/common/config.py | 25 ++++ services/common/db.py | 51 +++++++ services/common/ingest.py | 28 ++++ services/common/schema.py | 68 +++++++++ services/common/schema_timescale.sql | 27 ++++ services/common/signals.py | 68 +++++++++ services/ui/Dockerfile | 6 + services/ui/app.py | 155 ++++++++++++++++++++ services/ui/requirements.txt | 22 +++ services/worker/Dockerfile | 6 + services/worker/requirements.txt | 19 +++ services/worker/run_worker.py | 16 ++ services/worker/services_common/__init__.py | 1 + 29 files changed, 735 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 render.yaml create mode 100644 services/api/Dockerfile create mode 100644 services/api/main.py create mode 100644 services/api/requirements.txt create mode 100644 services/api/services_common/__init__.py create mode 100644 services/common/__init__.py create mode 100644 services/common/adapters/__init__.py create mode 100644 services/common/adapters/exchanges.py create mode 100644 services/common/adapters/headlines.py create mode 100644 services/common/adapters/open_interest.py create mode 100644 services/common/adapters/sentiment.py create mode 100644 services/common/adapters/volatility.py create mode 100644 services/common/config.py create mode 100644 services/common/db.py create mode 100644 services/common/ingest.py create mode 100644 services/common/schema.py create mode 100644 services/common/schema_timescale.sql create mode 100644 services/common/signals.py create mode 100644 services/ui/Dockerfile create mode 100644 services/ui/app.py create mode 100644 services/ui/requirements.txt create mode 100644 services/worker/Dockerfile create mode 100644 services/worker/requirements.txt create mode 100644 services/worker/run_worker.py create mode 100644 services/worker/services_common/__init__.py diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..919ed824 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +POSTGRES_USER=cryptouser +POSTGRES_PASSWORD=cryptopass +POSTGRES_DB=cryptodb +POSTGRES_HOST=db +POSTGRES_PORT=5432 +REDIS_URL=redis://redis:6379/0 +SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT +SCHEDULE_MINUTES=5 +REDDIT_CLIENT_ID= +REDDIT_CLIENT_SECRET= +REDDIT_USER_AGENT=crypto-risk-app/0.1 by you +X_BEARER_TOKEN= +CRYPTOPANIC_API_KEY= +NEWSAPI_KEY= +COINGLASS_API_KEY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..91bf069a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +.env +db_data/ +.idea/ +.vscode/ +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 00000000..c88e06ae --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# Crypto Risk Dashboard — Self-Hosted Starter + +Self-hosted, Dockerized stack with: +- **API** (FastAPI) +- **Worker** (scheduled ingest + signals) +- **DB** (Postgres; Timescale-ready) +- **UI** (Streamlit) with graphs, meters, hot signals +- **Adapters** for candles, funding, OI, sentiment, headlines (mock by default) + +## Quick start (local Docker) +1) `cp .env.example .env` +2) `docker compose up --build` +3) UI: http://localhost:8501, API: http://localhost:8000 + +## Render deploy +Use the `render.yaml` blueprint (free Postgres + API + UI + worker). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..d2c3f71c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +services: + db: + image: postgres:16 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - db_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 20 + + redis: + image: redis:7 + ports: + - "6379:6379" + + api: + build: ./services/api + env_file: .env + depends_on: + db: + condition: service_healthy + ports: + - "8000:8000" + + worker: + build: ./services/worker + env_file: .env + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + + ui: + build: ./services/ui + env_file: .env + depends_on: + api: + condition: service_started + ports: + - "8501:8501" + +volumes: + db_data: diff --git a/render.yaml b/render.yaml new file mode 100644 index 00000000..1be37499 --- /dev/null +++ b/render.yaml @@ -0,0 +1,44 @@ +databases: + - name: cryptodb + databaseName: cryptodb + plan: free + +services: + - type: web + name: crypto-risk-api + runtime: docker + rootDir: services/api + plan: free + envVars: + - key: DATABASE_URL + fromDatabase: + name: cryptodb + property: connectionString + - key: SYMBOLS + value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + - key: SCHEDULE_MINUTES + value: "5" + + - type: web + name: crypto-risk-ui + runtime: docker + rootDir: services/ui + plan: free + envVars: + - key: API_CANDIDATES + value: https://crypto-risk-api.onrender.com,http://api:8000,http://localhost:8000 + + - type: worker + name: crypto-risk-worker + runtime: docker + rootDir: services/worker + plan: free + envVars: + - key: DATABASE_URL + fromDatabase: + name: cryptodb + property: connectionString + - key: SYMBOLS + value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + - key: SCHEDULE_MINUTES + value: "5" diff --git a/services/api/Dockerfile b/services/api/Dockerfile new file mode 100644 index 00000000..f8fda52b --- /dev/null +++ b/services/api/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/api/main.py b/services/api/main.py new file mode 100644 index 00000000..6bf82a73 --- /dev/null +++ b/services/api/main.py @@ -0,0 +1,35 @@ +import os +from fastapi import FastAPI, Query +from services_common.db import fetch_df +from services_common.signals import latest_signals_for_pairs, signal_explanations +from services_common.config import load_config + +app = FastAPI(title="Crypto Risk API", version="0.1.0") +cfg = load_config() + +@app.get("/health") +def health(): + return {"ok": True} + +@app.get("/pairs") +def pairs(): + return {"pairs": cfg.symbols} + +@app.get("/signals") +def get_signals(pairs: str = Query(None)): + if pairs: + pairs_list = [p.strip() for p in pairs.split(",") if p.strip()] + else: + pairs_list = cfg.symbols + data = latest_signals_for_pairs(pairs_list) + return {"signals": data, "explanations": signal_explanations()} + +@app.get("/timeseries/{metric}") +def timeseries(metric: str, pair: str, limit: int = 500): + table_map = {"candles":"candles","funding":"funding_rates","oi":"open_interest","vol":"volatility","sentiment":"sentiment"} + table = table_map.get(metric) + if not table: + return {"error":"unknown metric"} + q = f"""SELECT * FROM {table} WHERE pair=%(pair)s ORDER BY ts DESC LIMIT %(limit)s""" + df = fetch_df(q, {"pair": pair, "limit": limit}) + return {"columns": list(df.columns), "rows": df.to_dict(orient="records")} diff --git a/services/api/requirements.txt b/services/api/requirements.txt new file mode 100644 index 00000000..0941b5de --- /dev/null +++ b/services/api/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.115.2 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +python-dotenv==1.0.1 +psycopg2-binary==2.9.9 +SQLAlchemy==2.0.36 +alembic==1.13.2 +redis==5.0.8 +celery==5.4.0 +pandas==2.2.3 +numpy==2.1.2 +plotly==5.24.1 +matplotlib==3.9.2 +ccxt==4.4.6 +scikit-learn==1.5.2 +textblob==0.18.0.post0 +nltk==3.9.1 +requests==2.32.3 +beautifulsoup4==4.12.3 diff --git a/services/api/services_common/__init__.py b/services/api/services_common/__init__.py new file mode 100644 index 00000000..49d5e07d --- /dev/null +++ b/services/api/services_common/__init__.py @@ -0,0 +1 @@ +from pathlib import Path as _P; import sys as _s; _s.path.append(str(_P(__file__).resolve().parents[2])) diff --git a/services/common/__init__.py b/services/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/common/adapters/__init__.py b/services/common/adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/common/adapters/exchanges.py b/services/common/adapters/exchanges.py new file mode 100644 index 00000000..bd9ffbf4 --- /dev/null +++ b/services/common/adapters/exchanges.py @@ -0,0 +1,16 @@ +import datetime as dt, ccxt + +def fetch_candles(exchange_pair: str, timeframe="1h", limit=200): + ex_name, pair = exchange_pair.split(":", 1) + ex = getattr(ccxt, ex_name)() + ohlcv = ex.fetch_ohlcv(pair, timeframe=timeframe, limit=limit) + rows = [] + for t,o,h,l,c,v in ohlcv: + ts = dt.datetime.utcfromtimestamp(t/1000).replace(tzinfo=dt.timezone.utc) + rows.append({"pair": exchange_pair, "ts": ts, "open": o, "high": h, "low": l, "close": c, "volume": v}) + return rows + +def mock_funding(exchange_pair: str, candles_df): + rate = (candles_df["close"].pct_change().fillna(0).tail(1).iloc[0]) / 10 if len(candles_df) else 0.0 + ts = candles_df["ts"].tail(1).iloc[0] if len(candles_df) else dt.datetime.now(dt.timezone.utc) + return [{"pair": exchange_pair, "ts": ts, "rate": float(rate)}] diff --git a/services/common/adapters/headlines.py b/services/common/adapters/headlines.py new file mode 100644 index 00000000..2a5d60d9 --- /dev/null +++ b/services/common/adapters/headlines.py @@ -0,0 +1,8 @@ +import datetime as dt +def fetch_headlines_mock(): + now = dt.datetime.now(dt.timezone.utc) + return [{ + "ts": now, "source": "mock", + "title": "Market wobbles as OI surges; funding flips negative", + "url": "https://example.com", "keywords": ["open interest","funding"] + }] diff --git a/services/common/adapters/open_interest.py b/services/common/adapters/open_interest.py new file mode 100644 index 00000000..04879d30 --- /dev/null +++ b/services/common/adapters/open_interest.py @@ -0,0 +1,6 @@ +import datetime as dt, random + +def fetch_open_interest(exchange_pair: str): + now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + base = 1_000_000 + random.randint(-50_000, 50_000) + return [{"pair": exchange_pair, "ts": now, "value_usd": float(base)}] diff --git a/services/common/adapters/sentiment.py b/services/common/adapters/sentiment.py new file mode 100644 index 00000000..49aa852d --- /dev/null +++ b/services/common/adapters/sentiment.py @@ -0,0 +1,9 @@ +import datetime as dt, random +KEYWORDS = ["liquidation","margin call","rekt","funding","open interest"] + +def fetch_sentiment_mock(exchange_pair: str): + now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + mentions = random.randint(5, 50) + score = random.uniform(-1, 1) + kw_counts = {k: random.randint(0, mentions//2) for k in KEYWORDS} + return [{"pair": exchange_pair, "ts": now, "mentions": mentions, "score_norm": score, "keywords": kw_counts}] diff --git a/services/common/adapters/volatility.py b/services/common/adapters/volatility.py new file mode 100644 index 00000000..26eacc7e --- /dev/null +++ b/services/common/adapters/volatility.py @@ -0,0 +1,10 @@ +def compute_atr_like(candles_df, window=14): + if len(candles_df)==0: return None + df = candles_df.copy().sort_values("ts") + hl = df["high"] - df["low"] + hc = (df["high"] - df["close"].shift()).abs() + lc = (df["low"] - df["close"].shift()).abs() + tr = (hl.to_frame("hl").join(hc.to_frame("hc")).join(lc.to_frame("lc"))).max(axis=1) + atr = tr.rolling(window).mean() + last = df["ts"].iloc[-1] + return [{"pair": df["pair"].iloc[-1], "ts": last, "atr": float(atr.iloc[-1])}] diff --git a/services/common/config.py b/services/common/config.py new file mode 100644 index 00000000..0f515f72 --- /dev/null +++ b/services/common/config.py @@ -0,0 +1,25 @@ +import os +from dataclasses import dataclass + +@dataclass +class Config: + pg_user: str + pg_pass: str + pg_db: str + pg_host: str + pg_port: int + redis_url: str + symbols: list[str] + +def load_config() -> Config: + symbols_env = os.getenv("SYMBOLS", "binance:BTC/USDT") + symbols = [s.strip() for s in symbols_env.split(",") if s.strip()] + return Config( + pg_user=os.getenv("POSTGRES_USER","cryptouser"), + pg_pass=os.getenv("POSTGRES_PASSWORD","cryptopass"), + pg_db=os.getenv("POSTGRES_DB","cryptodb"), + pg_host=os.getenv("POSTGRES_HOST","db"), + pg_port=int(os.getenv("POSTGRES_PORT","5432")), + redis_url=os.getenv("REDIS_URL","redis://redis:6379/0"), + symbols=symbols + ) diff --git a/services/common/db.py b/services/common/db.py new file mode 100644 index 00000000..35aa44f9 --- /dev/null +++ b/services/common/db.py @@ -0,0 +1,51 @@ +import os, psycopg2, pandas as pd +from psycopg2.extras import execute_values + +def _conn(): + url = os.getenv("DATABASE_URL") + if url: + return psycopg2.connect(url) + return psycopg2.connect( + host=_cfg.pg_host, port=_cfg.pg_port, dbname=_cfg.pg_db, + user=_cfg.pg_user, password=_cfg.pg_pass + # fallback to local docker compose + host = os.getenv("POSTGRES_HOST","db") + port = int(os.getenv("POSTGRES_PORT","5432")) + db = os.getenv("POSTGRES_DB","cryptodb") + user = os.getenv("POSTGRES_USER","cryptouser") + pwd = os.getenv("POSTGRES_PASSWORD","cryptopass") + return psycopg2.connect(host=host, port=port, dbname=db, user=user, password=pwd) + +def execute(sql, params=None): + with _conn() as conn, conn.cursor() as cur: + cur.execute(sql, params or {}) + +def fetch_df(sql, params=None) -> pd.DataFrame: + with _conn() as conn: + return pd.read_sql(sql, conn, params=params) + +def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_cols: list[str]): + if not rows: return + cols = list(rows[0].keys()) + vals = [[r[c] for c in cols] for r in rows] + on_conflict = ", ".join(conflict_cols) + updates = ", ".join([f"{c}=EXCLUDED.{c}" for c in update_cols]) + sql = f""" + INSERT INTO {table} ({",".join(cols)}) + VALUES %s + ON CONFLICT ({on_conflict}) DO UPDATE SET {updates} + """ + with _conn() as conn, conn.cursor() as cur: + execute_values(cur, sql, vals) + +def ensure_schema(): + from services_common.schema import SCHEMA_SQL + execute(SCHEMA_SQL) + try: + from pathlib import Path + ts_sql = (Path(__file__).parent / "schema_timescale.sql") + if ts_sql.exists(): + with _conn() as conn, conn.cursor() as cur: + cur.execute(ts_sql.read_text(encoding="utf-8")) + except Exception: + pass diff --git a/services/common/ingest.py b/services/common/ingest.py new file mode 100644 index 00000000..2c510370 --- /dev/null +++ b/services/common/ingest.py @@ -0,0 +1,28 @@ +import pandas as pd +from services_common.config import load_config +from services_common.db import upsert_many +from services_common.adapters.exchanges import fetch_candles, mock_funding +from services_common.adapters.open_interest import fetch_open_interest +from services_common.adapters.volatility import compute_atr_like +from services_common.adapters.sentiment import fetch_sentiment_mock +from services_common.adapters.headlines import fetch_headlines_mock + +cfg = load_config() + +def run_ingest_cycle(): + for pair in cfg.symbols: + candle_rows = fetch_candles(pair, timeframe="1h", limit=200) + upsert_many("candles", candle_rows, ["pair","ts"], ["open","high","low","close","volume"]) + + df = pd.DataFrame(candle_rows) + fr = mock_funding(pair, df); upsert_many("funding_rates", fr, ["pair","ts"], ["rate"]) + + oi = fetch_open_interest(pair); upsert_many("open_interest", oi, ["pair","ts"], ["value_usd"]) + + vol = compute_atr_like(df) + if vol: upsert_many("volatility", vol, ["pair","ts"], ["atr"]) + + sent = fetch_sentiment_mock(pair); upsert_many("sentiment", sent, ["pair","ts"], ["mentions","score_norm","keywords"]) + + h = fetch_headlines_mock() + upsert_many("headlines", h, ["id"], []) diff --git a/services/common/schema.py b/services/common/schema.py new file mode 100644 index 00000000..0aac2a76 --- /dev/null +++ b/services/common/schema.py @@ -0,0 +1,68 @@ +SCHEMA_SQL = ''' +CREATE TABLE IF NOT EXISTS candles ( + pair text NOT NULL, + ts timestamptz NOT NULL, + open double precision, + high double precision, + low double precision, + close double precision, + volume double precision, + PRIMARY KEY (pair, ts) +); + +CREATE TABLE IF NOT EXISTS funding_rates ( + pair text NOT NULL, + ts timestamptz NOT NULL, + rate double precision, + PRIMARY KEY (pair, ts) +); + +CREATE TABLE IF NOT EXISTS open_interest ( + pair text NOT NULL, + ts timestamptz NOT NULL, + value_usd double precision, + PRIMARY KEY (pair, ts) +); + +CREATE TABLE IF NOT EXISTS volatility ( + pair text NOT NULL, + ts timestamptz NOT NULL, + atr double precision, + PRIMARY KEY (pair, ts) +); + +CREATE TABLE IF NOT EXISTS sentiment ( + pair text NOT NULL, + ts timestamptz NOT NULL, + mentions integer, + score_norm double precision, + keywords jsonb, + PRIMARY KEY (pair, ts) +); + +CREATE TABLE IF NOT EXISTS headlines ( + id bigserial PRIMARY KEY, + ts timestamptz NOT NULL, + source text, + title text, + url text, + keywords jsonb +); + +CREATE TABLE IF NOT EXISTS signals ( + id bigserial PRIMARY KEY, + ts timestamptz NOT NULL, + pair text NOT NULL, + regime text, + bias text, + long_prob double precision, + short_prob double precision, + summary text +); + +CREATE TABLE IF NOT EXISTS kv_store ( + k text PRIMARY KEY, + v jsonb, + updated_at timestamptz DEFAULT now() +); +''' diff --git a/services/common/schema_timescale.sql b/services/common/schema_timescale.sql new file mode 100644 index 00000000..15e97882 --- /dev/null +++ b/services/common/schema_timescale.sql @@ -0,0 +1,27 @@ +CREATE EXTENSION IF NOT EXISTS timescaledb; +SELECT create_hypertable('candles', 'ts', if_not_exists => TRUE); +SELECT create_hypertable('funding_rates', 'ts', if_not_exists => TRUE); +SELECT create_hypertable('open_interest', 'ts', if_not_exists => TRUE); +SELECT create_hypertable('volatility', 'ts', if_not_exists => TRUE); +SELECT create_hypertable('sentiment', 'ts', if_not_exists => TRUE); +SELECT create_hypertable('signals', 'ts', if_not_exists => TRUE); + +CREATE MATERIALIZED VIEW IF NOT EXISTS candles_1h +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 hour', ts) AS bucket, + pair, + first(open, ts) AS o, + max(high) AS h, + min(low) AS l, + last(close, ts) AS c, + sum(volume) AS v +FROM candles +GROUP BY 1,2; + +SELECT add_continuous_aggregate_policy( + 'candles_1h', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '5 minutes', + schedule_interval => INTERVAL '15 minutes' +); diff --git a/services/common/signals.py b/services/common/signals.py new file mode 100644 index 00000000..fcc08387 --- /dev/null +++ b/services/common/signals.py @@ -0,0 +1,68 @@ +import pandas as pd +from services_common.db import fetch_df, upsert_many +from datetime import datetime, timezone + +def _percentile(series: pd.Series, value: float): + if series.empty: return None + return (series < value).mean() * 100.0 + +def compute_market_stress(pair: str): + oi = fetch_df("""SELECT ts, value_usd FROM open_interest WHERE pair=%(pair)s AND ts > now() - interval '30 days' ORDER BY ts""", {"pair": pair}) + fr = fetch_df("""SELECT ts, rate FROM funding_rates WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts""", {"pair": pair}) + sent = fetch_df("""SELECT ts, mentions, score_norm, keywords FROM sentiment WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts""", {"pair": pair}) + + regime, bias = "Unknown", "Neutral" + long_prob = short_prob = 0.5 + summary = "Insufficient data." + + if not oi.empty and not fr.empty and not sent.empty: + latest_oi = oi["value_usd"].iloc[-1] + oi_pct = _percentile(oi["value_usd"], latest_oi) + latest_funding = fr["rate"].iloc[-1] + + sent["liq_kw"] = sent["keywords"].apply(lambda d: (d.get("liquidation",0) if isinstance(d, dict) else 0) + (d.get("margin call",0) if isinstance(d, dict) else 0)) + last_week = sent.tail(max(1, len(sent)//2)) + first_week = sent.head(len(sent)-len(last_week)) if len(sent)>1 else sent.head(1) + base = max(1, first_week["liq_kw"].sum()) + spike_ratio = (last_week["liq_kw"].sum()) / base + + if (oi_pct is not None) and oi_pct >= 90 and latest_funding < 0 and spike_ratio >= 2.0: + regime = "Risky / High Liquidation Risk"; bias = "Short"; long_prob, short_prob = 0.25, 0.75 + summary = "OI in 90th pct+, funding negative, and liquidation mentions up ≥200%. Consider caution or short bias." + else: + candles = fetch_df("""SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 50""", {"pair": pair}).sort_values("ts") + slope = candles["close"].pct_change().fillna(0).tail(10).mean() if not candles.empty else 0.0 + if slope > 0 and latest_funding >= 0: + regime = "Constructive"; bias = "Long"; long_prob, short_prob = 0.65, 0.35 + summary = "Upward momentum + non-negative funding → modest long bias." + elif slope < 0 and latest_funding <= 0: + regime = "Weak"; bias = "Short"; long_prob, short_prob = 0.35, 0.65 + summary = "Downward momentum + non-positive funding → modest short bias." + else: + regime = "Balanced / Choppy"; bias = "Flat"; long_prob = short_prob = 0.5 + summary = "Mixed signals; consider mean-reversion or wait." + return {"pair": pair, "regime": regime, "bias": bias, "long_prob": float(long_prob), "short_prob": float(short_prob), "summary": summary} + +def compute_all_signals(): + dfpairs = fetch_df("SELECT DISTINCT pair FROM candles") + out = [] + for p in dfpairs.get("pair", []): + s = compute_market_stress(p); out.append(s) + now = datetime.now(timezone.utc) + rows = [{ + "ts": now, "pair": s["pair"], "regime": s["regime"], "bias": s["bias"], + "long_prob": s["long_prob"], "short_prob": s["short_prob"], "summary": s["summary"] + } for s in out] + if rows: upsert_many("signals", rows, ["id"], []) + +def latest_signals_for_pairs(pairs: list[str]): + placeholders = ",".join(["%s"]*len(pairs)) + q = f"""SELECT DISTINCT ON (pair) pair, ts, regime, bias, long_prob, short_prob, summary + FROM signals WHERE pair IN ({placeholders}) ORDER BY pair, ts DESC""" + df = fetch_df(q, pairs) + result = {} + for _, r in df.iterrows(): result[r["pair"]] = dict(r) + return result + +def signal_explanations(): + return {"market_stress": "OI ≥ 90th pct + negative funding + liquidation keyword spike ≥ 200% → bearish risk."} diff --git a/services/ui/Dockerfile b/services/ui/Dockerfile new file mode 100644 index 00000000..0cdd3f3a --- /dev/null +++ b/services/ui/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] diff --git a/services/ui/app.py b/services/ui/app.py new file mode 100644 index 00000000..54929773 --- /dev/null +++ b/services/ui/app.py @@ -0,0 +1,155 @@ +import os, requests, pandas as pd, time +import streamlit as st +from urllib.parse import urlencode + +DEFAULT_CANDIDATES = [ + # 1) Render default if you kept the blueprint names: + "https://crypto-risk-api.onrender.com", + # 2) Local docker compose: + "http://api:8000", + "http://localhost:8000", +] + +def probe_api(base: str, timeout=2.0) -> bool: + try: + r = requests.get(f"{base}/health", timeout=timeout) + return r.ok and r.json().get("ok") is True + except Exception: + return False + +def resolve_api_base(): + # Priority 1: explicit env var + env_base = os.getenv("API_BASE", "").strip() + if env_base and probe_api(env_base): + return env_base + + # Priority 2: comma-separated list of candidates from env + env_candidates = os.getenv("API_CANDIDATES", "") + candidates = [c.strip() for c in env_candidates.split(",") if c.strip()] if env_candidates else [] + # Merge with defaults (preserve order; avoid dups) + seen = set() + merged = [] + for x in (candidates + DEFAULT_CANDIDATES): + if x not in seen: + merged.append(x) + seen.add(x) + + for base in merged: + if probe_api(base): + return base + + return None # nothing worked + +API_BASE = resolve_api_base() + +st.set_page_config(page_title="Crypto Risk Dashboard", layout="wide") +st.title("🧭 Crypto Risk Dashboard") +st.caption("Self-hosted. Graphs • Meters • Hot Signals") + +if not API_BASE: + st.error("Could not locate the API automatically.") + manual = st.text_input("Enter your API base URL (e.g., https://crypto-risk-api.onrender.com)") + if manual: + if probe_api(manual): + API_BASE = manual + st.success("Connected!") + else: + st.warning("That URL didn’t respond at /health. Double-check and try again.") + +if not API_BASE: + st.stop() + +# Pairs +resp = requests.get(f"{API_BASE}/pairs", timeout=30).json() +pairs = resp.get("pairs", ["binance:BTC/USDT"]) + +col1, col2 = st.columns([2,1]) +with col1: + pair = st.selectbox("Pair", pairs, index=0) +with col2: + refresh = st.button("Refresh") + +def fetch(metric, pair, limit=500): + qs = urlencode({"pair": pair, "limit": limit}) + r = requests.get(f"{API_BASE}/timeseries/{metric}?{qs}", timeout=30) + return r.json() + +def get_signals(pairs=None): + qs = "" + if pairs: + qs = "?pairs=" + ",".join(pairs) + r = requests.get(f"{API_BASE}/signals{qs}", timeout=30) + return r.json() + +sig = get_signals([pair]) +signals = sig.get("signals", {}) +explanations = sig.get("explanations", {}) + +st.subheader("Hot Signals") +if signals.get(pair): + s = signals[pair] + c1, c2, c3, c4 = st.columns(4) + with c1: + st.metric("Market Regime", s.get("regime","—")) + with c2: + st.metric("Bias", s.get("bias","—")) + with c3: + st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) + with c4: + st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) + st.info(s.get("summary","")) +else: + st.warning("No signals yet. The worker may still be seeding data.") + +st.divider() +st.subheader("Charts") + +tabs = st.tabs(["Candles", "Funding", "Open Interest", "Volatility", "Sentiment"]) + +with tabs[0]: + data = fetch("candles", pair, limit=300) + rows = data.get("rows", []) + if rows: + df = pd.DataFrame(rows).sort_values("ts") + st.line_chart(df, x="ts", y=["close"]) + else: + st.write("No data yet.") + +with tabs[1]: + data = fetch("funding", pair, limit=500) + rows = data.get("rows", []) + if rows: + df = pd.DataFrame(rows).sort_values("ts") + st.line_chart(df, x="ts", y=["rate"]) + else: + st.write("No data yet.") + +with tabs[2]: + data = fetch("oi", pair, limit=500) + rows = data.get("rows", []) + if rows: + df = pd.DataFrame(rows).sort_values("ts") + st.line_chart(df, x="ts", y=["value_usd"]) + else: + st.write("No data yet.") + +with tabs[3]: + data = fetch("vol", pair, limit=500) + rows = data.get("rows", []) + if rows: + df = pd.DataFrame(rows).sort_values("ts") + st.line_chart(df, x="ts", y=["atr"]) + else: + st.write("No data yet.") + +with tabs[4]: + data = fetch("sentiment", pair, limit=200) + rows = data.get("rows", []) + if rows: + df = pd.DataFrame(rows).sort_values("ts") + st.line_chart(df, x="ts", y=["score_norm"]) + else: + st.write("No data yet.") + +st.divider() +st.caption("Tip: set API_BASE or API_CANDIDATES env vars to skip auto-detect.") diff --git a/services/ui/requirements.txt b/services/ui/requirements.txt new file mode 100644 index 00000000..3f258ee0 --- /dev/null +++ b/services/ui/requirements.txt @@ -0,0 +1,22 @@ +fastapi==0.115.2 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +python-dotenv==1.0.1 +psycopg2-binary==2.9.9 +SQLAlchemy==2.0.36 +alembic==1.13.2 +redis==5.0.8 +celery==5.4.0 +pandas==2.2.3 +numpy==2.1.2 +plotly==5.24.1 +matplotlib==3.9.2 +ccxt==4.4.6 +scikit-learn==1.5.2 +textblob==0.18.0.post0 +nltk==3.9.1 +requests==2.32.3 +beautifulsoup4==4.12.3 + +streamlit==1.39.0 +streamlit_echarts==0.4.0 diff --git a/services/worker/Dockerfile b/services/worker/Dockerfile new file mode 100644 index 00000000..d927251c --- /dev/null +++ b/services/worker/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python", "run_worker.py"] diff --git a/services/worker/requirements.txt b/services/worker/requirements.txt new file mode 100644 index 00000000..0941b5de --- /dev/null +++ b/services/worker/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.115.2 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +python-dotenv==1.0.1 +psycopg2-binary==2.9.9 +SQLAlchemy==2.0.36 +alembic==1.13.2 +redis==5.0.8 +celery==5.4.0 +pandas==2.2.3 +numpy==2.1.2 +plotly==5.24.1 +matplotlib==3.9.2 +ccxt==4.4.6 +scikit-learn==1.5.2 +textblob==0.18.0.post0 +nltk==3.9.1 +requests==2.32.3 +beautifulsoup4==4.12.3 diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py new file mode 100644 index 00000000..47c22a47 --- /dev/null +++ b/services/worker/run_worker.py @@ -0,0 +1,16 @@ +import os, time +from services_common.db import ensure_schema +from services_common.ingest import run_ingest_cycle +from services_common.signals import compute_all_signals + +def main(): + ensure_schema() + interval = int(os.getenv("SCHEDULE_MINUTES","5")) + print(f"[worker] schedule {interval}m") + while True: + print("[worker] ingest..."); run_ingest_cycle() + print("[worker] signals..."); compute_all_signals() + print("[worker] sleep..."); time.sleep(interval*60) + +if __name__ == "__main__": + main() diff --git a/services/worker/services_common/__init__.py b/services/worker/services_common/__init__.py new file mode 100644 index 00000000..49d5e07d --- /dev/null +++ b/services/worker/services_common/__init__.py @@ -0,0 +1 @@ +from pathlib import Path as _P; import sys as _s; _s.path.append(str(_P(__file__).resolve().parents[2])) From eca2a10a804a278bd4046bd07c8dfe5df429815f Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 15:12:00 +0100 Subject: [PATCH 03/43] v2 --- .env.example | 16 +- README.md | 31 +- docker-compose.yml | 7 - services/api/main.py | 31 +- services/api/requirements.txt | 2 + services/common/adapters/exchanges.py | 9 +- services/common/adapters/headlines.py | 5 +- services/common/adapters/open_interest.py | 1 - services/common/adapters/sentiment.py | 3 +- services/common/adapters/volatility.py | 8 +- services/common/db.py | 27 +- services/common/ingest.py | 12 +- services/common/schema_timescale.sql | 5 + services/common/signals.py | 99 ++-- services/ui/app.py | 47 +- services/ui/requirements.txt | 1 - services/worker/requirements.txt | 2 + services/worker/run_worker.py | 14 +- timescale.env | 5 + upload_to_existing_repo.py | 545 ---------------------- 20 files changed, 196 insertions(+), 674 deletions(-) create mode 100644 timescale.env delete mode 100644 upload_to_existing_repo.py diff --git a/.env.example b/.env.example index 919ed824..b984d17f 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,25 @@ +# EITHER prefer DATABASE_URL (works for Postgres or TimescaleDB) ... +# DATABASE_URL=postgresql://user:pass@host:5432/dbname + +# ... OR use individual Postgres params (used only when DATABASE_URL is not set) POSTGRES_USER=cryptouser POSTGRES_PASSWORD=cryptopass POSTGRES_DB=cryptodb POSTGRES_HOST=db POSTGRES_PORT=5432 -REDIS_URL=redis://redis:6379/0 -SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + +# Optional: scheduling and symbols SCHEDULE_MINUTES=5 +SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + +# Optional external APIs (leave blank to use mock adapters) REDDIT_CLIENT_ID= REDDIT_CLIENT_SECRET= -REDDIT_USER_AGENT=crypto-risk-app/0.1 by you +REDDIT_USER_AGENT=crypto-risk-app/0.2 by you X_BEARER_TOKEN= CRYPTOPANIC_API_KEY= NEWSAPI_KEY= COINGLASS_API_KEY= + +# UI: You can suggest candidates for the API base URL (comma-separated) +API_CANDIDATES=https://crypto-risk-api.onrender.com,http://api:8000,http://localhost:8000 diff --git a/README.md b/README.md index c88e06ae..e3049688 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,23 @@ -# Crypto Risk Dashboard — Self-Hosted Starter +# Crypto Risk Dashboard — Self-Hosted Starter (v2, Timescale-ready) -Self-hosted, Dockerized stack with: -- **API** (FastAPI) -- **Worker** (scheduled ingest + signals) -- **DB** (Postgres; Timescale-ready) -- **UI** (Streamlit) with graphs, meters, hot signals -- **Adapters** for candles, funding, OI, sentiment, headlines (mock by default) +**What’s inside** +- API: FastAPI +- UI: Streamlit (with automatic API URL detect) +- Worker: scheduled ingest + signal engine +- DB: Postgres schema (Timescale-ready: auto-hypertables + 1h continuous aggregate if extension is available) +- Render Blueprint: `render.yaml` (API + UI + Worker + free Postgres) -## Quick start (local Docker) -1) `cp .env.example .env` +**Quick start (Docker Compose, local)** +1) Copy `.env.example` to `.env` and edit as needed. You can set `DATABASE_URL` (preferred) or individual POSTGRES_* vars. 2) `docker compose up --build` -3) UI: http://localhost:8501, API: http://localhost:8000 +3) UI: http://localhost:8501 | API: http://localhost:8000 -## Render deploy -Use the `render.yaml` blueprint (free Postgres + API + UI + worker). +**Render deployment** +- Push to GitHub, then in Render → Blueprints → New from repo. +- Set `API_CANDIDATES` on the UI service (defaults provided in render.yaml). The UI auto-detects API via /health. +- Set `DATABASE_URL` on API + Worker (Render sets it automatically if you bind the included Postgres; + or paste your Timescale Cloud connection string). + +**Security note** +Do NOT commit real secrets (API keys, DB URLs) to Git. +Use Render environment variables or a local `.env` that you do not push. diff --git a/docker-compose.yml b/docker-compose.yml index d2c3f71c..1a6db525 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,11 +15,6 @@ services: timeout: 5s retries: 20 - redis: - image: redis:7 - ports: - - "6379:6379" - api: build: ./services/api env_file: .env @@ -35,8 +30,6 @@ services: depends_on: db: condition: service_healthy - redis: - condition: service_started ui: build: ./services/ui diff --git a/services/api/main.py b/services/api/main.py index 6bf82a73..ce6a6500 100644 --- a/services/api/main.py +++ b/services/api/main.py @@ -1,10 +1,9 @@ -import os from fastapi import FastAPI, Query from services_common.db import fetch_df from services_common.signals import latest_signals_for_pairs, signal_explanations from services_common.config import load_config -app = FastAPI(title="Crypto Risk API", version="0.1.0") +app = FastAPI(title="Crypto Risk API", version="0.2.0") cfg = load_config() @app.get("/health") @@ -16,20 +15,28 @@ def pairs(): return {"pairs": cfg.symbols} @app.get("/signals") -def get_signals(pairs: str = Query(None)): - if pairs: - pairs_list = [p.strip() for p in pairs.split(",") if p.strip()] - else: - pairs_list = cfg.symbols +def get_signals(pairs: str | None = Query(None)): + pairs_list = [p.strip() for p in pairs.split(",")] if pairs else cfg.symbols data = latest_signals_for_pairs(pairs_list) return {"signals": data, "explanations": signal_explanations()} @app.get("/timeseries/{metric}") def timeseries(metric: str, pair: str, limit: int = 500): - table_map = {"candles":"candles","funding":"funding_rates","oi":"open_interest","vol":"volatility","sentiment":"sentiment"} - table = table_map.get(metric) - if not table: - return {"error":"unknown metric"} - q = f"""SELECT * FROM {table} WHERE pair=%(pair)s ORDER BY ts DESC LIMIT %(limit)s""" + table_map = { + "candles": "candles", + "funding": "funding_rates", + "oi": "open_interest", + "vol": "volatility", + "sentiment": "sentiment" + } + if metric not in table_map: + return {"error": "unknown metric"} + table = table_map[metric] + q = f""" + SELECT * FROM {table} + WHERE pair = %(pair)s + ORDER BY ts DESC + LIMIT %(limit)s + """ df = fetch_df(q, {"pair": pair, "limit": limit}) return {"columns": list(df.columns), "rows": df.to_dict(orient="records")} diff --git a/services/api/requirements.txt b/services/api/requirements.txt index 0941b5de..9d2300aa 100644 --- a/services/api/requirements.txt +++ b/services/api/requirements.txt @@ -17,3 +17,5 @@ textblob==0.18.0.post0 nltk==3.9.1 requests==2.32.3 beautifulsoup4==4.12.3 +streamlit==1.39.0 +streamlit_echarts==0.4.0 diff --git a/services/common/adapters/exchanges.py b/services/common/adapters/exchanges.py index bd9ffbf4..77e2b095 100644 --- a/services/common/adapters/exchanges.py +++ b/services/common/adapters/exchanges.py @@ -1,4 +1,5 @@ -import datetime as dt, ccxt +import time, pandas as pd, datetime as dt +import ccxt def fetch_candles(exchange_pair: str, timeframe="1h", limit=200): ex_name, pair = exchange_pair.split(":", 1) @@ -10,7 +11,7 @@ def fetch_candles(exchange_pair: str, timeframe="1h", limit=200): rows.append({"pair": exchange_pair, "ts": ts, "open": o, "high": h, "low": l, "close": c, "volume": v}) return rows -def mock_funding(exchange_pair: str, candles_df): - rate = (candles_df["close"].pct_change().fillna(0).tail(1).iloc[0]) / 10 if len(candles_df) else 0.0 - ts = candles_df["ts"].tail(1).iloc[0] if len(candles_df) else dt.datetime.now(dt.timezone.utc) +def mock_funding(exchange_pair: str, candles_df: pd.DataFrame): + rate = (candles_df["close"].pct_change().fillna(0).tail(1).iloc[0]) / 10 + ts = candles_df["ts"].tail(1).iloc[0] return [{"pair": exchange_pair, "ts": ts, "rate": float(rate)}] diff --git a/services/common/adapters/headlines.py b/services/common/adapters/headlines.py index 2a5d60d9..2e848133 100644 --- a/services/common/adapters/headlines.py +++ b/services/common/adapters/headlines.py @@ -2,7 +2,6 @@ def fetch_headlines_mock(): now = dt.datetime.now(dt.timezone.utc) return [{ - "ts": now, "source": "mock", - "title": "Market wobbles as OI surges; funding flips negative", - "url": "https://example.com", "keywords": ["open interest","funding"] + "ts": now, "source": "mock", "title": "Market wobbles as OI surges; funding flips negative", + "url": "https://example.com", "keywords": ["open interest", "funding"] }] diff --git a/services/common/adapters/open_interest.py b/services/common/adapters/open_interest.py index 04879d30..c7ef46c5 100644 --- a/services/common/adapters/open_interest.py +++ b/services/common/adapters/open_interest.py @@ -1,5 +1,4 @@ import datetime as dt, random - def fetch_open_interest(exchange_pair: str): now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) base = 1_000_000 + random.randint(-50_000, 50_000) diff --git a/services/common/adapters/sentiment.py b/services/common/adapters/sentiment.py index 49aa852d..fafd59cf 100644 --- a/services/common/adapters/sentiment.py +++ b/services/common/adapters/sentiment.py @@ -1,6 +1,5 @@ import datetime as dt, random -KEYWORDS = ["liquidation","margin call","rekt","funding","open interest"] - +KEYWORDS = ["liquidation", "margin call", "rekt", "funding", "open interest"] def fetch_sentiment_mock(exchange_pair: str): now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) mentions = random.randint(5, 50) diff --git a/services/common/adapters/volatility.py b/services/common/adapters/volatility.py index 26eacc7e..049148bd 100644 --- a/services/common/adapters/volatility.py +++ b/services/common/adapters/volatility.py @@ -1,10 +1,12 @@ -def compute_atr_like(candles_df, window=14): - if len(candles_df)==0: return None +import pandas as pd +def compute_atr_like(candles_df: pd.DataFrame, window=14): + if candles_df.empty: + return None df = candles_df.copy().sort_values("ts") hl = df["high"] - df["low"] hc = (df["high"] - df["close"].shift()).abs() lc = (df["low"] - df["close"].shift()).abs() - tr = (hl.to_frame("hl").join(hc.to_frame("hc")).join(lc.to_frame("lc"))).max(axis=1) + tr = pd.concat([hl, hc, lc], axis=1).max(axis=1) atr = tr.rolling(window).mean() last = df["ts"].iloc[-1] return [{"pair": df["pair"].iloc[-1], "ts": last, "atr": float(atr.iloc[-1])}] diff --git a/services/common/db.py b/services/common/db.py index 35aa44f9..545405b8 100644 --- a/services/common/db.py +++ b/services/common/db.py @@ -1,5 +1,8 @@ import os, psycopg2, pandas as pd from psycopg2.extras import execute_values +from services_common.config import load_config + +_cfg = load_config() def _conn(): url = os.getenv("DATABASE_URL") @@ -8,13 +11,7 @@ def _conn(): return psycopg2.connect( host=_cfg.pg_host, port=_cfg.pg_port, dbname=_cfg.pg_db, user=_cfg.pg_user, password=_cfg.pg_pass - # fallback to local docker compose - host = os.getenv("POSTGRES_HOST","db") - port = int(os.getenv("POSTGRES_PORT","5432")) - db = os.getenv("POSTGRES_DB","cryptodb") - user = os.getenv("POSTGRES_USER","cryptouser") - pwd = os.getenv("POSTGRES_PASSWORD","cryptopass") - return psycopg2.connect(host=host, port=port, dbname=db, user=user, password=pwd) + ) def execute(sql, params=None): with _conn() as conn, conn.cursor() as cur: @@ -25,7 +22,8 @@ def fetch_df(sql, params=None) -> pd.DataFrame: return pd.read_sql(sql, conn, params=params) def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_cols: list[str]): - if not rows: return + if not rows: + return cols = list(rows[0].keys()) vals = [[r[c] for c in cols] for r in rows] on_conflict = ", ".join(conflict_cols) @@ -41,11 +39,16 @@ def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_c def ensure_schema(): from services_common.schema import SCHEMA_SQL execute(SCHEMA_SQL) + _try_apply_timescale() + +def _try_apply_timescale(): + # Silently attempt to enable Timescale and create hypertables/CAGGs try: from pathlib import Path - ts_sql = (Path(__file__).parent / "schema_timescale.sql") - if ts_sql.exists(): + path = Path(__file__).parent / "schema_timescale.sql" + if path.exists(): + sql = path.read_text(encoding="utf-8") with _conn() as conn, conn.cursor() as cur: - cur.execute(ts_sql.read_text(encoding="utf-8")) + cur.execute(sql) except Exception: - pass + pass # extension may be unavailable; that's fine diff --git a/services/common/ingest.py b/services/common/ingest.py index 2c510370..c5dffffe 100644 --- a/services/common/ingest.py +++ b/services/common/ingest.py @@ -15,14 +15,18 @@ def run_ingest_cycle(): upsert_many("candles", candle_rows, ["pair","ts"], ["open","high","low","close","volume"]) df = pd.DataFrame(candle_rows) - fr = mock_funding(pair, df); upsert_many("funding_rates", fr, ["pair","ts"], ["rate"]) + fr = mock_funding(pair, df) + upsert_many("funding_rates", fr, ["pair","ts"], ["rate"]) - oi = fetch_open_interest(pair); upsert_many("open_interest", oi, ["pair","ts"], ["value_usd"]) + oi = fetch_open_interest(pair) + upsert_many("open_interest", oi, ["pair","ts"], ["value_usd"]) vol = compute_atr_like(df) - if vol: upsert_many("volatility", vol, ["pair","ts"], ["atr"]) + if vol: + upsert_many("volatility", vol, ["pair","ts"], ["atr"]) - sent = fetch_sentiment_mock(pair); upsert_many("sentiment", sent, ["pair","ts"], ["mentions","score_norm","keywords"]) + sent = fetch_sentiment_mock(pair) + upsert_many("sentiment", sent, ["pair","ts"], ["mentions","score_norm","keywords"]) h = fetch_headlines_mock() upsert_many("headlines", h, ["id"], []) diff --git a/services/common/schema_timescale.sql b/services/common/schema_timescale.sql index 15e97882..0783971f 100644 --- a/services/common/schema_timescale.sql +++ b/services/common/schema_timescale.sql @@ -1,4 +1,7 @@ +-- Enable Timescale if available CREATE EXTENSION IF NOT EXISTS timescaledb; + +-- Convert base tables to hypertables SELECT create_hypertable('candles', 'ts', if_not_exists => TRUE); SELECT create_hypertable('funding_rates', 'ts', if_not_exists => TRUE); SELECT create_hypertable('open_interest', 'ts', if_not_exists => TRUE); @@ -6,6 +9,7 @@ SELECT create_hypertable('volatility', 'ts', if_not_exists => TRUE); SELECT create_hypertable('sentiment', 'ts', if_not_exists => TRUE); SELECT create_hypertable('signals', 'ts', if_not_exists => TRUE); +-- Example continuous aggregate for faster OHLC queries CREATE MATERIALIZED VIEW IF NOT EXISTS candles_1h WITH (timescaledb.continuous) AS SELECT @@ -19,6 +23,7 @@ SELECT FROM candles GROUP BY 1,2; +-- Auto-refresh policy SELECT add_continuous_aggregate_policy( 'candles_1h', start_offset => INTERVAL '3 days', diff --git a/services/common/signals.py b/services/common/signals.py index fcc08387..c186dc28 100644 --- a/services/common/signals.py +++ b/services/common/signals.py @@ -1,67 +1,114 @@ import pandas as pd from services_common.db import fetch_df, upsert_many -from datetime import datetime, timezone def _percentile(series: pd.Series, value: float): - if series.empty: return None + if series.empty: + return None return (series < value).mean() * 100.0 def compute_market_stress(pair: str): - oi = fetch_df("""SELECT ts, value_usd FROM open_interest WHERE pair=%(pair)s AND ts > now() - interval '30 days' ORDER BY ts""", {"pair": pair}) - fr = fetch_df("""SELECT ts, rate FROM funding_rates WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts""", {"pair": pair}) - sent = fetch_df("""SELECT ts, mentions, score_norm, keywords FROM sentiment WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts""", {"pair": pair}) + oi = fetch_df(""" + SELECT ts, value_usd FROM open_interest + WHERE pair=%(pair)s AND ts > now() - interval '30 days' + ORDER BY ts + """, {"pair": pair}) + fr = fetch_df(""" + SELECT ts, rate FROM funding_rates + WHERE pair=%(pair)s AND ts > now() - interval '14 days' + ORDER BY ts + """, {"pair": pair}) + sent = fetch_df(""" + SELECT ts, mentions, score_norm, keywords FROM sentiment + WHERE pair=%(pair)s AND ts > now() - interval '14 days' + ORDER BY ts + """, {"pair": pair}) - regime, bias = "Unknown", "Neutral" - long_prob = short_prob = 0.5 + regime = "Unknown" + bias = "Neutral" + long_prob = 0.5 + short_prob = 0.5 summary = "Insufficient data." if not oi.empty and not fr.empty and not sent.empty: latest_oi = oi["value_usd"].iloc[-1] oi_pct = _percentile(oi["value_usd"], latest_oi) latest_funding = fr["rate"].iloc[-1] - sent["liq_kw"] = sent["keywords"].apply(lambda d: (d.get("liquidation",0) if isinstance(d, dict) else 0) + (d.get("margin call",0) if isinstance(d, dict) else 0)) last_week = sent.tail(max(1, len(sent)//2)) first_week = sent.head(len(sent)-len(last_week)) if len(sent)>1 else sent.head(1) base = max(1, first_week["liq_kw"].sum()) spike_ratio = (last_week["liq_kw"].sum()) / base - if (oi_pct is not None) and oi_pct >= 90 and latest_funding < 0 and spike_ratio >= 2.0: - regime = "Risky / High Liquidation Risk"; bias = "Short"; long_prob, short_prob = 0.25, 0.75 + if oi_pct is not None and oi_pct >= 90 and latest_funding < 0 and spike_ratio >= 2.0: + regime = "Risky / High Liquidation Risk" + bias = "Short" + long_prob = 0.25 + short_prob = 0.75 summary = "OI in 90th pct+, funding negative, and liquidation mentions up ≥200%. Consider caution or short bias." else: - candles = fetch_df("""SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 50""", {"pair": pair}).sort_values("ts") - slope = candles["close"].pct_change().fillna(0).tail(10).mean() if not candles.empty else 0.0 + candles = fetch_df(""" + SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 50 + """, {"pair": pair}).sort_values("ts") + slope = 0.0 + if not candles.empty: + y = candles["close"].pct_change().fillna(0).tail(10) + slope = y.mean() if slope > 0 and latest_funding >= 0: - regime = "Constructive"; bias = "Long"; long_prob, short_prob = 0.65, 0.35 - summary = "Upward momentum + non-negative funding → modest long bias." + regime = "Constructive" + bias = "Long" + long_prob = 0.65 + short_prob = 0.35 + summary = "Upward momentum with non-negative funding suggests a modest long bias." elif slope < 0 and latest_funding <= 0: - regime = "Weak"; bias = "Short"; long_prob, short_prob = 0.35, 0.65 - summary = "Downward momentum + non-positive funding → modest short bias." + regime = "Weak" + bias = "Short" + long_prob = 0.35 + short_prob = 0.65 + summary = "Downward momentum with non-positive funding suggests a modest short bias." else: - regime = "Balanced / Choppy"; bias = "Flat"; long_prob = short_prob = 0.5 - summary = "Mixed signals; consider mean-reversion or wait." - return {"pair": pair, "regime": regime, "bias": bias, "long_prob": float(long_prob), "short_prob": float(short_prob), "summary": summary} + regime = "Balanced / Choppy" + bias = "Flat" + long_prob = 0.5 + short_prob = 0.5 + summary = "Mixed signals; prefer mean-reversion or wait for clarity." + + return { + "pair": pair, + "regime": regime, + "bias": bias, + "long_prob": float(long_prob), + "short_prob": float(short_prob), + "summary": summary + } def compute_all_signals(): - dfpairs = fetch_df("SELECT DISTINCT pair FROM candles") + pairs_df = fetch_df("SELECT DISTINCT pair FROM candles") + pairs = list(pairs_df["pair"]) if "pair" in pairs_df else [] out = [] - for p in dfpairs.get("pair", []): - s = compute_market_stress(p); out.append(s) + for p in pairs: + s = compute_market_stress(p) + out.append(s) + from datetime import datetime, timezone now = datetime.now(timezone.utc) rows = [{ "ts": now, "pair": s["pair"], "regime": s["regime"], "bias": s["bias"], "long_prob": s["long_prob"], "short_prob": s["short_prob"], "summary": s["summary"] } for s in out] - if rows: upsert_many("signals", rows, ["id"], []) + if rows: + upsert_many("signals", rows, ["id"], []) def latest_signals_for_pairs(pairs: list[str]): placeholders = ",".join(["%s"]*len(pairs)) - q = f"""SELECT DISTINCT ON (pair) pair, ts, regime, bias, long_prob, short_prob, summary - FROM signals WHERE pair IN ({placeholders}) ORDER BY pair, ts DESC""" + q = f""" + SELECT DISTINCT ON (pair) pair, ts, regime, bias, long_prob, short_prob, summary + FROM signals + WHERE pair IN ({placeholders}) + ORDER BY pair, ts DESC + """ df = fetch_df(q, pairs) result = {} - for _, r in df.iterrows(): result[r["pair"]] = dict(r) + for _, r in df.iterrows(): + result[r["pair"]] = dict(r) return result def signal_explanations(): diff --git a/services/ui/app.py b/services/ui/app.py index 54929773..bbc58b2c 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -1,11 +1,9 @@ -import os, requests, pandas as pd, time +import os, requests, pandas as pd import streamlit as st from urllib.parse import urlencode DEFAULT_CANDIDATES = [ - # 1) Render default if you kept the blueprint names: "https://crypto-risk-api.onrender.com", - # 2) Local docker compose: "http://api:8000", "http://localhost:8000", ] @@ -18,27 +16,19 @@ def probe_api(base: str, timeout=2.0) -> bool: return False def resolve_api_base(): - # Priority 1: explicit env var env_base = os.getenv("API_BASE", "").strip() if env_base and probe_api(env_base): return env_base - - # Priority 2: comma-separated list of candidates from env env_candidates = os.getenv("API_CANDIDATES", "") candidates = [c.strip() for c in env_candidates.split(",") if c.strip()] if env_candidates else [] - # Merge with defaults (preserve order; avoid dups) - seen = set() - merged = [] + seen, merged = set(), [] for x in (candidates + DEFAULT_CANDIDATES): if x not in seen: - merged.append(x) - seen.add(x) - + merged.append(x); seen.add(x) for base in merged: if probe_api(base): return base - - return None # nothing worked + return None API_BASE = resolve_api_base() @@ -49,17 +39,12 @@ def resolve_api_base(): if not API_BASE: st.error("Could not locate the API automatically.") manual = st.text_input("Enter your API base URL (e.g., https://crypto-risk-api.onrender.com)") - if manual: - if probe_api(manual): - API_BASE = manual - st.success("Connected!") - else: - st.warning("That URL didn’t respond at /health. Double-check and try again.") - + if manual and probe_api(manual): + API_BASE = manual + st.success("Connected!") if not API_BASE: st.stop() -# Pairs resp = requests.get(f"{API_BASE}/pairs", timeout=30).json() pairs = resp.get("pairs", ["binance:BTC/USDT"]) @@ -75,9 +60,7 @@ def fetch(metric, pair, limit=500): return r.json() def get_signals(pairs=None): - qs = "" - if pairs: - qs = "?pairs=" + ",".join(pairs) + qs = "?pairs=" + ",".join(pairs) if pairs else "" r = requests.get(f"{API_BASE}/signals{qs}", timeout=30) return r.json() @@ -89,14 +72,10 @@ def get_signals(pairs=None): if signals.get(pair): s = signals[pair] c1, c2, c3, c4 = st.columns(4) - with c1: - st.metric("Market Regime", s.get("regime","—")) - with c2: - st.metric("Bias", s.get("bias","—")) - with c3: - st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) - with c4: - st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) + with c1: st.metric("Market Regime", s.get("regime","—")) + with c2: st.metric("Bias", s.get("bias","—")) + with c3: st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) + with c4: st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) st.info(s.get("summary","")) else: st.warning("No signals yet. The worker may still be seeding data.") @@ -152,4 +131,4 @@ def get_signals(pairs=None): st.write("No data yet.") st.divider() -st.caption("Tip: set API_BASE or API_CANDIDATES env vars to skip auto-detect.") +st.caption("Configure pairs & scheduler in `.env`. Add API keys for live data.") diff --git a/services/ui/requirements.txt b/services/ui/requirements.txt index 3f258ee0..9d2300aa 100644 --- a/services/ui/requirements.txt +++ b/services/ui/requirements.txt @@ -17,6 +17,5 @@ textblob==0.18.0.post0 nltk==3.9.1 requests==2.32.3 beautifulsoup4==4.12.3 - streamlit==1.39.0 streamlit_echarts==0.4.0 diff --git a/services/worker/requirements.txt b/services/worker/requirements.txt index 0941b5de..9d2300aa 100644 --- a/services/worker/requirements.txt +++ b/services/worker/requirements.txt @@ -17,3 +17,5 @@ textblob==0.18.0.post0 nltk==3.9.1 requests==2.32.3 beautifulsoup4==4.12.3 +streamlit==1.39.0 +streamlit_echarts==0.4.0 diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 47c22a47..120c2c72 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -2,15 +2,19 @@ from services_common.db import ensure_schema from services_common.ingest import run_ingest_cycle from services_common.signals import compute_all_signals +from services_common.config import load_config def main(): ensure_schema() - interval = int(os.getenv("SCHEDULE_MINUTES","5")) - print(f"[worker] schedule {interval}m") + interval = int(os.getenv("SCHEDULE_MINUTES", "5")) + print(f"[worker] schedule every {interval} minutes.") while True: - print("[worker] ingest..."); run_ingest_cycle() - print("[worker] signals..."); compute_all_signals() - print("[worker] sleep..."); time.sleep(interval*60) + print("[worker] ingest cycle...") + run_ingest_cycle() + print("[worker] compute signals...") + compute_all_signals() + print("[worker] sleep...") + time.sleep(60 * interval) if __name__ == "__main__": main() diff --git a/timescale.env b/timescale.env new file mode 100644 index 00000000..5683c363 --- /dev/null +++ b/timescale.env @@ -0,0 +1,5 @@ +# For local/testing only. Keep this PRIVATE and out of git. +DATABASE_URL=postgres://tsdbadmin:hl7bjpkhq8ziestj@glmv89ewih.enc0lseujb.tsdb.cloud.timescale.com:34312/tsdb?sslmode=require +SCHEDULE_MINUTES=5 +SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT +API_CANDIDATES=http://localhost:8000 diff --git a/upload_to_existing_repo.py b/upload_to_existing_repo.py deleted file mode 100644 index a067c115..00000000 --- a/upload_to_existing_repo.py +++ /dev/null @@ -1,545 +0,0 @@ -#!/usr/bin/env python3 -import os, sys, json, textwrap, base64, shutil -from pathlib import Path -import urllib.request, urllib.parse - -def gh_request(method, url, token, data=None, headers=None): - hdr = { - "Authorization": f"token {token}", - "Accept": "application/vnd.github+json", - "User-Agent": "crypto-risk-dashboard-uploader" - } - if headers: hdr.update(headers) - req = urllib.request.Request(url, data=data, headers=hdr, method=method) - try: - with urllib.request.urlopen(req) as resp: - return resp.getcode(), resp.read() - except urllib.error.HTTPError as e: - body = e.read().decode("utf-8","ignore") - raise SystemExit(f"HTTP {e.code} Error at {url}\n{body}") - -def gh_put_file(user, repo, token, path, content_bytes, message, branch="main"): - url = f"https://api.github.com/repos/{user}/{repo}/contents/{urllib.parse.quote(path)}" - payload = { - "message": message, - "content": base64.b64encode(content_bytes).decode("utf-8"), - "branch": branch - } - code, body = gh_request("PUT", url, token, data=json.dumps(payload).encode("utf-8")) - return json.loads(body.decode("utf-8")) - -print("=== Upload crypto-risk-dashboard to an EXISTING GitHub repo ===") -gh_user = input("GitHub username: ").strip() -token = input("GitHub Personal Access Token (classic, scope: repo; SSO authorized if needed): ").strip() -repo = input("Existing repo name (exact): ").strip() -print("NOTE: The repo must already exist and have a README so branch 'main' exists.") - -root = Path.cwd() / "crypto-risk-dashboard-starter" -if root.exists(): shutil.rmtree(root) -root.mkdir(parents=True, exist_ok=True) - -def w(relpath: str, content: str): - p = root / relpath - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(textwrap.dedent(content).lstrip("\n"), encoding="utf-8") - -# ------- Minimal starter (same structure you need) ------- -w("README.md", """ -# Crypto Risk Dashboard — Self-Hosted Starter -Dockerized stack: FastAPI API, worker (ingest+signals), Postgres (Timescale-ready), Streamlit UI. -Local: `cp .env.example .env` → `docker compose up --build` -Render deploy: use `render.yaml` (free Postgres + API + UI + worker). -""") - -w(".env.example", """ -POSTGRES_USER=cryptouser -POSTGRES_PASSWORD=cryptopass -POSTGRES_DB=cryptodb -POSTGRES_HOST=db -POSTGRES_PORT=5432 -REDIS_URL=redis://redis:6379/0 -SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT -SCHEDULE_MINUTES=5 -REDDIT_CLIENT_ID= -REDDIT_CLIENT_SECRET= -REDDIT_USER_AGENT=crypto-risk-app/0.1 by you -X_BEARER_TOKEN= -CRYPTOPANIC_API_KEY= -NEWSAPI_KEY= -COINGLASS_API_KEY= -""") - -w("docker-compose.yml", """ -services: - db: - image: postgres:16 - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - volumes: [ "db_data:/var/lib/postgresql/data" ] - ports: [ "5432:5432" ] - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] - interval: 5s - timeout: 5s - retries: 20 - redis: - image: redis:7 - ports: [ "6379:6379" ] - api: - build: ./services/api - env_file: .env - depends_on: { db: { condition: service_healthy } } - ports: [ "8000:8000" ] - worker: - build: ./services/worker - env_file: .env - depends_on: - db: { condition: service_healthy } - redis: { condition: service_started } - ui: - build: ./services/ui - env_file: .env - depends_on: { api: { condition: service_started } } - ports: [ "8501:8501" ] -volumes: { db_data: {} } -""") - -w("render.yaml", """ -databases: - - name: cryptodb - databaseName: cryptodb - plan: free -services: - - type: web - name: crypto-risk-api - runtime: docker - rootDir: services/api - plan: free - envVars: - - key: DATABASE_URL - fromDatabase: { name: cryptodb, property: connectionString } - - key: SYMBOLS - value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT - - key: SCHEDULE_MINUTES - value: "5" - - type: web - name: crypto-risk-ui - runtime: docker - rootDir: services/ui - plan: free - envVars: - - key: API_CANDIDATES - value: https://crypto-risk-api.onrender.com,http://api:8000,http://localhost:8000 - - type: worker - name: crypto-risk-worker - runtime: docker - rootDir: services/worker - plan: free - envVars: - - key: DATABASE_URL - fromDatabase: { name: cryptodb, property: connectionString } - - key: SYMBOLS - value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT - - key: SCHEDULE_MINUTES - value: "5" -""") - -common_reqs = """ -fastapi==0.115.2 -uvicorn[standard]==0.30.6 -pydantic==2.9.2 -python-dotenv==1.0.1 -psycopg2-binary==2.9.9 -SQLAlchemy==2.0.36 -alembic==1.13.2 -redis==5.0.8 -celery==5.4.0 -pandas==2.2.3 -numpy==2.1.2 -plotly==5.24.1 -matplotlib==3.9.2 -ccxt==4.4.6 -scikit-learn==1.5.2 -textblob==0.18.0.post0 -nltk==3.9.1 -requests==2.32.3 -beautifulsoup4==4.12.3 -""" - -# API -(api := root / "services/api"); api.mkdir(parents=True, exist_ok=True) -(api / "Dockerfile").write_text("""FROM python:3.11-slim -WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -CMD ["uvicorn","main:app","--host","0.0.0.0","--port","8000"]""", encoding="utf-8") -(api / "requirements.txt").write_text(common_reqs, encoding="utf-8") -(api / "main.py").write_text(""" -import os -from fastapi import FastAPI, Query -from services_common.db import fetch_df -from services_common.signals import latest_signals_for_pairs, signal_explanations -from services_common.config import load_config - -app = FastAPI(title="Crypto Risk API", version="0.1.0") -cfg = load_config() - -@app.get("/health") -def health(): return {"ok": True} - -@app.get("/pairs") -def pairs(): return {"pairs": cfg.symbols} - -@app.get("/signals") -def get_signals(pairs: str = Query(None)): - pairs_list = [p.strip() for p in pairs.split(",")] if pairs else cfg.symbols - data = latest_signals_for_pairs(pairs_list) - return {"signals": data, "explanations": signal_explanations()} - -@app.get("/timeseries/{metric}") -def timeseries(metric: str, pair: str, limit: int = 500): - table_map = {"candles":"candles","funding":"funding_rates","oi":"open_interest","vol":"volatility","sentiment":"sentiment"} - table = table_map.get(metric) - if not table: return {"error":"unknown metric"} - q = f"SELECT * FROM {table} WHERE pair=%(pair)s ORDER BY ts DESC LIMIT %(limit)s" - df = fetch_df(q, {"pair": pair, "limit": limit}) - return {"columns": list(df.columns), "rows": df.to_dict(orient='records')} -""", encoding="utf-8") - -# Worker -(worker := root / "services/worker"); worker.mkdir(parents=True, exist_ok=True) -(worker / "Dockerfile").write_text("""FROM python:3.11-slim -WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -CMD ["python","run_worker.py"]""", encoding="utf-8") -(worker / "requirements.txt").write_text(common_reqs, encoding="utf-8") -(worker / "run_worker.py").write_text(""" -import os, time -from services_common.db import ensure_schema -from services_common.ingest import run_ingest_cycle -from services_common.signals import compute_all_signals -def main(): - ensure_schema() - interval = int(os.getenv("SCHEDULE_MINUTES","5")) - print(f"[worker] schedule {interval}m") - while True: - print("[worker] ingest..."); run_ingest_cycle() - print("[worker] signals..."); compute_all_signals() - print("[worker] sleep..."); time.sleep(interval*60) -if __name__=="__main__": main() -""", encoding="utf-8") - -# UI -(ui := root / "services/ui"); ui.mkdir(parents=True, exist_ok=True) -(ui / "Dockerfile").write_text("""FROM python:3.11-slim -WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -CMD ["streamlit","run","app.py","--server.port=8501","--server.address=0.0.0.0"]""", encoding="utf-8") -(ui / "requirements.txt").write_text(common_reqs + "\nstreamlit==1.39.0\nstreamlit_echarts==0.4.0\n", encoding="utf-8") -(ui / "app.py").write_text(""" -import os, requests, pandas as pd -import streamlit as st -from urllib.parse import urlencode - -DEFAULT_CANDIDATES = [ - "https://crypto-risk-api.onrender.com", - "http://api:8000", - "http://localhost:8000", -] - -def probe_api(base: str, timeout=2.0) -> bool: - try: - r = requests.get(f"{base}/health", timeout=timeout) - return r.ok and r.json().get("ok") is True - except Exception: return False - -def resolve_api_base(): - env_base = os.getenv("API_BASE","").strip() - if env_base and probe_api(env_base): return env_base - env_candidates = os.getenv("API_CANDIDATES","") - candidates = [c.strip() for c in env_candidates.split(",") if c.strip()] if env_candidates else [] - merged, seen = [], set() - for x in (candidates + DEFAULT_CANDIDATES): - if x not in seen: merged.append(x); seen.add(x) - for base in merged: - if probe_api(base): return base - return None - -API_BASE = resolve_api_base() - -st.set_page_config(page_title="Crypto Risk Dashboard", layout="wide") -st.title("🧭 Crypto Risk Dashboard") -st.caption("Self-hosted. Graphs • Meters • Hot Signals") - -if not API_BASE: - st.error("Could not locate the API automatically.") - manual = st.text_input("Enter your API base URL") - if manual and probe_api(manual): - API_BASE = manual; st.success("Connected!") -if not API_BASE: st.stop() - -resp = requests.get(f"{API_BASE}/pairs", timeout=30).json() -pairs = resp.get("pairs", ["binance:BTC/USDT"]) - -col1, col2 = st.columns([2,1]) -with col1: pair = st.selectbox("Pair", pairs, index=0) -with col2: refresh = st.button("Refresh") - -def fetch(metric, pair, limit=500): - qs = urlencode({"pair": pair, "limit": limit}) - return requests.get(f"{API_BASE}/timeseries/{metric}?{qs}", timeout=30).json() - -def get_signals(pairs=None): - qs = "?pairs=" + ",".join(pairs) if pairs else "" - return requests.get(f"{API_BASE}/signals{qs}", timeout=30).json() - -sig = get_signals([pair]); signals = sig.get("signals", {}) - -st.subheader("Hot Signals") -if signals.get(pair): - s = signals[pair] - c1,c2,c3,c4 = st.columns(4) - with c1: st.metric("Market Regime", s.get("regime","—")) - with c2: st.metric("Bias", s.get("bias","—")) - with c3: st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) - with c4: st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) - st.info(s.get("summary","")) -else: st.warning("No signals yet. The worker may still be seeding data.") -st.divider(); st.subheader("Charts") -tabs = st.tabs(["Candles","Funding","Open Interest","Volatility","Sentiment"]) -with tabs[0]: - rows = fetch("candles", pair, 300).get("rows", []) - if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["close"]) - else: st.write("No data yet.") -with tabs[1]: - rows = fetch("funding", pair, 500).get("rows", []) - if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["rate"]) - else: st.write("No data yet.") -with tabs[2]: - rows = fetch("oi", pair, 500).get("rows", []) - if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["value_usd"]) - else: st.write("No data yet.") -with tabs[3]: - rows = fetch("vol", pair, 500).get("rows", []) - if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["atr"]) - else: st.write("No data yet.") -with tabs[4]: - rows = fetch("sentiment", pair, 200).get("rows", []) - if rows: st.line_chart(pd.DataFrame(rows).sort_values("ts"), x="ts", y=["score_norm"]) - else: st.write("No data yet.") -st.divider(); st.caption("Set API_BASE or API_CANDIDATES to skip auto-detect.") -""", encoding="utf-8") - -# Shared lib -(common := root / "services/common"); common.mkdir(parents=True, exist_ok=True) -(common / "__init__.py").write_text("", encoding="utf-8") -(common / "config.py").write_text(""" -import os -from dataclasses import dataclass -@dataclass -class Config: - pg_user: str; pg_pass: str; pg_db: str; pg_host: str; pg_port: int; redis_url: str; symbols: list[str] -def load_config() -> Config: - symbols = [s.strip() for s in os.getenv("SYMBOLS","binance:BTC/USDT").split(",") if s.strip()] - return Config( - pg_user=os.getenv("POSTGRES_USER","cryptouser"), - pg_pass=os.getenv("POSTGRES_PASSWORD","cryptopass"), - pg_db=os.getenv("POSTGRES_DB","cryptodb"), - pg_host=os.getenv("POSTGRES_HOST","db"), - pg_port=int(os.getenv("POSTGRES_PORT","5432")), - redis_url=os.getenv("REDIS_URL","redis://redis:6379/0"), - symbols=symbols - ) -""", encoding="utf-8") -(common / "db.py").write_text(""" -import os, psycopg2, pandas as pd -from psycopg2.extras import execute_values -def _conn(): - url = os.getenv("DATABASE_URL") - if url: return psycopg2.connect(url) - return psycopg2.connect( - host=os.getenv("POSTGRES_HOST","db"), - port=int(os.getenv("POSTGRES_PORT","5432")), - dbname=os.getenv("POSTGRES_DB","cryptodb"), - user=os.getenv("POSTGRES_USER","cryptouser"), - password=os.getenv("POSTGRES_PASSWORD","cryptopass") - ) -def execute(sql, params=None): - with _conn() as conn, conn.cursor() as cur: cur.execute(sql, params or {}) -def fetch_df(sql, params=None)->pd.DataFrame: - with _conn() as conn: return pd.read_sql(sql, conn, params=params) -def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_cols: list[str]): - if not rows: return - cols=list(rows[0].keys()); vals=[[r[c] for c in cols] for r in rows] - on_conflict=", ".join(conflict_cols); updates=", ".join([f"{c}=EXCLUDED.{c}" for c in update_cols]) - sql=f"INSERT INTO {table} ({','.join(cols)}) VALUES %s ON CONFLICT ({on_conflict}) DO UPDATE SET {updates}" - with _conn() as conn, conn.cursor() as cur: execute_values(cur, sql, vals) -def ensure_schema(): - from services_common.schema import SCHEMA_SQL - execute(SCHEMA_SQL) -""", encoding="utf-8") -(common / "schema.py").write_text(""" -SCHEMA_SQL = ''' -CREATE TABLE IF NOT EXISTS candles (pair text NOT NULL, ts timestamptz NOT NULL, - open double precision, high double precision, low double precision, close double precision, volume double precision, - PRIMARY KEY (pair, ts)); -CREATE TABLE IF NOT EXISTS funding_rates (pair text NOT NULL, ts timestamptz NOT NULL, rate double precision, PRIMARY KEY (pair, ts)); -CREATE TABLE IF NOT EXISTS open_interest (pair text NOT NULL, ts timestamptz NOT NULL, value_usd double precision, PRIMARY KEY (pair, ts)); -CREATE TABLE IF NOT EXISTS volatility (pair text NOT NULL, ts timestamptz NOT NULL, atr double precision, PRIMARY KEY (pair, ts)); -CREATE TABLE IF NOT EXISTS sentiment (pair text NOT NULL, ts timestamptz NOT NULL, mentions integer, score_norm double precision, keywords jsonb, PRIMARY KEY (pair, ts)); -CREATE TABLE IF NOT EXISTS headlines (id bigserial PRIMARY KEY, ts timestamptz NOT NULL, source text, title text, url text, keywords jsonb); -CREATE TABLE IF NOT EXISTS signals (id bigserial PRIMARY KEY, ts timestamptz NOT NULL, pair text NOT NULL, regime text, bias text, long_prob double precision, short_prob double precision, summary text); -CREATE TABLE IF NOT EXISTS kv_store (k text PRIMARY KEY, v jsonb, updated_at timestamptz DEFAULT now()); -'; -""", encoding="utf-8") - -# Adapters & ingest & signals -(ad := common / "adapters"); ad.mkdir(parents=True, exist_ok=True) -(ad / "__init__.py").write_text("", encoding="utf-8") -(ad / "exchanges.py").write_text(""" -import datetime as dt, ccxt -def fetch_candles(exchange_pair: str, timeframe="1h", limit=200): - ex_name, pair = exchange_pair.split(":", 1) - ex = getattr(ccxt, ex_name)() - ohlcv = ex.fetch_ohlcv(pair, timeframe=timeframe, limit=limit) - rows=[] - for t,o,h,l,c,v in ohlcv: - ts=dt.datetime.utcfromtimestamp(t/1000).replace(tzinfo=dt.timezone.utc) - rows.append({"pair": exchange_pair,"ts": ts,"open": o,"high": h,"low": l,"close": c,"volume": v}) - return rows -def mock_funding(exchange_pair: str, candles_df): - rate=(candles_df["close"].pct_change().fillna(0).tail(1).iloc[0])/10 if len(candles_df) else 0.0 - ts=candles_df["ts"].tail(1).iloc[0] if len(candles_df) else dt.datetime.now(dt.timezone.utc) - return [{"pair": exchange_pair, "ts": ts, "rate": float(rate)}] -""", encoding="utf-8") -(ad / "open_interest.py").write_text(""" -import datetime as dt, random -def fetch_open_interest(exchange_pair: str): - now=dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) - base=1_000_000 + random.randint(-50_000, 50_000) - return [{"pair": exchange_pair, "ts": now, "value_usd": float(base)}] -""", encoding="utf-8") -(ad / "volatility.py").write_text(""" -def compute_atr_like(candles_df, window=14): - if len(candles_df)==0: return None - df=candles_df.copy().sort_values("ts") - hl=df["high"]-df["low"] - hc=(df["high"]-df["close"].shift()).abs() - lc=(df["low"]-df["close"].shift()).abs() - tr=(hl.to_frame("hl").join(hc.to_frame("hc")).join(lc.to_frame("lc"))).max(axis=1) - atr=tr.rolling(window).mean() - last=df["ts"].iloc[-1] - return [{"pair": df["pair"].iloc[-1], "ts": last, "atr": float(atr.iloc[-1])}] -""", encoding="utf-8") -(ad / "sentiment.py").write_text(""" -import datetime as dt, random -KEYWORDS=["liquidation","margin call","rekt","funding","open interest"] -def fetch_sentiment_mock(exchange_pair: str): - now=dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) - mentions=random.randint(5,50); score=random.uniform(-1,1) - kw_counts={k: random.randint(0, mentions//2) for k in KEYWORDS} - return [{"pair": exchange_pair, "ts": now, "mentions": mentions, "score_norm": score, "keywords": kw_counts}] -""", encoding="utf-8") -(ad / "headlines.py").write_text(""" -import datetime as dt -def fetch_headlines_mock(): - now=dt.datetime.now(dt.timezone.utc) - return [{"ts": now,"source":"mock","title":"Market wobbles as OI surges; funding flips negative","url":"https://example.com","keywords":["open interest","funding"]}] -""", encoding="utf-8") - -(common / "ingest.py").write_text(""" -import pandas as pd -from services_common.config import load_config -from services_common.db import upsert_many -from services_common.adapters.exchanges import fetch_candles, mock_funding -from services_common.adapters.open_interest import fetch_open_interest -from services_common.adapters.volatility import compute_atr_like -from services_common.adapters.sentiment import fetch_sentiment_mock -from services_common.adapters.headlines import fetch_headlines_mock -cfg = load_config() -def run_ingest_cycle(): - for pair in cfg.symbols: - candle_rows = fetch_candles(pair, timeframe="1h", limit=200) - upsert_many("candles", candle_rows, ["pair","ts"], ["open","high","low","close","volume"]) - df = pd.DataFrame(candle_rows) - upsert_many("funding_rates", mock_funding(pair, df), ["pair","ts"], ["rate"]) - upsert_many("open_interest", fetch_open_interest(pair), ["pair","ts"], ["value_usd"]) - vol = compute_atr_like(df) - if vol: upsert_many("volatility", vol, ["pair","ts"], ["atr"]) - upsert_many("sentiment", fetch_sentiment_mock(pair), ["pair","ts"], ["mentions","score_norm","keywords"]) - upsert_many("headlines", fetch_headlines_mock(), ["id"], []) -""", encoding="utf-8") - -(common / "signals.py").write_text(""" -import pandas as pd -from services_common.db import fetch_df, upsert_many -from datetime import datetime, timezone -def _percentile(series: pd.Series, value: float): - if series.empty: return None - return (series < value).mean() * 100.0 -def compute_market_stress(pair: str): - oi = fetch_df("SELECT ts, value_usd FROM open_interest WHERE pair=%(pair)s AND ts > now() - interval '30 days' ORDER BY ts", {"pair": pair}) - fr = fetch_df("SELECT ts, rate FROM funding_rates WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts", {"pair": pair}) - sent = fetch_df("SELECT ts, mentions, score_norm, keywords FROM sentiment WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts", {"pair": pair}) - regime, bias = "Unknown", "Neutral"; long_prob = short_prob = 0.5; summary = "Insufficient data." - if not oi.empty and not fr.empty and not sent.empty: - latest_oi = oi["value_usd"].iloc[-1]; oi_pct = _percentile(oi["value_usd"], latest_oi); latest_funding = fr["rate"].iloc[-1] - sent["liq_kw"] = sent["keywords"].apply(lambda d: (d.get("liquidation",0) if isinstance(d, dict) else 0) + (d.get("margin call",0) if isinstance(d, dict) else 0)) - last_week = sent.tail(max(1, len(sent)//2)); first_week = sent.head(len(sent)-len(last_week)) if len(sent)>1 else sent.head(1) - base = max(1, first_week["liq_kw"].sum()); spike_ratio = (last_week["liq_kw"].sum()) / base - if (oi_pct is not None) and oi_pct >= 90 and latest_funding < 0 and spike_ratio >= 2.0: - regime, bias, long_prob, short_prob = "Risky / High Liquidation Risk","Short",0.25,0.75 - summary = "OI in 90th pct+, funding negative, and liquidation mentions up ≥200%. Consider caution or short bias." - else: - candles = fetch_df("SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 50", {"pair": pair}).sort_values("ts") - slope = candles["close"].pct_change().fillna(0).tail(10).mean() if not candles.empty else 0.0 - if slope > 0 and latest_funding >= 0: - regime, bias, long_prob, short_prob = "Constructive","Long",0.65,0.35 - summary = "Upward momentum + non-negative funding → modest long bias." - elif slope < 0 and latest_funding <= 0: - regime, bias, long_prob, short_prob = "Weak","Short",0.35,0.65 - summary = "Downward momentum + non-positive funding → modest short bias." - else: - regime, bias, long_prob, short_prob = "Balanced / Choppy","Flat",0.5,0.5 - summary = "Mixed signals; consider mean-reversion or wait." - return {"pair": pair, "regime": regime, "bias": bias, "long_prob": float(long_prob), "short_prob": float(short_prob), "summary": summary} -def compute_all_signals(): - dfpairs = fetch_df("SELECT DISTINCT pair FROM candles"); out=[] - for p in dfpairs.get("pair", []): out.append(compute_market_stress(p)) - now = datetime.now(timezone.utc) - rows = [{"ts": now,"pair": s["pair"],"regime": s["regime"],"bias": s["bias"],"long_prob": s["long_prob"],"short_prob": s["short_prob"],"summary": s["summary"]} for s in out] - if rows: upsert_many("signals", rows, ["id"], []) -def latest_signals_for_pairs(pairs: list[str]): - placeholders = ",".join(["%s"]*len(pairs)) - q = f"SELECT DISTINCT ON (pair) pair, ts, regime, bias, long_prob, short_prob, summary FROM signals WHERE pair IN ({placeholders}) ORDER BY pair, ts DESC" - df = fetch_df(q, pairs); result={} - for _, r in df.iterrows(): result[r["pair"]] = dict(r) - return result -def signal_explanations(): - return {"market_stress": "OI ≥ 90th pct + negative funding + liquidation keyword spike ≥ 200% → bearish risk."} -""", encoding="utf-8") - -# Make services_common importable in Docker builds -(api / "services_common").mkdir(exist_ok=True) -(worker / "services_common").mkdir(exist_ok=True) -(api / "services_common/__init__.py").write_text("from pathlib import Path as _P; import sys as _s; _s.path.append(str(_P(__file__).resolve().parents[2]))\n", encoding="utf-8") -(worker / "services_common/__init__.py").write_text("from pathlib import Path as _P; import sys as _s; _s.path.append(str(_P(__file__).resolve().parents[2]))\n", encoding="utf-8") - -# Upload all files -print("Uploading files to GitHub (branch 'main')...") -for p in root.rglob("*"): - if p.is_dir(): continue - rel = str(p.relative_to(root)).replace("\\", "/") - content = p.read_bytes() - gh_put_file(gh_user, repo, token, rel, content, f"Add {rel}", branch="main") - -print("\n✅ Done!") -print(f"GitHub repository: https://github.com/{gh_user}/{repo}") -print("Next: Render → Blueprints → New Blueprint → pick this repo → Create.") From 0b2e69e2e52dd7d0f717aa98f1e2edab394001b2 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 16:40:09 +0100 Subject: [PATCH 04/43] v3 --- .github/workflows/crypto-worker.yml | 26 ++++++ README.md | 38 ++++---- render.yaml | 15 +--- services/common/adapters/exchanges.py | 59 ++++++++++--- services/common/adapters/headlines.py | 65 +++++++++++++- services/common/adapters/open_interest.py | 101 +++++++++++++++++++++- services/common/adapters/sentiment.py | 89 ++++++++++++++++++- services/common/ingest.py | 57 ++++++++---- services/worker/requirements.txt | 5 ++ services/worker/run_worker.py | 24 ++--- 10 files changed, 400 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/crypto-worker.yml diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml new file mode 100644 index 00000000..bd240ec7 --- /dev/null +++ b/.github/workflows/crypto-worker.yml @@ -0,0 +1,26 @@ +name: Crypto Data Worker +on: + schedule: + - cron: '*/5 * * * *' # Every 5 minutes + workflow_dispatch: # Manual trigger + +jobs: + worker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pip install -r services/worker/requirements.txt + pip install -r services/common/requirements.txt + pip install -r services/api/requirements.txt + - name: Run Data Worker + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + SCHEDULE_MINUTES: 5 + run: python services/worker/run_worker.py \ No newline at end of file diff --git a/README.md b/README.md index e3049688..08ad039a 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,23 @@ -# Crypto Risk Dashboard — Self-Hosted Starter (v2, Timescale-ready) +# Crypto Risk Dashboard — Free Cloud Deployment -**What’s inside** -- API: FastAPI -- UI: Streamlit (with automatic API URL detect) -- Worker: scheduled ingest + signal engine -- DB: Postgres schema (Timescale-ready: auto-hypertables + 1h continuous aggregate if extension is available) -- Render Blueprint: `render.yaml` (API + UI + Worker + free Postgres) +**Deployment**: Render (API + UI) + GitHub Actions (Worker) -**Quick start (Docker Compose, local)** -1) Copy `.env.example` to `.env` and edit as needed. You can set `DATABASE_URL` (preferred) or individual POSTGRES_* vars. -2) `docker compose up --build` -3) UI: http://localhost:8501 | API: http://localhost:8000 +## Quick Deploy: -**Render deployment** -- Push to GitHub, then in Render → Blueprints → New from repo. -- Set `API_CANDIDATES` on the UI service (defaults provided in render.yaml). The UI auto-detects API via /health. -- Set `DATABASE_URL` on API + Worker (Render sets it automatically if you bind the included Postgres; - or paste your Timescale Cloud connection string). +1. **Deploy to Render**: + - Connect your GitHub repo to Render + - Use the `render.yaml` blueprint + - Render will automatically create API, UI, and PostgreSQL -**Security note** -Do NOT commit real secrets (API keys, DB URLs) to Git. -Use Render environment variables or a local `.env` that you do not push. +2. **Setup GitHub Actions Worker**: + - Get `DATABASE_URL` from Render dashboard + - Add to GitHub Secrets: `DATABASE_URL = your_render_postgres_url` + - Push the code - worker runs automatically every 5 minutes + +3. **Access Your Dashboard**: + - UI: `https://crypto-risk-ui.onrender.com` + - API: `https://crypto-risk-api.onrender.com` + +## Local Development: +```bash +docker compose up --build \ No newline at end of file diff --git a/render.yaml b/render.yaml index 1be37499..c1a848d1 100644 --- a/render.yaml +++ b/render.yaml @@ -16,8 +16,6 @@ services: property: connectionString - key: SYMBOLS value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT - - key: SCHEDULE_MINUTES - value: "5" - type: web name: crypto-risk-ui @@ -27,18 +25,7 @@ services: envVars: - key: API_CANDIDATES value: https://crypto-risk-api.onrender.com,http://api:8000,http://localhost:8000 - - - type: worker - name: crypto-risk-worker - runtime: docker - rootDir: services/worker - plan: free - envVars: - key: DATABASE_URL fromDatabase: name: cryptodb - property: connectionString - - key: SYMBOLS - value: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT - - key: SCHEDULE_MINUTES - value: "5" + property: connectionString \ No newline at end of file diff --git a/services/common/adapters/exchanges.py b/services/common/adapters/exchanges.py index 77e2b095..b94acaf2 100644 --- a/services/common/adapters/exchanges.py +++ b/services/common/adapters/exchanges.py @@ -1,17 +1,54 @@ -import time, pandas as pd, datetime as dt +import time +import pandas as pd +import datetime as dt import ccxt +from .open_interest import fetch_funding_rate def fetch_candles(exchange_pair: str, timeframe="1h", limit=200): - ex_name, pair = exchange_pair.split(":", 1) - ex = getattr(ccxt, ex_name)() - ohlcv = ex.fetch_ohlcv(pair, timeframe=timeframe, limit=limit) + """Fetch real candles using CCXT""" + try: + ex_name, pair = exchange_pair.split(":", 1) + ex = getattr(ccxt, ex_name)({ + 'timeout': 10000, + 'enableRateLimit': True, + }) + ohlcv = ex.fetch_ohlcv(pair, timeframe=timeframe, limit=limit) + rows = [] + for t, o, h, l, c, v in ohlcv: + ts = dt.datetime.utcfromtimestamp(t/1000).replace(tzinfo=dt.timezone.utc) + rows.append({ + "pair": exchange_pair, + "ts": ts, + "open": o, + "high": h, + "low": l, + "close": c, + "volume": v + }) + return rows + except Exception as e: + print(f"Real candles failed for {exchange_pair}, using mock: {e}") + return mock_candles(exchange_pair, limit) + +def mock_candles(exchange_pair: str, limit=200): + """Fallback mock candles""" + import random rows = [] - for t,o,h,l,c,v in ohlcv: - ts = dt.datetime.utcfromtimestamp(t/1000).replace(tzinfo=dt.timezone.utc) - rows.append({"pair": exchange_pair, "ts": ts, "open": o, "high": h, "low": l, "close": c, "volume": v}) + base_price = 50000 if "BTC" in exchange_pair else 3000 + now = dt.datetime.now(dt.timezone.utc).replace(minute=0, second=0, microsecond=0) + + for i in range(limit): + ts = now - dt.timedelta(hours=limit-i-1) + price = base_price * (1 + random.uniform(-0.1, 0.1)) + rows.append({ + "pair": exchange_pair, + "ts": ts, + "open": price * 0.999, + "high": price * 1.002, + "low": price * 0.998, + "close": price, + "volume": random.uniform(1000, 5000) + }) return rows -def mock_funding(exchange_pair: str, candles_df: pd.DataFrame): - rate = (candles_df["close"].pct_change().fillna(0).tail(1).iloc[0]) / 10 - ts = candles_df["ts"].tail(1).iloc[0] - return [{"pair": exchange_pair, "ts": ts, "rate": float(rate)}] +# Remove mock_funding since we have real funding in open_interest.py \ No newline at end of file diff --git a/services/common/adapters/headlines.py b/services/common/adapters/headlines.py index 2e848133..6f928243 100644 --- a/services/common/adapters/headlines.py +++ b/services/common/adapters/headlines.py @@ -1,7 +1,68 @@ import datetime as dt +import requests +from typing import List, Dict + +def fetch_headlines_cryptopanic(api_key: str = None) -> List[Dict]: + """Fetch real headlines from CryptoPanic""" + if not api_key: + return fetch_headlines_mock() + + try: + url = "https://cryptopanic.com/api/v1/posts/" + params = { + 'auth_token': api_key, + 'public': 'true', + 'kind': 'news' + } + response = requests.get(url, params=params, timeout=10) + data = response.json() + + headlines = [] + for post in data.get('results', [])[:10]: # Get latest 10 + headlines.append({ + "ts": dt.datetime.fromisoformat(post['created_at'].replace('Z', '+00:00')), + "source": "CryptoPanic", + "title": post['title'], + "url": post['url'], + "keywords": extract_keywords(post['title']) + }) + return headlines + + except Exception as e: + print(f"CryptoPanic headlines error: {e}") + return fetch_headlines_mock() + +def extract_keywords(title: str) -> List[str]: + """Extract relevant keywords from title""" + keywords = [] + crypto_terms = ['bitcoin', 'btc', 'ethereum', 'eth', 'solana', 'sol', + 'funding', 'liquidat', 'margin', 'oi', 'open interest', + 'crash', 'rally', 'surge', 'dump'] + + title_lower = title.lower() + for term in crypto_terms: + if term in title_lower: + keywords.append(term) + return keywords + def fetch_headlines_mock(): + """Fallback mock headlines""" now = dt.datetime.now(dt.timezone.utc) return [{ - "ts": now, "source": "mock", "title": "Market wobbles as OI surges; funding flips negative", - "url": "https://example.com", "keywords": ["open interest", "funding"] + "ts": now, + "source": "mock", + "title": "Market shows mixed signals as OI surges and funding turns negative", + "url": "https://example.com", + "keywords": ["open interest", "funding"] }] + +def fetch_headlines(api_key: str = None) -> List[Dict]: + """Main headlines function with real data fallback""" + try: + real_headlines = fetch_headlines_cryptopanic(api_key) + if real_headlines: + return real_headlines + except Exception as e: + print(f"Real headlines failed, using mock: {e}") + + return fetch_headlines_mock() \ No newline at end of file diff --git a/services/common/adapters/open_interest.py b/services/common/adapters/open_interest.py index c7ef46c5..3a14f6f3 100644 --- a/services/common/adapters/open_interest.py +++ b/services/common/adapters/open_interest.py @@ -1,5 +1,102 @@ -import datetime as dt, random -def fetch_open_interest(exchange_pair: str): +import datetime as dt +import requests +import pandas as pd +from typing import List, Dict, Optional + +def fetch_open_interest_binance(symbol: str) -> Optional[float]: + """Fetch real OI from Binance without API key""" + try: + sym = symbol.replace("/", "").replace("binance:", "") + url = f"https://fapi.binance.com/fapi/v1/openInterest?symbol={sym}" + response = requests.get(url, timeout=10) + data = response.json() + return float(data['openInterest']) * 1000 # Convert to approximate USD + except Exception as e: + print(f"Binance OI error for {symbol}: {e}") + return None + +def fetch_open_interest_bybit(symbol: str) -> Optional[float]: + """Fetch real OI from Bybit without API key""" + try: + sym = symbol.replace("/", "").replace("bybit:", "") + url = f"https://api.bybit.com/v5/market/open-interest?category=linear&symbol={sym}&interval=5min" + response = requests.get(url, timeout=10) + data = response.json() + return float(data['result']['list'][0]['openInterest']) + except Exception as e: + print(f"Bybit OI error for {symbol}: {e}") + return None + +def fetch_funding_rate_binance(symbol: str) -> Optional[float]: + """Fetch real funding rate from Binance""" + try: + sym = symbol.replace("/", "").replace("binance:", "") + url = f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={sym}&limit=1" + response = requests.get(url, timeout=10) + data = response.json() + return float(data[0]['fundingRate']) + except Exception as e: + print(f"Binance funding error for {symbol}: {e}") + return None + +def fetch_funding_rate_bybit(symbol: str) -> Optional[float]: + """Fetch real funding rate from Bybit""" + try: + sym = symbol.replace("/", "").replace("bybit:", "") + url = f"https://api.bybit.com/v5/market/funding/history?category=linear&symbol={sym}&limit=1" + response = requests.get(url, timeout=10) + data = response.json() + return float(data['result']['list'][0]['fundingRate']) + except Exception as e: + print(f"Bybit funding error for {symbol}: {e}") + return None + +def fetch_open_interest(exchange_pair: str) -> List[Dict]: + """Get real OI data with fallback to mock""" now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + + try: + if exchange_pair.startswith("binance:"): + oi_value = fetch_open_interest_binance(exchange_pair) + elif exchange_pair.startswith("bybit:"): + oi_value = fetch_open_interest_bybit(exchange_pair) + else: + oi_value = None + + if oi_value is not None: + return [{"pair": exchange_pair, "ts": now, "value_usd": float(oi_value)}] + + except Exception as e: + print(f"Real OI failed for {exchange_pair}, using mock: {e}") + + # Fallback to mock data + import random base = 1_000_000 + random.randint(-50_000, 50_000) return [{"pair": exchange_pair, "ts": now, "value_usd": float(base)}] + +def fetch_funding_rate(exchange_pair: str, candles_df: pd.DataFrame = None) -> List[Dict]: + """Get real funding rate data with fallback to mock""" + now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + + try: + if exchange_pair.startswith("binance:"): + rate = fetch_funding_rate_binance(exchange_pair) + elif exchange_pair.startswith("bybit:"): + rate = fetch_funding_rate_bybit(exchange_pair) + else: + rate = None + + if rate is not None: + return [{"pair": exchange_pair, "ts": now, "rate": float(rate)}] + + except Exception as e: + print(f"Real funding failed for {exchange_pair}, using mock: {e}") + + # Fallback to mock data + if candles_df is not None and not candles_df.empty: + rate = (candles_df["close"].pct_change().fillna(0).tail(1).iloc[0]) / 10 + else: + import random + rate = random.uniform(-0.0005, 0.0005) + + return [{"pair": exchange_pair, "ts": now, "rate": float(rate)}] \ No newline at end of file diff --git a/services/common/adapters/sentiment.py b/services/common/adapters/sentiment.py index fafd59cf..db547532 100644 --- a/services/common/adapters/sentiment.py +++ b/services/common/adapters/sentiment.py @@ -1,8 +1,91 @@ -import datetime as dt, random -KEYWORDS = ["liquidation", "margin call", "rekt", "funding", "open interest"] +import datetime as dt +import requests +import random +from typing import List, Dict + +KEYWORDS = ["liquidation", "margin call", "rekt", "funding", "open interest", "crash", "rally"] + +def fetch_sentiment_cryptopanic(api_key: str = None) -> List[Dict]: + """Fetch real sentiment from CryptoPanic (free tier)""" + if not api_key: + return fetch_sentiment_mock("global") + + try: + url = "https://cryptopanic.com/api/v1/posts/" + params = { + 'auth_token': api_key, + 'public': 'true', + 'kind': 'news', + 'filter': 'important' + } + response = requests.get(url, params=params, timeout=10) + data = response.json() + + posts = data.get('results', []) + mentions = len(posts) + + # Simple sentiment analysis based on post titles + positive_words = ['rally', 'surge', 'bull', 'up', 'green', 'gain'] + negative_words = ['crash', 'drop', 'bear', 'down', 'red', 'loss', 'liquidat'] + + score = 0 + keyword_counts = {k: 0 for k in KEYWORDS} + + for post in posts: + title = post.get('title', '').lower() + # Count keywords + for keyword in KEYWORDS: + if keyword in title: + keyword_counts[keyword] += 1 + # Calculate sentiment + if any(word in title for word in positive_words): + score += 1 + if any(word in title for word in negative_words): + score -= 1 + + # Normalize score + if mentions > 0: + score_norm = score / mentions + else: + score_norm = 0 + + now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) + return [{ + "pair": "global", + "ts": now, + "mentions": mentions, + "score_norm": score_norm, + "keywords": keyword_counts + }] + + except Exception as e: + print(f"CryptoPanic error: {e}") + return fetch_sentiment_mock("global") + def fetch_sentiment_mock(exchange_pair: str): + """Fallback mock sentiment""" now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) mentions = random.randint(5, 50) score = random.uniform(-1, 1) kw_counts = {k: random.randint(0, mentions//2) for k in KEYWORDS} - return [{"pair": exchange_pair, "ts": now, "mentions": mentions, "score_norm": score, "keywords": kw_counts}] + return [{ + "pair": exchange_pair, + "ts": now, + "mentions": mentions, + "score_norm": score, + "keywords": kw_counts + }] + +def fetch_sentiment(exchange_pair: str, api_key: str = None) -> List[Dict]: + """Main sentiment function with real data fallback""" + try: + # Try real data first + real_data = fetch_sentiment_cryptopanic(api_key) + if real_data: + # Convert global sentiment to pair-specific + real_data[0]["pair"] = exchange_pair + return real_data + except Exception as e: + print(f"Real sentiment failed, using mock: {e}") + + return fetch_sentiment_mock(exchange_pair) \ No newline at end of file diff --git a/services/common/ingest.py b/services/common/ingest.py index c5dffffe..60b7230e 100644 --- a/services/common/ingest.py +++ b/services/common/ingest.py @@ -1,32 +1,57 @@ import pandas as pd from services_common.config import load_config from services_common.db import upsert_many -from services_common.adapters.exchanges import fetch_candles, mock_funding -from services_common.adapters.open_interest import fetch_open_interest +from services_common.adapters.exchanges import fetch_candles +from services_common.adapters.open_interest import fetch_open_interest, fetch_funding_rate from services_common.adapters.volatility import compute_atr_like -from services_common.adapters.sentiment import fetch_sentiment_mock -from services_common.adapters.headlines import fetch_headlines_mock +from services_common.adapters.sentiment import fetch_sentiment +from services_common.adapters.headlines import fetch_headlines cfg = load_config() def run_ingest_cycle(): + print(f"[ingest] Starting cycle for {len(cfg.symbols)} pairs...") + + # Process each trading pair for pair in cfg.symbols: + print(f"[ingest] Processing {pair}...") + + # 1. Fetch real candles candle_rows = fetch_candles(pair, timeframe="1h", limit=200) - upsert_many("candles", candle_rows, ["pair","ts"], ["open","high","low","close","volume"]) + if candle_rows: + upsert_many("candles", candle_rows, ["pair","ts"], ["open","high","low","close","volume"]) + print(f"[ingest] Saved {len(candle_rows)} candles for {pair}") - df = pd.DataFrame(candle_rows) - fr = mock_funding(pair, df) - upsert_many("funding_rates", fr, ["pair","ts"], ["rate"]) + # 2. Fetch real funding rates (using real adapter) + df = pd.DataFrame(candle_rows) if candle_rows else pd.DataFrame() + fr = fetch_funding_rate(pair, df) + if fr: + upsert_many("funding_rates", fr, ["pair","ts"], ["rate"]) + print(f"[ingest] Saved funding rate for {pair}") + # 3. Fetch real open interest oi = fetch_open_interest(pair) - upsert_many("open_interest", oi, ["pair","ts"], ["value_usd"]) + if oi: + upsert_many("open_interest", oi, ["pair","ts"], ["value_usd"]) + print(f"[ingest] Saved OI for {pair}") - vol = compute_atr_like(df) - if vol: - upsert_many("volatility", vol, ["pair","ts"], ["atr"]) + # 4. Compute volatility from candles + if not df.empty: + vol = compute_atr_like(df) + if vol: + upsert_many("volatility", vol, ["pair","ts"], ["atr"]) + print(f"[ingest] Saved volatility for {pair}") - sent = fetch_sentiment_mock(pair) - upsert_many("sentiment", sent, ["pair","ts"], ["mentions","score_norm","keywords"]) + # 5. Fetch sentiment (real with fallback) + sent = fetch_sentiment(pair) # Now uses real data with CryptoPanic fallback + if sent: + upsert_many("sentiment", sent, ["pair","ts"], ["mentions","score_norm","keywords"]) + print(f"[ingest] Saved sentiment for {pair}") - h = fetch_headlines_mock() - upsert_many("headlines", h, ["id"], []) + # 6. Fetch headlines (real with fallback) + h = fetch_headlines() # Now uses real data with CryptoPanic fallback + if h: + upsert_many("headlines", h, ["id"], []) + print(f"[ingest] Saved {len(h)} headlines") + + print("[ingest] Cycle completed successfully!") \ No newline at end of file diff --git a/services/worker/requirements.txt b/services/worker/requirements.txt index 9d2300aa..7ff118af 100644 --- a/services/worker/requirements.txt +++ b/services/worker/requirements.txt @@ -19,3 +19,8 @@ requests==2.32.3 beautifulsoup4==4.12.3 streamlit==1.39.0 streamlit_echarts==0.4.0 +psycopg2-binary==2.9.9 +pandas==2.1.4 +requests==2.31.0 +ccxt==4.1.60 +python-dotenv==1.0.0 diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 120c2c72..69d3e521 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -1,20 +1,20 @@ -import os, time +import os +import sys +# Add the common module to path +sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) + from services_common.db import ensure_schema from services_common.ingest import run_ingest_cycle from services_common.signals import compute_all_signals -from services_common.config import load_config def main(): + print("[worker] Starting worker cycle...") ensure_schema() - interval = int(os.getenv("SCHEDULE_MINUTES", "5")) - print(f"[worker] schedule every {interval} minutes.") - while True: - print("[worker] ingest cycle...") - run_ingest_cycle() - print("[worker] compute signals...") - compute_all_signals() - print("[worker] sleep...") - time.sleep(60 * interval) + print("[worker] Running ingest cycle...") + run_ingest_cycle() + print("[worker] Computing signals...") + compute_all_signals() + print("[worker] Worker cycle completed successfully!") if __name__ == "__main__": - main() + main() \ No newline at end of file From 58b5bf06f48021e6b82321fc887d78284b93ad87 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 16:59:21 +0100 Subject: [PATCH 05/43] Optimize Dockerfile with multi-stage build Refactor Dockerfile to use multi-stage build and add health check. --- services/api/Dockerfile | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/services/api/Dockerfile b/services/api/Dockerfile index f8fda52b..a62b999a 100644 --- a/services/api/Dockerfile +++ b/services/api/Dockerfile @@ -1,6 +1,25 @@ -FROM python:3.11-slim +# Use multi-stage build for smaller image +FROM python:3.11-slim as builder + WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt + +FROM python:3.11-slim + +# Create non-root user +RUN groupadd -r appuser && useradd -r -g appuser appuser + +WORKDIR /app +COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin COPY . . -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] + +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=3 \ + CMD curl --fail http://localhost:8000/health || exit 1 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"] From 820e1f5c10d50e88e8cb6d1ff207f5924a9ed39e Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:00:04 +0100 Subject: [PATCH 06/43] Enhance Docker Compose with health checks and cleanup Updated Docker Compose configuration to include health checks for the API and database services, and removed the Redis service dependency from the worker. --- docker-compose.yml | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d2c3f71c..50005e57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,6 @@ +version: "3.9" services: + db: image: postgres:16 environment: @@ -14,11 +16,12 @@ services: interval: 5s timeout: 5s retries: 20 - - redis: - image: redis:7 - ports: - - "6379:6379" + restart: always + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" api: build: ./services/api @@ -28,6 +31,18 @@ services: condition: service_healthy ports: - "8000:8000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 10s + timeout: 5s + retries: 5 + restart: on-failure + user: "1000:1000" + deploy: + resources: + limits: + memory: 1024M + cpus: "1.0" worker: build: ./services/worker @@ -35,8 +50,13 @@ services: depends_on: db: condition: service_healthy - redis: - condition: service_started + restart: on-failure + user: "1000:1000" + deploy: + resources: + limits: + memory: 1024M + cpus: "1.0" ui: build: ./services/ui @@ -46,6 +66,13 @@ services: condition: service_started ports: - "8501:8501" + restart: on-failure + user: "1000:1000" + deploy: + resources: + limits: + memory: 1024M + cpus: "1.0" volumes: db_data: From 956c238b9e464839d1c485b1234ff1f30e78095d Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:00:46 +0100 Subject: [PATCH 07/43] Add Streamlit and Streamlit Echarts to requirements --- services/worker/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/worker/requirements.txt b/services/worker/requirements.txt index 0941b5de..9d2300aa 100644 --- a/services/worker/requirements.txt +++ b/services/worker/requirements.txt @@ -17,3 +17,5 @@ textblob==0.18.0.post0 nltk==3.9.1 requests==2.32.3 beautifulsoup4==4.12.3 +streamlit==1.39.0 +streamlit_echarts==0.4.0 From fcb5f30d3587eccb0013550127210e18118d0be1 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:01:01 +0100 Subject: [PATCH 08/43] Remove streamlit from requirements.txt Removed streamlit version from requirements. --- services/ui/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/services/ui/requirements.txt b/services/ui/requirements.txt index 3f258ee0..9d2300aa 100644 --- a/services/ui/requirements.txt +++ b/services/ui/requirements.txt @@ -17,6 +17,5 @@ textblob==0.18.0.post0 nltk==3.9.1 requests==2.32.3 beautifulsoup4==4.12.3 - streamlit==1.39.0 streamlit_echarts==0.4.0 From e92daa4b179622eac789bb03f2f202907b41f14f Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:01:22 +0100 Subject: [PATCH 09/43] Add Streamlit and Streamlit Echarts to requirements --- services/api/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/api/requirements.txt b/services/api/requirements.txt index 0941b5de..9d2300aa 100644 --- a/services/api/requirements.txt +++ b/services/api/requirements.txt @@ -17,3 +17,5 @@ textblob==0.18.0.post0 nltk==3.9.1 requests==2.32.3 beautifulsoup4==4.12.3 +streamlit==1.39.0 +streamlit_echarts==0.4.0 From f0f96a9a7da1888b3bd0beb13bb9fc1c47dcbb7b Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:16:31 +0100 Subject: [PATCH 10/43] Refactor app.py for improved readability and structure Refactor code for better readability and structure, including improvements to API base resolution and data fetching functions. --- services/ui/app.py | 185 +++++++++++++++++++-------------------------- 1 file changed, 78 insertions(+), 107 deletions(-) diff --git a/services/ui/app.py b/services/ui/app.py index 54929773..35f527f6 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -1,11 +1,12 @@ -import os, requests, pandas as pd, time +import os +import requests +import pandas as pd import streamlit as st from urllib.parse import urlencode +from typing import Optional DEFAULT_CANDIDATES = [ - # 1) Render default if you kept the blueprint names: "https://crypto-risk-api.onrender.com", - # 2) Local docker compose: "http://api:8000", "http://localhost:8000", ] @@ -17,139 +18,109 @@ def probe_api(base: str, timeout=2.0) -> bool: except Exception: return False -def resolve_api_base(): - # Priority 1: explicit env var +def resolve_api_base() -> Optional[str]: env_base = os.getenv("API_BASE", "").strip() if env_base and probe_api(env_base): return env_base - - # Priority 2: comma-separated list of candidates from env env_candidates = os.getenv("API_CANDIDATES", "") candidates = [c.strip() for c in env_candidates.split(",") if c.strip()] if env_candidates else [] - # Merge with defaults (preserve order; avoid dups) - seen = set() - merged = [] + seen, merged = set(), [] for x in (candidates + DEFAULT_CANDIDATES): if x not in seen: merged.append(x) seen.add(x) - for base in merged: if probe_api(base): return base - - return None # nothing worked + return None API_BASE = resolve_api_base() st.set_page_config(page_title="Crypto Risk Dashboard", layout="wide") st.title("🧭 Crypto Risk Dashboard") -st.caption("Self-hosted. Graphs • Meters • Hot Signals") if not API_BASE: - st.error("Could not locate the API automatically.") - manual = st.text_input("Enter your API base URL (e.g., https://crypto-risk-api.onrender.com)") - if manual: - if probe_api(manual): - API_BASE = manual - st.success("Connected!") + st.error("Could not connect to API backend. Please ensure the API is running and reachable.") + st.stop() + +@st.cache_data(ttl=300) +def fetch(endpoint: str, params: Optional[dict] = None) -> Optional[pd.DataFrame]: + url = f"{API_BASE}/{endpoint}" + try: + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + data = response.json() + if "data" in data and isinstance(data["data"], list): + return pd.DataFrame(data["data"]) else: - st.warning("That URL didn’t respond at /health. Double-check and try again.") + st.warning(f"No data found for endpoint: {endpoint}") + return None + except Exception as e: + st.error(f"Error fetching {endpoint}: {e}") + return None + +# Refresh button to clear cache +if st.button("Refresh Data"): + fetch.clear() + +pairs_df = fetch("pairs") +pair = None +if pairs_df is not None and not pairs_df.empty: + pair = st.selectbox("Select trading pair", pairs_df["pair"].unique()) +else: + st.warning("No trading pairs available from API.") -if not API_BASE: +# If no pair selected, stop further processing +if not pair: + st.info("Select a pair to see data.") st.stop() -# Pairs -resp = requests.get(f"{API_BASE}/pairs", timeout=30).json() -pairs = resp.get("pairs", ["binance:BTC/USDT"]) - -col1, col2 = st.columns([2,1]) -with col1: - pair = st.selectbox("Pair", pairs, index=0) -with col2: - refresh = st.button("Refresh") - -def fetch(metric, pair, limit=500): - qs = urlencode({"pair": pair, "limit": limit}) - r = requests.get(f"{API_BASE}/timeseries/{metric}?{qs}", timeout=30) - return r.json() - -def get_signals(pairs=None): - qs = "" - if pairs: - qs = "?pairs=" + ",".join(pairs) - r = requests.get(f"{API_BASE}/signals{qs}", timeout=30) - return r.json() - -sig = get_signals([pair]) -signals = sig.get("signals", {}) -explanations = sig.get("explanations", {}) - -st.subheader("Hot Signals") -if signals.get(pair): - s = signals[pair] - c1, c2, c3, c4 = st.columns(4) - with c1: - st.metric("Market Regime", s.get("regime","—")) - with c2: - st.metric("Bias", s.get("bias","—")) - with c3: - st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) - with c4: - st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) - st.info(s.get("summary","")) -else: - st.warning("No signals yet. The worker may still be seeding data.") +limit = 1000 -st.divider() -st.subheader("Charts") +st.subheader(f"Data for Pair: {pair}") -tabs = st.tabs(["Candles", "Funding", "Open Interest", "Volatility", "Sentiment"]) +data_tabs = st.tabs(["Candles", "Funding Rates", "Open Interest", "Volatility", "Sentiment", "Signals"]) -with tabs[0]: - data = fetch("candles", pair, limit=300) - rows = data.get("rows", []) - if rows: - df = pd.DataFrame(rows).sort_values("ts") - st.line_chart(df, x="ts", y=["close"]) +with data_tabs[0]: + candles = fetch("candles", params={"pair": pair, "limit": limit}) + if candles is not None and not candles.empty: + st.line_chart(candles.set_index("ts")[["open", "high", "low", "close"]]) else: - st.write("No data yet.") - -with tabs[1]: - data = fetch("funding", pair, limit=500) - rows = data.get("rows", []) - if rows: - df = pd.DataFrame(rows).sort_values("ts") - st.line_chart(df, x="ts", y=["rate"]) + st.write("No candles data available.") + +with data_tabs[1]: + funding = fetch("funding", params={"pair": pair, "limit": limit}) + if funding is not None and not funding.empty: + st.line_chart(funding.set_index("ts")["funding_rate"]) else: - st.write("No data yet.") - -with tabs[2]: - data = fetch("oi", pair, limit=500) - rows = data.get("rows", []) - if rows: - df = pd.DataFrame(rows).sort_values("ts") - st.line_chart(df, x="ts", y=["value_usd"]) + st.write("No funding rate data available.") + +with data_tabs[2]: + oi = fetch("open_interest", params={"pair": pair, "limit": limit}) + if oi is not None and not oi.empty: + st.line_chart(oi.set_index("ts")["open_interest"]) else: - st.write("No data yet.") - -with tabs[3]: - data = fetch("vol", pair, limit=500) - rows = data.get("rows", []) - if rows: - df = pd.DataFrame(rows).sort_values("ts") - st.line_chart(df, x="ts", y=["atr"]) + st.write("No open interest data available.") + +with data_tabs[3]: + vol = fetch("volatility", params={"pair": pair, "limit": limit}) + if vol is not None and not vol.empty: + st.line_chart(vol.set_index("ts")["atr"]) else: - st.write("No data yet.") - -with tabs[4]: - data = fetch("sentiment", pair, limit=200) - rows = data.get("rows", []) - if rows: - df = pd.DataFrame(rows).sort_values("ts") - st.line_chart(df, x="ts", y=["score_norm"]) + st.write("No volatility data available.") + +with data_tabs[4]: + sentiment = fetch("sentiment", params={"pair": pair, "limit": limit}) + if sentiment is not None and not sentiment.empty: + st.line_chart(sentiment.set_index("ts")["sentiment_score"]) + else: + st.write("No sentiment data available.") + +with data_tabs[5]: + signals = fetch("signals", params={"pair": pair}) + if signals is not None and not signals.empty: + st.write(signals) else: - st.write("No data yet.") + st.write("No signals data available.") -st.divider() -st.caption("Tip: set API_BASE or API_CANDIDATES env vars to skip auto-detect.") +st.caption("Configure pairs, scheduler, and API keys in your .env file or Render.com environment settings.") From 86536bf3c31bb93d253f8e7ee35b8207523dfe63 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:17:27 +0100 Subject: [PATCH 11/43] Enhance sentiment fetching with CryptoPanic API Refactor sentiment fetching to include real API integration and enhance keyword handling. --- services/common/adapters/sentiment.py | 87 ++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/services/common/adapters/sentiment.py b/services/common/adapters/sentiment.py index 49aa852d..3d9f52da 100644 --- a/services/common/adapters/sentiment.py +++ b/services/common/adapters/sentiment.py @@ -1,9 +1,78 @@ -import datetime as dt, random -KEYWORDS = ["liquidation","margin call","rekt","funding","open interest"] - -def fetch_sentiment_mock(exchange_pair: str): - now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) - mentions = random.randint(5, 50) - score = random.uniform(-1, 1) - kw_counts = {k: random.randint(0, mentions//2) for k in KEYWORDS} - return [{"pair": exchange_pair, "ts": now, "mentions": mentions, "score_norm": score, "keywords": kw_counts}] +import datetime as dt +import requests +import random +import logging +from typing import List, Dict, Optional +from functools import lru_cache + +# Externalize keywords to lists +KEYWORDS = [ + "liquidation", "margin call", "rekt", "funding", "open interest", + "crash", "rally" +] + +POSITIVE_WORDS = ["rally", "surge", "bull", "up", "green", "gain"] +NEGATIVE_WORDS = ["crash", "drop", "bear", "down", "red", "loss", "liquidat"] + +@lru_cache(maxsize=128) +def fetch_sentiment_cryptopanic(api_key: Optional[str] = None) -> List[Dict]: + """Fetch real sentiment from CryptoPanic API, fallback to mock data.""" + if not api_key: + return fetch_sentiment_mock("global") + + try: + url = "https://cryptopanic.com/api/v1/posts/" + params = { + 'auth_token': api_key, + 'public': 'true', + 'kind': 'news', + 'filter': 'important' + } + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + data = response.json() + posts = data.get('results', []) + + mentions = len(posts) + score = 0 + keyword_counts = {k: 0 for k in KEYWORDS} + + for post in posts: + title = post.get('title', '').lower() + # Count keywords + for keyword in KEYWORDS: + if keyword in title: + keyword_counts[keyword] += 1 + # Simple sentiment scoring + for word in POSITIVE_WORDS: + if word in title: + score += 1 + for word in NEGATIVE_WORDS: + if word in title: + score -= 1 + + normalized_score = score / max(mentions, 1) # Avoid division by zero + + return [{ + "timestamp": dt.datetime.utcnow().isoformat(), + "mentions": mentions, + "sentiment_score": normalized_score, + "keyword_counts": keyword_counts + }] + + except Exception as e: + logging.error(f"Error fetching sentiment from CryptoPanic: {e}") + return fetch_sentiment_mock("global") + +def fetch_sentiment_mock(pair: str) -> List[Dict]: + """Generate mock sentiment data.""" + mentions = random.randint(0, 20) + sentiment_score = random.uniform(-1, 1) + keyword_counts = {k: random.randint(0, 5) for k in KEYWORDS} + + return [{ + "timestamp": dt.datetime.utcnow().isoformat(), + "mentions": mentions, + "sentiment_score": sentiment_score, + "keyword_counts": keyword_counts + }] From 730afc21e5cd20f20ac6dce31914084cfe765e2e Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 17:20:12 +0100 Subject: [PATCH 12/43] Enhance TimescaleDB schema with indexes and aggregates Added indexes for faster filtering and created a continuous aggregate for 1-hour candles data with a refresh policy. --- services/common/schema_timescale.sql | 40 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/services/common/schema_timescale.sql b/services/common/schema_timescale.sql index 15e97882..1b770a9b 100644 --- a/services/common/schema_timescale.sql +++ b/services/common/schema_timescale.sql @@ -1,4 +1,7 @@ +-- Enable TimescaleDB extension if available CREATE EXTENSION IF NOT EXISTS timescaledb; + +-- Convert base tables to hypertables if not already SELECT create_hypertable('candles', 'ts', if_not_exists => TRUE); SELECT create_hypertable('funding_rates', 'ts', if_not_exists => TRUE); SELECT create_hypertable('open_interest', 'ts', if_not_exists => TRUE); @@ -6,22 +9,35 @@ SELECT create_hypertable('volatility', 'ts', if_not_exists => TRUE); SELECT create_hypertable('sentiment', 'ts', if_not_exists => TRUE); SELECT create_hypertable('signals', 'ts', if_not_exists => TRUE); +-- Add indexes for faster filtering by 'pair' +CREATE INDEX IF NOT EXISTS idx_candles_pair_ts ON candles (pair, ts DESC); +CREATE INDEX IF NOT EXISTS idx_funding_rates_pair_ts ON funding_rates (pair, ts DESC); +CREATE INDEX IF NOT EXISTS idx_open_interest_pair_ts ON open_interest (pair, ts DESC); +CREATE INDEX IF NOT EXISTS idx_volatility_pair_ts ON volatility (pair, ts DESC); +CREATE INDEX IF NOT EXISTS idx_sentiment_pair_ts ON sentiment (pair, ts DESC); +CREATE INDEX IF NOT EXISTS idx_signals_pair_ts ON signals (pair, ts DESC); + +-- Create continuous aggregate for candles 1-hour data with auto-refresh CREATE MATERIALIZED VIEW IF NOT EXISTS candles_1h WITH (timescaledb.continuous) AS SELECT - time_bucket('1 hour', ts) AS bucket, - pair, - first(open, ts) AS o, - max(high) AS h, - min(low) AS l, - last(close, ts) AS c, - sum(volume) AS v + time_bucket('1 hour', ts) AS bucket, + pair, + first(open, ts) AS o, + max(high) AS h, + min(low) AS l, + last(close, ts) AS c, + sum(volume) AS v FROM candles -GROUP BY 1,2; +GROUP BY 1, 2; +-- Add continuous aggregate refresh policy for candles_1h SELECT add_continuous_aggregate_policy( - 'candles_1h', - start_offset => INTERVAL '3 days', - end_offset => INTERVAL '5 minutes', - schedule_interval => INTERVAL '15 minutes' + 'candles_1h', + start_offset => INTERVAL '3 days', + end_offset => INTERVAL '5 minutes', + schedule_interval => INTERVAL '15 minutes' ); + +-- Monitor hypertable chunk sizes and adjust chunk_time_interval as needed +-- Example: ALTER TABLE candles SET (timescaledb.chunk_time_interval = INTERVAL '1 day'); From 3dc81ab94584b4758f97979ad15af3ff0cb7eca6 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 23:14:45 +0100 Subject: [PATCH 13/43] v4 --- .github/workflows/crypto-worker.yml | 62 +++++++--- RAILWAY_DEPLOY.md | 182 ++++++++++++++++++++++++++++ README.md | 91 +++++++++++--- docker-compose.yml | 8 +- railway.json | 12 ++ railway.toml | 9 ++ services/api/Dockerfile | 17 ++- services/api/railway.json | 13 ++ services/ui/Dockerfile | 17 ++- services/ui/app.py | 3 +- services/ui/railway.json | 13 ++ 11 files changed, 384 insertions(+), 43 deletions(-) create mode 100644 RAILWAY_DEPLOY.md create mode 100644 railway.json create mode 100644 railway.toml create mode 100644 services/api/railway.json create mode 100644 services/ui/railway.json diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index bd240ec7..31b6a084 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -1,26 +1,52 @@ name: Crypto Data Worker + on: schedule: - - cron: '*/5 * * * *' # Every 5 minutes - workflow_dispatch: # Manual trigger + - cron: '*/5 * * * *' # Every 5 minutes + workflow_dispatch: jobs: worker: runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Install dependencies - run: | - pip install -r services/worker/requirements.txt - pip install -r services/common/requirements.txt - pip install -r services/api/requirements.txt - - name: Run Data Worker - env: - DATABASE_URL: ${{ secrets.DATABASE_URL }} - SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT - SCHEDULE_MINUTES: 5 - run: python services/worker/run_worker.py \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Cache Python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/common/requirements.txt', 'services/api/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r services/worker/requirements.txt + pip install -r services/common/requirements.txt + pip install -r services/api/requirements.txt + + - name: Run Data Worker + env: + # DATABASE_URL should be set from Railway PostgreSQL service + # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL + # Add it to GitHub Secrets as DATABASE_URL + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + SCHEDULE_MINUTES: 5 + run: python services/worker/run_worker.py + + # Optional Slack notification on failure + - name: Notify Slack on failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: failure + fields: repo,message,commit,author,workflow + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/RAILWAY_DEPLOY.md b/RAILWAY_DEPLOY.md new file mode 100644 index 00000000..778e91b9 --- /dev/null +++ b/RAILWAY_DEPLOY.md @@ -0,0 +1,182 @@ +# Railway Deployment Guide + +This guide provides detailed instructions for deploying the Crypto Risk Dashboard to Railway. + +## Prerequisites + +- GitHub account with this repository +- Railway account (sign up at [railway.app](https://railway.app)) +- Railway CLI (optional, for local testing) + +## Architecture + +- **Railway Services**: API, UI, and PostgreSQL database +- **GitHub Actions**: Worker service (runs every 5 minutes) + +## Deployment Steps + +### 1. Create Railway Project + +1. Go to [railway.app](https://railway.app) and sign in with GitHub +2. Click "New Project" +3. Select "Empty Project" or "Deploy from GitHub repo" +4. Connect your GitHub repository + +### 2. Create PostgreSQL Database + +1. In your Railway project, click "+ New" +2. Select "Database" → "Add PostgreSQL" +3. Railway will automatically provision a PostgreSQL instance +4. Note the service name (e.g., "Postgres") + +### 3. Deploy API Service + +1. Click "+ New" → "GitHub Repo" +2. Select your repository +3. In the service settings: + - **Root Directory**: Set to `services/api` + - **Build Command**: Leave empty (uses Dockerfile) + - Railway will detect the Dockerfile automatically + +4. **Add Environment Variables**: + - Go to "Variables" tab + - Click "New Variable" + - `DATABASE_URL`: Click "Reference Variable" → Select your PostgreSQL service → Select `DATABASE_URL` + - `SYMBOLS`: `binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT` + +5. **Generate Public URL**: + - Go to "Settings" → "Networking" + - Click "Generate Domain" + - Note your API URL (e.g., `https://crypto-risk-api-production.up.railway.app`) + +### 4. Deploy UI Service + +1. Click "+ New" → "GitHub Repo" +2. Select the same repository +3. In the service settings: + - **Root Directory**: Set to `services/ui` + - **Build Command**: Leave empty (uses Dockerfile) + +4. **Add Environment Variables**: + - `API_CANDIDATES`: Your API URL from step 3 (e.g., `https://crypto-risk-api-production.up.railway.app,http://api:8000,http://localhost:8000`) + - `DATABASE_URL`: Reference from PostgreSQL service (same as API service) + +5. **Generate Public URL**: + - Generate a domain for UI service + - Note your UI URL + +### 5. Configure GitHub Actions Worker + +The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). + +1. **Get Database Connection String**: + - In Railway → Your PostgreSQL service → "Variables" tab + - Copy the `DATABASE_URL` value + - It should look like: `postgresql://user:password@host:port/dbname` + +2. **Add GitHub Secret**: + - Go to your GitHub repository + - Settings → Secrets and variables → Actions + - Click "New repository secret" + - Name: `DATABASE_URL` + - Value: Paste your Railway PostgreSQL `DATABASE_URL` + - Click "Add secret" + +3. **Verify Workflow**: + - Go to Actions tab in GitHub + - The "Crypto Data Worker" workflow should run automatically every 5 minutes + - You can manually trigger it using "workflow_dispatch" + +### 6. Verify Deployment + +1. **Check API Health**: + - Visit your API URL: `https://your-api-url.railway.app/health` + - Should return: `{"ok": true}` + +2. **Check UI**: + - Visit your UI URL + - Dashboard should load and show pair selection + +3. **Check Worker**: + - Go to GitHub Actions + - Check recent runs are successful + - Wait a few minutes for initial data ingestion + +## Environment Variables Reference + +### API Service +- `DATABASE_URL`: PostgreSQL connection string (referenced from database service) +- `SYMBOLS`: Comma-separated trading pairs (e.g., `binance:BTC/USDT,binance:ETH/USDT`) +- `PORT`: Automatically set by Railway (don't set manually) + +### UI Service +- `API_CANDIDATES`: Comma-separated API URLs to try (include your Railway API URL) +- `DATABASE_URL`: PostgreSQL connection string (referenced from database service) +- `PORT`: Automatically set by Railway (don't set manually) + +### Worker (GitHub Actions) +- `DATABASE_URL`: PostgreSQL connection string (from GitHub Secrets) +- `SYMBOLS`: Trading pairs (set in workflow file) +- `SCHEDULE_MINUTES`: Worker schedule interval (set in workflow file) + +## Troubleshooting + +### API not connecting to database +- Verify `DATABASE_URL` is correctly referenced from PostgreSQL service +- Check PostgreSQL service is running in Railway dashboard +- Ensure database has been initialized (first worker run will do this) + +### UI can't find API +- Verify `API_CANDIDATES` includes your Railway API URL +- Check API service is running and health endpoint works +- Ensure API URL is publicly accessible (not internal Railway hostname) + +### Worker not running +- Check GitHub Actions workflow is enabled +- Verify `DATABASE_URL` secret is set correctly +- Check workflow logs for errors +- Ensure repository has Actions enabled + +### Service not building +- Check Dockerfile exists in service directory +- Verify Root Directory is set correctly (`services/api` or `services/ui`) +- Check build logs in Railway dashboard + +## Railway CLI (Optional) + +You can also deploy using Railway CLI: + +```bash +# Install Railway CLI +npm i -g @railway/cli + +# Login +railway login + +# Link project +railway link + +# Deploy +railway up +``` + +## Cost Considerations + +- **Railway Free Tier**: $5 credit per month + - Perfect for testing and small deployments + - May need to upgrade for production traffic +- **GitHub Actions**: Free tier includes 2,000 minutes/month + - Worker runs every 5 minutes = ~8,640 runs/month + - Each run takes ~1-2 minutes = ~17,280 minutes/month + - May need to optimize or reduce frequency for free tier + +## Updating Services + +Railway auto-deploys on git push to your main branch. To update: + +1. Make changes to your code +2. Commit and push to main branch +3. Railway will automatically rebuild and deploy + +You can also manually trigger deployments from Railway dashboard. + diff --git a/README.md b/README.md index 08ad039a..e6a7d64d 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,85 @@ -# Crypto Risk Dashboard — Free Cloud Deployment +# Crypto Risk Dashboard — Cloud Deployment -**Deployment**: Render (API + UI) + GitHub Actions (Worker) +**Deployment**: Railway (API + UI) + GitHub Actions (Worker) -## Quick Deploy: +## Quick Deploy with Railway: -1. **Deploy to Render**: - - Connect your GitHub repo to Render - - Use the `render.yaml` blueprint - - Render will automatically create API, UI, and PostgreSQL +### Step 1: Create Railway Project and Database -2. **Setup GitHub Actions Worker**: - - Get `DATABASE_URL` from Render dashboard - - Add to GitHub Secrets: `DATABASE_URL = your_render_postgres_url` - - Push the code - worker runs automatically every 5 minutes +1. **Sign up/Login to Railway**: Go to [railway.app](https://railway.app) and connect your GitHub account -3. **Access Your Dashboard**: - - UI: `https://crypto-risk-ui.onrender.com` - - API: `https://crypto-risk-api.onrender.com` +2. **Create New Project**: + - Click "New Project" + - Select "Empty Project" + +3. **Add PostgreSQL Database**: + - Click "+ New" → "Database" → "Add PostgreSQL" + - Railway will automatically create a PostgreSQL database + - Note the database - it will be used in the next steps + +### Step 2: Deploy API Service + +1. **Add API Service**: + - In your Railway project, click "+ New" → "GitHub Repo" + - Select this repository + - Select the repository and connect it + +2. **Configure API Service**: + - In the service settings: + - **Root Directory**: Leave empty or set to repository root (Railway will auto-detect Dockerfile) + - Or set **Root Directory**: `services/api` (Railway uses repo root as build context) + - Railway will automatically detect the Dockerfile in `services/api/Dockerfile` + +3. **Add Environment Variables**: + - Click on "Variables" tab + - Add `DATABASE_URL` → Reference from PostgreSQL service (Railway will auto-generate this) + - Add `SYMBOLS` → `binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT` + - Railway will automatically expose the service and generate a URL + +4. **Generate Public URL**: + - In API service settings → "Networking" → "Generate Domain" + - Note your API URL (e.g., `https://crypto-risk-api-production.up.railway.app`) + +### Step 3: Deploy UI Service + +1. **Add UI Service**: + - Click "+ New" → "GitHub Repo" (same repository) + - Or duplicate the API service and modify settings + +2. **Configure UI Service**: + - **Root Directory**: Leave empty or set to repository root + - Or set **Root Directory**: `services/ui` (Railway uses repo root as build context) + - Railway will automatically detect the Dockerfile in `services/ui/Dockerfile` + +3. **Add Environment Variables**: + - `API_CANDIDATES` → Your API URL from Step 2 (e.g., `https://crypto-risk-api-production.up.railway.app,http://api:8000,http://localhost:8000`) + - `DATABASE_URL` → Reference from PostgreSQL service + +4. **Generate Public URL**: + - Generate a domain for UI service (e.g., `https://crypto-risk-ui-production.up.railway.app`) + +### Step 4: Setup GitHub Actions Worker + +1. **Get Database Connection String**: + - In Railway project → PostgreSQL database → "Variables" tab + - Copy the `DATABASE_URL` value (format: `postgresql://user:pass@host:port/dbname`) + +2. **Configure GitHub Secrets**: + - Go to your GitHub repository → Settings → Secrets and variables → Actions + - Click "New repository secret" + - Add: `DATABASE_URL` = your Railway PostgreSQL connection string + - The worker will run automatically every 5 minutes via GitHub Actions + +3. **Verify Worker**: + - Go to Actions tab in GitHub + - Check "Crypto Data Worker" workflow runs every 5 minutes + - You can manually trigger it via "workflow_dispatch" + +### Step 5: Access Your Dashboard + +- **UI**: Your Railway UI service URL +- **API**: Your Railway API service URL +- Both services will auto-deploy on git push to main branch ## Local Development: ```bash diff --git a/docker-compose.yml b/docker-compose.yml index 1a6db525..1e125767 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,9 @@ services: retries: 20 api: - build: ./services/api + build: + context: . + dockerfile: services/api/Dockerfile env_file: .env depends_on: db: @@ -32,7 +34,9 @@ services: condition: service_healthy ui: - build: ./services/ui + build: + context: . + dockerfile: services/ui/Dockerfile env_file: .env depends_on: api: diff --git a/railway.json b/railway.json new file mode 100644 index 00000000..a13df2fa --- /dev/null +++ b/railway.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://railway.app/railway.schema.json", + "build": { + "builder": "NIXPACKS" + }, + "deploy": { + "startCommand": "uvicorn main:app --host 0.0.0.0 --port $PORT", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 10 + } +} + diff --git a/railway.toml b/railway.toml new file mode 100644 index 00000000..6f86e95e --- /dev/null +++ b/railway.toml @@ -0,0 +1,9 @@ +# Railway configuration for Crypto Risk Dashboard +# This file defines services for Railway deployment + +[build] +builder = "DOCKERFILE" + +# Service configuration is typically done via Railway dashboard +# or through railway.json in each service directory + diff --git a/services/api/Dockerfile b/services/api/Dockerfile index f8fda52b..53c791f3 100644 --- a/services/api/Dockerfile +++ b/services/api/Dockerfile @@ -1,6 +1,15 @@ FROM python:3.11-slim WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] + +# Copy common module (services_common needs this) +# Note: Railway uses repo root as build context when Root Directory is set +COPY services/common /app/services/common + +# Copy API service files +COPY services/api/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY services/api /app + +# Use PORT env var (Railway) or default to 8000 +CMD sh -c "uvicorn main:app --host 0.0.0.0 --port ${PORT:-8000}" diff --git a/services/api/railway.json b/services/api/railway.json new file mode 100644 index 00000000..a5cec656 --- /dev/null +++ b/services/api/railway.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://railway.app/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "services/api/Dockerfile" + }, + "deploy": { + "startCommand": "uvicorn main:app --host 0.0.0.0 --port $PORT", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 10 + } +} + diff --git a/services/ui/Dockerfile b/services/ui/Dockerfile index 0cdd3f3a..f9078f8a 100644 --- a/services/ui/Dockerfile +++ b/services/ui/Dockerfile @@ -1,6 +1,15 @@ FROM python:3.11-slim WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] + +# Copy common module (if UI needs it) +# Note: Railway uses repo root as build context when Root Directory is set +COPY services/common /app/services/common + +# Copy UI service files +COPY services/ui/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY services/ui /app + +# Use PORT env var (Railway) or default to 8501 +CMD sh -c "streamlit run app.py --server.port=${PORT:-8501} --server.address=0.0.0.0" diff --git a/services/ui/app.py b/services/ui/app.py index bbc58b2c..e349dfc3 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -3,6 +3,7 @@ from urllib.parse import urlencode DEFAULT_CANDIDATES = [ + "https://crypto-risk-api-production.up.railway.app", "https://crypto-risk-api.onrender.com", "http://api:8000", "http://localhost:8000", @@ -38,7 +39,7 @@ def resolve_api_base(): if not API_BASE: st.error("Could not locate the API automatically.") - manual = st.text_input("Enter your API base URL (e.g., https://crypto-risk-api.onrender.com)") + manual = st.text_input("Enter your API base URL (e.g., https://crypto-risk-api-production.up.railway.app)") if manual and probe_api(manual): API_BASE = manual st.success("Connected!") diff --git a/services/ui/railway.json b/services/ui/railway.json new file mode 100644 index 00000000..c062f872 --- /dev/null +++ b/services/ui/railway.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://railway.app/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "services/ui/Dockerfile" + }, + "deploy": { + "startCommand": "streamlit run app.py --server.port=$PORT --server.address=0.0.0.0", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 10 + } +} + From 44c6393ff6125ec630b9cad894f2216c2a4f22dd Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 23:30:51 +0100 Subject: [PATCH 14/43] Add crypto data worker GitHub Actions workflow --- .github/workflows/crypto-worker.yml | 52 +++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/crypto-worker.yml diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml new file mode 100644 index 00000000..31b6a084 --- /dev/null +++ b/.github/workflows/crypto-worker.yml @@ -0,0 +1,52 @@ +name: Crypto Data Worker + +on: + schedule: + - cron: '*/5 * * * *' # Every 5 minutes + workflow_dispatch: + +jobs: + worker: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Cache Python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/common/requirements.txt', 'services/api/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r services/worker/requirements.txt + pip install -r services/common/requirements.txt + pip install -r services/api/requirements.txt + + - name: Run Data Worker + env: + # DATABASE_URL should be set from Railway PostgreSQL service + # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL + # Add it to GitHub Secrets as DATABASE_URL + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + SCHEDULE_MINUTES: 5 + run: python services/worker/run_worker.py + + # Optional Slack notification on failure + - name: Notify Slack on failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: failure + fields: repo,message,commit,author,workflow + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From 3045ecb293285e63a068c07e739013e54d34c65d Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 23:45:12 +0100 Subject: [PATCH 15/43] v --- .github/workflows/crypto-worker.yml | 4 ++-- services/common/requirements.txt | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 services/common/requirements.txt diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 31b6a084..ab086148 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -41,9 +41,9 @@ jobs: SCHEDULE_MINUTES: 5 run: python services/worker/run_worker.py - # Optional Slack notification on failure + # Optional Slack notification on failure (only runs if SLACK_WEBHOOK_URL secret is set) - name: Notify Slack on failure - if: failure() + if: failure() && secrets.SLACK_WEBHOOK_URL != '' uses: 8398a7/action-slack@v3 with: status: failure diff --git a/services/common/requirements.txt b/services/common/requirements.txt new file mode 100644 index 00000000..e1365d15 --- /dev/null +++ b/services/common/requirements.txt @@ -0,0 +1,8 @@ +# Core dependencies for services/common module +psycopg2-binary==2.9.9 +pandas==2.2.3 +numpy==2.1.2 +requests==2.32.3 +ccxt==4.4.6 +python-dotenv==1.0.1 + From 279ff30f0677950f3844255771a4502d73ca5407 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 23:48:43 +0100 Subject: [PATCH 16/43] Remove Slack failure notification from workflow Deleted the optional Slack notification step on job failure from the crypto-worker GitHub Actions workflow. This simplifies the workflow and removes dependency on the SLACK_WEBHOOK_URL secret. --- .github/workflows/crypto-worker.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index ab086148..5f92cc97 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -40,13 +40,3 @@ jobs: SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT SCHEDULE_MINUTES: 5 run: python services/worker/run_worker.py - - # Optional Slack notification on failure (only runs if SLACK_WEBHOOK_URL secret is set) - - name: Notify Slack on failure - if: failure() && secrets.SLACK_WEBHOOK_URL != '' - uses: 8398a7/action-slack@v3 - with: - status: failure - fields: repo,message,commit,author,workflow - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From 8c4d5e30e24551e364c8249a690742f81d867524 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 23:52:41 +0100 Subject: [PATCH 17/43] Remove Slack notification on failure Removed optional Slack notification step on failure. --- .github/workflows/crypto-worker.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 31b6a084..5f92cc97 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -40,13 +40,3 @@ jobs: SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT SCHEDULE_MINUTES: 5 run: python services/worker/run_worker.py - - # Optional Slack notification on failure - - name: Notify Slack on failure - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: failure - fields: repo,message,commit,author,workflow - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} From efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 Mon Sep 17 00:00:00 2001 From: skymike Date: Fri, 31 Oct 2025 23:58:43 +0100 Subject: [PATCH 18/43] Modify caching and dependency installation in workflow Updated cache key for Python packages and removed installation of common requirements. --- .github/workflows/crypto-worker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 5f92cc97..8a75a554 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -16,7 +16,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/common/requirements.txt', 'services/api/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/api/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- @@ -28,7 +28,6 @@ jobs: - name: Install dependencies run: | pip install -r services/worker/requirements.txt - pip install -r services/common/requirements.txt pip install -r services/api/requirements.txt - name: Run Data Worker From 5a77d22d679190de5a0a1acd0f65f984b87d85f0 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 00:05:01 +0100 Subject: [PATCH 19/43] Refactor common module imports to services_common Replaces usage of 'services.common' with a new 'services_common' namespace package for easier imports. Updates worker script to use repo root for sys.path and removes redundant dependency installation from workflow. Adds services_common/__init__.py to expose common modules under the new namespace. --- .github/workflows/crypto-worker.yml | 3 +-- services/worker/run_worker.py | 12 +++++++-- services_common/__init__.py | 38 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 services_common/__init__.py diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 5f92cc97..8a75a554 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -16,7 +16,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/common/requirements.txt', 'services/api/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/api/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- @@ -28,7 +28,6 @@ jobs: - name: Install dependencies run: | pip install -r services/worker/requirements.txt - pip install -r services/common/requirements.txt pip install -r services/api/requirements.txt - name: Run Data Worker diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 69d3e521..d831c5aa 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -1,7 +1,15 @@ import os import sys -# Add the common module to path -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) +from pathlib import Path + +# Add the repo root to path so services_common imports work +# This file is at services/worker/run_worker.py +# So we need to go up 2 levels to get to repo root +repo_root = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(repo_root)) + +# Import the services_common namespace package +import services_common from services_common.db import ensure_schema from services_common.ingest import run_ingest_cycle diff --git a/services_common/__init__.py b/services_common/__init__.py new file mode 100644 index 00000000..2c219abe --- /dev/null +++ b/services_common/__init__.py @@ -0,0 +1,38 @@ +""" +services_common namespace package +This makes services/common importable as services_common +""" +import sys +from pathlib import Path +import importlib + +# Ensure repo root is in path +_repo_root = Path(__file__).resolve().parent +if str(_repo_root) not in sys.path: + sys.path.insert(0, str(_repo_root)) + +# Import services.common modules and make them available as services_common.* +_common_modules = [ + 'config', 'db', 'ingest', 'signals', 'schema' +] + +for mod_name in _common_modules: + try: + mod = importlib.import_module(f'services.common.{mod_name}') + sys.modules[f'services_common.{mod_name}'] = mod + except ImportError: + pass + +# Handle adapters package +try: + adapters = importlib.import_module('services.common.adapters') + sys.modules['services_common.adapters'] = adapters + for adapter in ['exchanges', 'open_interest', 'volatility', 'sentiment', 'headlines']: + try: + mod = importlib.import_module(f'services.common.adapters.{adapter}') + sys.modules[f'services_common.adapters.{adapter}'] = mod + except ImportError: + pass +except ImportError: + pass + From 2dc0d538f999b2fca7d30f3fbe92479e655c7d02 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 00:06:42 +0100 Subject: [PATCH 20/43] Improve services_common import handling and logging Added package docstrings to services and services.common. Updated services_common/__init__.py to eagerly import modules and adapters, log import errors for better debugging, and handle exceptions more robustly. Clarified import order in run_worker.py to ensure correct namespace initialization. --- services/__init__.py | 2 ++ services/common/__init__.py | 2 ++ services/worker/run_worker.py | 4 +++- services_common/__init__.py | 15 +++++++++------ 4 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 services/__init__.py diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 00000000..6d31e90a --- /dev/null +++ b/services/__init__.py @@ -0,0 +1,2 @@ +# Services package + diff --git a/services/common/__init__.py b/services/common/__init__.py index e69de29b..851daabf 100644 --- a/services/common/__init__.py +++ b/services/common/__init__.py @@ -0,0 +1,2 @@ +# services.common package + diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index d831c5aa..444f069e 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -8,9 +8,11 @@ repo_root = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(repo_root)) -# Import the services_common namespace package +# Import the services_common namespace package FIRST +# This must be imported before any services_common.* imports import services_common +# Now we can import from services_common from services_common.db import ensure_schema from services_common.ingest import run_ingest_cycle from services_common.signals import compute_all_signals diff --git a/services_common/__init__.py b/services_common/__init__.py index 2c219abe..b56a9485 100644 --- a/services_common/__init__.py +++ b/services_common/__init__.py @@ -12,6 +12,7 @@ sys.path.insert(0, str(_repo_root)) # Import services.common modules and make them available as services_common.* +# Do this eagerly so modules are available immediately _common_modules = [ 'config', 'db', 'ingest', 'signals', 'schema' ] @@ -19,9 +20,11 @@ for mod_name in _common_modules: try: mod = importlib.import_module(f'services.common.{mod_name}') + # Register in sys.modules for direct import access sys.modules[f'services_common.{mod_name}'] = mod - except ImportError: - pass + except Exception as e: + # Log the error but continue - this helps with debugging + print(f"Warning: Could not import services.common.{mod_name}: {e}", file=sys.stderr) # Handle adapters package try: @@ -31,8 +34,8 @@ try: mod = importlib.import_module(f'services.common.adapters.{adapter}') sys.modules[f'services_common.adapters.{adapter}'] = mod - except ImportError: - pass -except ImportError: - pass + except Exception as e: + print(f"Warning: Could not import services.common.adapters.{adapter}: {e}", file=sys.stderr) +except Exception as e: + print(f"Warning: Could not import services.common.adapters: {e}", file=sys.stderr) From ad2827de4868de0781f72e273a3aacc1cfad08e7 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 00:10:48 +0100 Subject: [PATCH 21/43] Update imports to use services.common namespace Changed all imports from 'services_common' to 'services.common' for consistency and reliability. Updated worker script to remove unnecessary namespace package import and use direct imports from the new namespace. --- services/common/db.py | 4 ++-- services/common/ingest.py | 14 +++++++------- services/common/signals.py | 2 +- services/worker/run_worker.py | 14 +++++--------- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/services/common/db.py b/services/common/db.py index 545405b8..d51e56fb 100644 --- a/services/common/db.py +++ b/services/common/db.py @@ -1,6 +1,6 @@ import os, psycopg2, pandas as pd from psycopg2.extras import execute_values -from services_common.config import load_config +from services.common.config import load_config _cfg = load_config() @@ -37,7 +37,7 @@ def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_c execute_values(cur, sql, vals) def ensure_schema(): - from services_common.schema import SCHEMA_SQL + from services.common.schema import SCHEMA_SQL execute(SCHEMA_SQL) _try_apply_timescale() diff --git a/services/common/ingest.py b/services/common/ingest.py index 60b7230e..5d38aee4 100644 --- a/services/common/ingest.py +++ b/services/common/ingest.py @@ -1,11 +1,11 @@ import pandas as pd -from services_common.config import load_config -from services_common.db import upsert_many -from services_common.adapters.exchanges import fetch_candles -from services_common.adapters.open_interest import fetch_open_interest, fetch_funding_rate -from services_common.adapters.volatility import compute_atr_like -from services_common.adapters.sentiment import fetch_sentiment -from services_common.adapters.headlines import fetch_headlines +from services.common.config import load_config +from services.common.db import upsert_many +from services.common.adapters.exchanges import fetch_candles +from services.common.adapters.open_interest import fetch_open_interest, fetch_funding_rate +from services.common.adapters.volatility import compute_atr_like +from services.common.adapters.sentiment import fetch_sentiment +from services.common.adapters.headlines import fetch_headlines cfg = load_config() diff --git a/services/common/signals.py b/services/common/signals.py index c186dc28..1ea94736 100644 --- a/services/common/signals.py +++ b/services/common/signals.py @@ -1,5 +1,5 @@ import pandas as pd -from services_common.db import fetch_df, upsert_many +from services.common.db import fetch_df, upsert_many def _percentile(series: pd.Series, value: float): if series.empty: diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 444f069e..5c8496b7 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -2,20 +2,16 @@ import sys from pathlib import Path -# Add the repo root to path so services_common imports work +# Add the repo root to path so services.common can be imported # This file is at services/worker/run_worker.py # So we need to go up 2 levels to get to repo root repo_root = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(repo_root)) -# Import the services_common namespace package FIRST -# This must be imported before any services_common.* imports -import services_common - -# Now we can import from services_common -from services_common.db import ensure_schema -from services_common.ingest import run_ingest_cycle -from services_common.signals import compute_all_signals +# Directly import from services.common (simpler and more reliable) +from services.common.db import ensure_schema +from services.common.ingest import run_ingest_cycle +from services.common.signals import compute_all_signals def main(): print("[worker] Starting worker cycle...") From 981e4ebb2d4255cf5b7a8cea2ed0b886e6d5d01a Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 00:17:07 +0100 Subject: [PATCH 22/43] Add debug prints and explicit services imports Added debug statements to verify repo and services paths. Explicitly imported the services package before importing submodules to ensure proper package recognition. --- services/worker/run_worker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 5c8496b7..24164a86 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -8,7 +8,17 @@ repo_root = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(repo_root)) -# Directly import from services.common (simpler and more reliable) +# Debug: Verify the path is correct +print(f"[DEBUG] Repo root: {repo_root}", file=sys.stderr) +print(f"[DEBUG] Services path exists: {(repo_root / 'services').exists()}", file=sys.stderr) +print(f"[DEBUG] Services/common path exists: {(repo_root / 'services' / 'common').exists()}", file=sys.stderr) + +# Ensure services package can be imported +# Import services first to make sure the package is recognized +import services +import services.common + +# Now import the specific modules we need from services.common.db import ensure_schema from services.common.ingest import run_ingest_cycle from services.common.signals import compute_all_signals From bdf720fa2e26cc25c98d7d833a5abdf89d545a92 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 00:19:34 +0100 Subject: [PATCH 23/43] Improve import handling for services.common modules Adds the common directory to sys.path and updates debug output to clarify import paths. Imports from services.common are now wrapped in a try/except block to provide better error visibility during import failures. --- services/worker/run_worker.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 24164a86..0a83a0fd 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -6,22 +6,28 @@ # This file is at services/worker/run_worker.py # So we need to go up 2 levels to get to repo root repo_root = Path(__file__).resolve().parent.parent.parent +common_dir = repo_root / "services" / "common" + +# Add repo root and common directory to path sys.path.insert(0, str(repo_root)) +sys.path.insert(0, str(common_dir)) # Debug: Verify the path is correct print(f"[DEBUG] Repo root: {repo_root}", file=sys.stderr) -print(f"[DEBUG] Services path exists: {(repo_root / 'services').exists()}", file=sys.stderr) -print(f"[DEBUG] Services/common path exists: {(repo_root / 'services' / 'common').exists()}", file=sys.stderr) - -# Ensure services package can be imported -# Import services first to make sure the package is recognized -import services -import services.common +print(f"[DEBUG] Common dir: {common_dir}", file=sys.stderr) +print(f"[DEBUG] Common dir exists: {common_dir.exists()}", file=sys.stderr) +print(f"[DEBUG] PYTHONPATH: {sys.path[:3]}", file=sys.stderr) -# Now import the specific modules we need -from services.common.db import ensure_schema -from services.common.ingest import run_ingest_cycle -from services.common.signals import compute_all_signals +# Import using absolute path from repo root +try: + from services.common.db import ensure_schema + from services.common.ingest import run_ingest_cycle + from services.common.signals import compute_all_signals + print("[DEBUG] Successfully imported from services.common", file=sys.stderr) +except ImportError as e: + print(f"[DEBUG] Import error: {e}", file=sys.stderr) + # Re-raise the error so we can see what's actually failing + raise def main(): print("[worker] Starting worker cycle...") From 80d7394abff94ab7fcc09de0ef7f2cf1c2204a6e Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 00:32:45 +0100 Subject: [PATCH 24/43] Remove merge conflict markers from requirements.txt Cleaned up unresolved merge conflict markers in requirements.txt to ensure proper dependency management and prevent installation errors. --- services/worker/requirements.txt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/services/worker/requirements.txt b/services/worker/requirements.txt index f57fa1fb..9d2300aa 100644 --- a/services/worker/requirements.txt +++ b/services/worker/requirements.txt @@ -19,11 +19,3 @@ requests==2.32.3 beautifulsoup4==4.12.3 streamlit==1.39.0 streamlit_echarts==0.4.0 -<<<<<<< HEAD -psycopg2-binary==2.9.9 -pandas==2.1.4 -requests==2.31.0 -ccxt==4.1.60 -python-dotenv==1.0.0 -======= ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 From 70a1ee742ef542d50efc845dcccea07ffce1df8e Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 02:53:35 +0100 Subject: [PATCH 25/43] 2 --- .github/workflows/{crypto-worker.yml => crypto-worker.ymls} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{crypto-worker.yml => crypto-worker.ymls} (100%) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.ymls similarity index 100% rename from .github/workflows/crypto-worker.yml rename to .github/workflows/crypto-worker.ymls From 20dbee5ee0f902c7890c424d5f15dd83b447e550 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 02:54:31 +0100 Subject: [PATCH 26/43] Delete .github/workflows/crypto-worker.yml --- .github/workflows/crypto-worker.yml | 41 ----------------------------- 1 file changed, 41 deletions(-) delete mode 100644 .github/workflows/crypto-worker.yml diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml deleted file mode 100644 index 8a75a554..00000000 --- a/.github/workflows/crypto-worker.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Crypto Data Worker - -on: - schedule: - - cron: '*/5 * * * *' # Every 5 minutes - workflow_dispatch: - -jobs: - worker: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Cache Python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/api/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - pip install -r services/worker/requirements.txt - pip install -r services/api/requirements.txt - - - name: Run Data Worker - env: - # DATABASE_URL should be set from Railway PostgreSQL service - # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL - # Add it to GitHub Secrets as DATABASE_URL - DATABASE_URL: ${{ secrets.DATABASE_URL }} - SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT - SCHEDULE_MINUTES: 5 - run: python services/worker/run_worker.py From 3081d799d2db620e65b9a2a24d9836bc0bc1d216 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 02:56:59 +0100 Subject: [PATCH 27/43] Add crypto data worker workflow --- .github/workflows/crypto-worker.yml | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/crypto-worker.yml diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml new file mode 100644 index 00000000..8a75a554 --- /dev/null +++ b/.github/workflows/crypto-worker.yml @@ -0,0 +1,41 @@ +name: Crypto Data Worker + +on: + schedule: + - cron: '*/5 * * * *' # Every 5 minutes + workflow_dispatch: + +jobs: + worker: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Cache Python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/api/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r services/worker/requirements.txt + pip install -r services/api/requirements.txt + + - name: Run Data Worker + env: + # DATABASE_URL should be set from Railway PostgreSQL service + # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL + # Add it to GitHub Secrets as DATABASE_URL + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + SCHEDULE_MINUTES: 5 + run: python services/worker/run_worker.py From eba28b921a13b9e52842c3986d286277866262b9 Mon Sep 17 00:00:00 2001 From: skymike Date: Sat, 1 Nov 2025 03:16:59 +0100 Subject: [PATCH 28/43] Refactor UI and sentiment adapter, fix merge artifacts Resolved merge conflict markers and refactored both the Streamlit UI and sentiment adapter for clarity and maintainability. Improved API base resolution, data fetching, and signal display in the UI. Enhanced sentiment analysis logic and typing in the adapter. Renamed workflow file for consistency. --- .../{crypto-worker.ymls => crypto-worker.yml} | 0 docker-compose.yml | 6 - services/common/adapters/sentiment.py | 151 ++++------------ services/ui/app.py | 171 +++++++++--------- 4 files changed, 116 insertions(+), 212 deletions(-) rename .github/workflows/{crypto-worker.ymls => crypto-worker.yml} (100%) diff --git a/.github/workflows/crypto-worker.ymls b/.github/workflows/crypto-worker.yml similarity index 100% rename from .github/workflows/crypto-worker.ymls rename to .github/workflows/crypto-worker.yml diff --git a/docker-compose.yml b/docker-compose.yml index 3a786bd4..57fdccaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,15 +16,12 @@ services: interval: 5s timeout: 5s retries: 20 -<<<<<<< HEAD -======= restart: always deploy: resources: limits: memory: 512M cpus: "0.5" ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 api: build: @@ -55,8 +52,6 @@ services: depends_on: db: condition: service_healthy -<<<<<<< HEAD -======= restart: on-failure user: "1000:1000" deploy: @@ -64,7 +59,6 @@ services: limits: memory: 1024M cpus: "1.0" ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 ui: build: diff --git a/services/common/adapters/sentiment.py b/services/common/adapters/sentiment.py index d666f8a1..1fcde58a 100644 --- a/services/common/adapters/sentiment.py +++ b/services/common/adapters/sentiment.py @@ -1,16 +1,20 @@ import datetime as dt import requests import random -<<<<<<< HEAD -from typing import List, Dict +from typing import List, Dict, Optional + +KEYWORDS = [ + "liquidation", "margin call", "rekt", "funding", "open interest", "crash", "rally" +] -KEYWORDS = ["liquidation", "margin call", "rekt", "funding", "open interest", "crash", "rally"] +POSITIVE_WORDS = ["rally", "surge", "bull", "up", "green", "gain"] +NEGATIVE_WORDS = ["crash", "drop", "bear", "down", "red", "loss", "liquidat"] -def fetch_sentiment_cryptopanic(api_key: str = None) -> List[Dict]: - """Fetch real sentiment from CryptoPanic (free tier)""" +def fetch_sentiment_cryptopanic(api_key: Optional[str] = None) -> List[Dict]: + """Fetch sentiment from CryptoPanic (free tier) with fallback to mock.""" if not api_key: return fetch_sentiment_mock("global") - + try: url = "https://cryptopanic.com/api/v1/posts/" params = { @@ -21,149 +25,60 @@ def fetch_sentiment_cryptopanic(api_key: str = None) -> List[Dict]: } response = requests.get(url, params=params, timeout=10) data = response.json() - + posts = data.get('results', []) mentions = len(posts) - - # Simple sentiment analysis based on post titles - positive_words = ['rally', 'surge', 'bull', 'up', 'green', 'gain'] - negative_words = ['crash', 'drop', 'bear', 'down', 'red', 'loss', 'liquidat'] - + score = 0 keyword_counts = {k: 0 for k in KEYWORDS} - + for post in posts: title = post.get('title', '').lower() - # Count keywords for keyword in KEYWORDS: if keyword in title: keyword_counts[keyword] += 1 - # Calculate sentiment - if any(word in title for word in positive_words): + if any(word in title for word in POSITIVE_WORDS): score += 1 - if any(word in title for word in negative_words): + if any(word in title for word in NEGATIVE_WORDS): score -= 1 - - # Normalize score - if mentions > 0: - score_norm = score / mentions - else: - score_norm = 0 - + + score_norm = score / mentions if mentions > 0 else 0 + now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) return [{ - "pair": "global", - "ts": now, - "mentions": mentions, - "score_norm": score_norm, + "pair": "global", + "ts": now, + "mentions": mentions, + "score_norm": score_norm, "keywords": keyword_counts }] - + except Exception as e: print(f"CryptoPanic error: {e}") return fetch_sentiment_mock("global") -def fetch_sentiment_mock(exchange_pair: str): - """Fallback mock sentiment""" +def fetch_sentiment_mock(exchange_pair: str) -> List[Dict]: + """Fallback mock sentiment.""" now = dt.datetime.now(dt.timezone.utc).replace(second=0, microsecond=0) mentions = random.randint(5, 50) score = random.uniform(-1, 1) - kw_counts = {k: random.randint(0, mentions//2) for k in KEYWORDS} + kw_counts = {k: random.randint(0, max(1, mentions//2)) for k in KEYWORDS} return [{ - "pair": exchange_pair, - "ts": now, - "mentions": mentions, - "score_norm": score, + "pair": exchange_pair, + "ts": now, + "mentions": mentions, + "score_norm": score, "keywords": kw_counts }] -def fetch_sentiment(exchange_pair: str, api_key: str = None) -> List[Dict]: - """Main sentiment function with real data fallback""" +def fetch_sentiment(exchange_pair: str, api_key: Optional[str] = None) -> List[Dict]: + """Main sentiment function with real data fallback.""" try: - # Try real data first real_data = fetch_sentiment_cryptopanic(api_key) if real_data: - # Convert global sentiment to pair-specific real_data[0]["pair"] = exchange_pair return real_data except Exception as e: print(f"Real sentiment failed, using mock: {e}") - - return fetch_sentiment_mock(exchange_pair) -======= -import logging -from typing import List, Dict, Optional -from functools import lru_cache - -# Externalize keywords to lists -KEYWORDS = [ - "liquidation", "margin call", "rekt", "funding", "open interest", - "crash", "rally" -] - -POSITIVE_WORDS = ["rally", "surge", "bull", "up", "green", "gain"] -NEGATIVE_WORDS = ["crash", "drop", "bear", "down", "red", "loss", "liquidat"] -@lru_cache(maxsize=128) -def fetch_sentiment_cryptopanic(api_key: Optional[str] = None) -> List[Dict]: - """Fetch real sentiment from CryptoPanic API, fallback to mock data.""" - if not api_key: - return fetch_sentiment_mock("global") - - try: - url = "https://cryptopanic.com/api/v1/posts/" - params = { - 'auth_token': api_key, - 'public': 'true', - 'kind': 'news', - 'filter': 'important' - } - response = requests.get(url, params=params, timeout=10) - response.raise_for_status() - data = response.json() - posts = data.get('results', []) - - mentions = len(posts) - score = 0 - keyword_counts = {k: 0 for k in KEYWORDS} - - for post in posts: - title = post.get('title', '').lower() - # Count keywords - for keyword in KEYWORDS: - if keyword in title: - keyword_counts[keyword] += 1 - # Simple sentiment scoring - for word in POSITIVE_WORDS: - if word in title: - score += 1 - for word in NEGATIVE_WORDS: - if word in title: - score -= 1 - - normalized_score = score / max(mentions, 1) # Avoid division by zero - - return [{ - "timestamp": dt.datetime.utcnow().isoformat(), - "mentions": mentions, - "sentiment_score": normalized_score, - "keyword_counts": keyword_counts - }] - - except Exception as e: - logging.error(f"Error fetching sentiment from CryptoPanic: {e}") - return fetch_sentiment_mock("global") - -def fetch_sentiment_mock(pair: str) -> List[Dict]: - """Generate mock sentiment data.""" - mentions = random.randint(0, 20) - sentiment_score = random.uniform(-1, 1) - keyword_counts = {k: random.randint(0, 5) for k in KEYWORDS} - - return [{ - "timestamp": dt.datetime.utcnow().isoformat(), - "mentions": mentions, - "sentiment_score": sentiment_score, - "keyword_counts": keyword_counts - }] ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 + return fetch_sentiment_mock(exchange_pair) diff --git a/services/ui/app.py b/services/ui/app.py index e4266b56..9151e4d2 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -1,36 +1,26 @@ -<<<<<<< HEAD -import os, requests, pandas as pd -======= import os import requests import pandas as pd ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 import streamlit as st from urllib.parse import urlencode from typing import Optional DEFAULT_CANDIDATES = [ -<<<<<<< HEAD "https://crypto-risk-api-production.up.railway.app", -======= ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 "https://crypto-risk-api.onrender.com", "http://api:8000", "http://localhost:8000", ] -def probe_api(base: str, timeout=2.0) -> bool: +def probe_api(base: str, timeout: float = 2.0) -> bool: try: r = requests.get(f"{base}/health", timeout=timeout) return r.ok and r.json().get("ok") is True except Exception: return False -<<<<<<< HEAD -def resolve_api_base(): -======= def resolve_api_base() -> Optional[str]: ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 + """Resolve API base URL from environment or known candidates.""" env_base = os.getenv("API_BASE", "").strip() if env_base and probe_api(env_base): return env_base @@ -39,12 +29,8 @@ def resolve_api_base() -> Optional[str]: seen, merged = set(), [] for x in (candidates + DEFAULT_CANDIDATES): if x not in seen: -<<<<<<< HEAD - merged.append(x); seen.add(x) -======= merged.append(x) seen.add(x) ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 for base in merged: if probe_api(base): return base @@ -53,9 +39,8 @@ def resolve_api_base() -> Optional[str]: API_BASE = resolve_api_base() st.set_page_config(page_title="Crypto Risk Dashboard", layout="wide") -st.title("🧭 Crypto Risk Dashboard") -<<<<<<< HEAD -st.caption("Self-hosted. Graphs • Meters • Hot Signals") +st.title("Crypto Risk Dashboard") +st.caption("Graphs, Meters, and Hot Signals") if not API_BASE: st.error("Could not locate the API automatically.") @@ -63,68 +48,63 @@ def resolve_api_base() -> Optional[str]: if manual and probe_api(manual): API_BASE = manual st.success("Connected!") -======= ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 if not API_BASE: st.error("Could not connect to API backend. Please ensure the API is running and reachable.") st.stop() -<<<<<<< HEAD -resp = requests.get(f"{API_BASE}/pairs", timeout=30).json() -pairs = resp.get("pairs", ["binance:BTC/USDT"]) -======= @st.cache_data(ttl=300) -def fetch(endpoint: str, params: Optional[dict] = None) -> Optional[pd.DataFrame]: - url = f"{API_BASE}/{endpoint}" +def fetch_pairs() -> list[str]: try: - response = requests.get(url, params=params, timeout=10) - response.raise_for_status() - data = response.json() - if "data" in data and isinstance(data["data"], list): - return pd.DataFrame(data["data"]) - else: - st.warning(f"No data found for endpoint: {endpoint}") + resp = requests.get(f"{API_BASE}/pairs", timeout=15) + resp.raise_for_status() + data = resp.json() + return data.get("pairs", ["binance:BTC/USDT"]) + except Exception as e: + st.error(f"Error fetching pairs: {e}") + return ["binance:BTC/USDT"] + +@st.cache_data(ttl=300) +def fetch_timeseries(metric: str, pair: str, limit: int = 500) -> Optional[pd.DataFrame]: + qs = urlencode({"pair": pair, "limit": limit}) + url = f"{API_BASE}/timeseries/{metric}?{qs}" + try: + r = requests.get(url, timeout=20) + r.raise_for_status() + payload = r.json() + rows = payload.get("rows", []) + if not rows: return None + df = pd.DataFrame(rows) + if "ts" in df.columns: + df["ts"] = pd.to_datetime(df["ts"], utc=True) + df = df.set_index("ts") + return df except Exception as e: - st.error(f"Error fetching {endpoint}: {e}") + st.error(f"Error fetching timeseries {metric}: {e}") return None ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 + +@st.cache_data(ttl=120) +def fetch_signals(pairs: list[str]) -> dict: + try: + qs = "?pairs=" + ",".join(pairs) if pairs else "" + r = requests.get(f"{API_BASE}/signals{qs}", timeout=20) + r.raise_for_status() + return r.json() + except Exception as e: + st.error(f"Error fetching signals: {e}") + return {} # Refresh button to clear cache if st.button("Refresh Data"): - fetch.clear() - -<<<<<<< HEAD -def fetch(metric, pair, limit=500): - qs = urlencode({"pair": pair, "limit": limit}) - r = requests.get(f"{API_BASE}/timeseries/{metric}?{qs}", timeout=30) - return r.json() + fetch_pairs.clear() + fetch_timeseries.clear() + fetch_signals.clear() -def get_signals(pairs=None): - qs = "?pairs=" + ",".join(pairs) if pairs else "" - r = requests.get(f"{API_BASE}/signals{qs}", timeout=30) - return r.json() - -sig = get_signals([pair]) -signals = sig.get("signals", {}) -explanations = sig.get("explanations", {}) - -st.subheader("Hot Signals") -if signals.get(pair): - s = signals[pair] - c1, c2, c3, c4 = st.columns(4) - with c1: st.metric("Market Regime", s.get("regime","—")) - with c2: st.metric("Bias", s.get("bias","—")) - with c3: st.metric("Long Prob. %", round(100*s.get("long_prob",0),1)) - with c4: st.metric("Short Prob. %", round(100*s.get("short_prob",0),1)) - st.info(s.get("summary","")) -======= -pairs_df = fetch("pairs") +pairs = fetch_pairs() pair = None -if pairs_df is not None and not pairs_df.empty: - pair = st.selectbox("Select trading pair", pairs_df["pair"].unique()) ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 +if pairs: + pair = st.selectbox("Select trading pair", pairs, index=0) else: st.warning("No trading pairs available from API.") @@ -137,53 +117,68 @@ def get_signals(pairs=None): st.subheader(f"Data for Pair: {pair}") -data_tabs = st.tabs(["Candles", "Funding Rates", "Open Interest", "Volatility", "Sentiment", "Signals"]) +sig_payload = fetch_signals([pair]) or {} +signals_map = sig_payload.get("signals", {}) +explanations = sig_payload.get("explanations", {}) + +st.subheader("Hot Signals") +if pair in signals_map: + s = signals_map[pair] + c1, c2, c3, c4 = st.columns(4) + with c1: + st.metric("Market Regime", s.get("regime", "Unknown")) + with c2: + st.metric("Bias", s.get("bias", "Neutral")) + with c3: + st.metric("Long Prob. %", round(100 * float(s.get("long_prob", 0)), 1)) + with c4: + st.metric("Short Prob. %", round(100 * float(s.get("short_prob", 0)), 1)) + if s.get("summary"): + st.info(s["summary"]) +else: + st.write("No signal for selected pair yet.") + +data_tabs = st.tabs(["Candles", "Funding", "Open Interest", "Volatility", "Sentiment"]) with data_tabs[0]: - candles = fetch("candles", params={"pair": pair, "limit": limit}) + candles = fetch_timeseries("candles", pair=pair, limit=limit) if candles is not None and not candles.empty: - st.line_chart(candles.set_index("ts")[["open", "high", "low", "close"]]) + st.line_chart(candles[["open", "high", "low", "close"]]) else: st.write("No candles data available.") with data_tabs[1]: - funding = fetch("funding", params={"pair": pair, "limit": limit}) + funding = fetch_timeseries("funding", pair=pair, limit=limit) if funding is not None and not funding.empty: - st.line_chart(funding.set_index("ts")["funding_rate"]) + col = "rate" if "rate" in funding.columns else funding.columns[-1] + st.line_chart(funding[col]) else: st.write("No funding rate data available.") with data_tabs[2]: - oi = fetch("open_interest", params={"pair": pair, "limit": limit}) + oi = fetch_timeseries("oi", pair=pair, limit=limit) if oi is not None and not oi.empty: - st.line_chart(oi.set_index("ts")["open_interest"]) + col = "value_usd" if "value_usd" in oi.columns else oi.columns[-1] + st.line_chart(oi[col]) else: st.write("No open interest data available.") with data_tabs[3]: - vol = fetch("volatility", params={"pair": pair, "limit": limit}) + vol = fetch_timeseries("vol", pair=pair, limit=limit) if vol is not None and not vol.empty: - st.line_chart(vol.set_index("ts")["atr"]) + col = "atr" if "atr" in vol.columns else vol.columns[-1] + st.line_chart(vol[col]) else: st.write("No volatility data available.") with data_tabs[4]: - sentiment = fetch("sentiment", params={"pair": pair, "limit": limit}) + sentiment = fetch_timeseries("sentiment", pair=pair, limit=limit) if sentiment is not None and not sentiment.empty: - st.line_chart(sentiment.set_index("ts")["sentiment_score"]) + col = "score_norm" if "score_norm" in sentiment.columns else sentiment.columns[-1] + st.line_chart(sentiment[col]) else: st.write("No sentiment data available.") -<<<<<<< HEAD st.divider() -st.caption("Configure pairs & scheduler in `.env`. Add API keys for live data.") -======= -with data_tabs[5]: - signals = fetch("signals", params={"pair": pair}) - if signals is not None and not signals.empty: - st.write(signals) - else: - st.write("No signals data available.") +st.caption("Configure pairs & scheduler in .env. Add API keys for live data.") -st.caption("Configure pairs, scheduler, and API keys in your .env file or Render.com environment settings.") ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 From f953f3497dc0fa3eb72bd366795dcc023bb96feb Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 18:34:57 +0100 Subject: [PATCH 29/43] Clean up schema_timescale.sql by removing merge conflict markers Resolved merge conflict artifacts in schema_timescale.sql, ensuring proper SQL syntax and clarity. The file now correctly enables the TimescaleDB extension, creates hypertables, adds necessary indexes, and sets up a continuous aggregate with an auto-refresh policy. --- services/common/schema_timescale.sql | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/services/common/schema_timescale.sql b/services/common/schema_timescale.sql index fdf9507b..1b770a9b 100644 --- a/services/common/schema_timescale.sql +++ b/services/common/schema_timescale.sql @@ -1,14 +1,7 @@ -<<<<<<< HEAD --- Enable Timescale if available -CREATE EXTENSION IF NOT EXISTS timescaledb; - --- Convert base tables to hypertables -======= -- Enable TimescaleDB extension if available CREATE EXTENSION IF NOT EXISTS timescaledb; -- Convert base tables to hypertables if not already ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 SELECT create_hypertable('candles', 'ts', if_not_exists => TRUE); SELECT create_hypertable('funding_rates', 'ts', if_not_exists => TRUE); SELECT create_hypertable('open_interest', 'ts', if_not_exists => TRUE); @@ -16,9 +9,6 @@ SELECT create_hypertable('volatility', 'ts', if_not_exists => TRUE); SELECT create_hypertable('sentiment', 'ts', if_not_exists => TRUE); SELECT create_hypertable('signals', 'ts', if_not_exists => TRUE); -<<<<<<< HEAD --- Example continuous aggregate for faster OHLC queries -======= -- Add indexes for faster filtering by 'pair' CREATE INDEX IF NOT EXISTS idx_candles_pair_ts ON candles (pair, ts DESC); CREATE INDEX IF NOT EXISTS idx_funding_rates_pair_ts ON funding_rates (pair, ts DESC); @@ -28,7 +18,6 @@ CREATE INDEX IF NOT EXISTS idx_sentiment_pair_ts ON sentiment (pair, ts DESC); CREATE INDEX IF NOT EXISTS idx_signals_pair_ts ON signals (pair, ts DESC); -- Create continuous aggregate for candles 1-hour data with auto-refresh ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 CREATE MATERIALIZED VIEW IF NOT EXISTS candles_1h WITH (timescaledb.continuous) AS SELECT @@ -42,11 +31,7 @@ SELECT FROM candles GROUP BY 1, 2; -<<<<<<< HEAD --- Auto-refresh policy -======= -- Add continuous aggregate refresh policy for candles_1h ->>>>>>> efba0ff1a8135bf80abb9945756f7a5ac17dfcb8 SELECT add_continuous_aggregate_policy( 'candles_1h', start_offset => INTERVAL '3 days', From bc29ec379fe66306ea54154064ce4005a1ba9938 Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 18:35:25 +0100 Subject: [PATCH 30/43] Update railway.json to switch builder to DOCKERFILE and specify dockerfilePath and context Changed the build configuration in railway.json to use DOCKERFILE as the builder, added the path to the Dockerfile, and set the context for the build process. This update enhances the deployment setup for the application. --- railway.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/railway.json b/railway.json index a13df2fa..7a1c29b0 100644 --- a/railway.json +++ b/railway.json @@ -1,10 +1,11 @@ { "$schema": "https://railway.app/railway.schema.json", "build": { - "builder": "NIXPACKS" + "builder": "DOCKERFILE", + "dockerfilePath": "services/api/Dockerfile", + "context": "." }, "deploy": { - "startCommand": "uvicorn main:app --host 0.0.0.0 --port $PORT", "restartPolicyType": "ON_FAILURE", "restartPolicyMaxRetries": 10 } From d8681cb949008f291b3b98ad947e8a2b1b11cb18 Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 21:18:35 +0100 Subject: [PATCH 31/43] Add UI enhancements and live market snapshot feature Implemented minimal CSS tweaks for a modern look in the Crypto Risk Dashboard. Added a live market snapshot feature that fetches and displays real-time cryptocurrency prices and changes. Updated the refresh button to clear the market snapshot cache. Improved the extraction of base symbols from trading pairs for better data handling. --- services/ui/app.py | 150 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/services/ui/app.py b/services/ui/app.py index 9151e4d2..ccf3b7f3 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -39,6 +39,64 @@ def resolve_api_base() -> Optional[str]: API_BASE = resolve_api_base() st.set_page_config(page_title="Crypto Risk Dashboard", layout="wide") + +# Minimal CSS tweaks for a softer, modern look +st.markdown( + """ + + """, + unsafe_allow_html=True, +) + st.title("Crypto Risk Dashboard") st.caption("Graphs, Meters, and Hot Signals") @@ -95,11 +153,71 @@ def fetch_signals(pairs: list[str]) -> dict: st.error(f"Error fetching signals: {e}") return {} + +COINGECKO_IDS = { + "BTC": "bitcoin", + "ETH": "ethereum", + "SOL": "solana", + "BNB": "binancecoin", + "XRP": "ripple", + "ADA": "cardano", + "DOGE": "dogecoin", + "AVAX": "avalanche-2", + "DOT": "polkadot", + "LINK": "chainlink", +} + + +def extract_base_symbols(pairs: list[str]) -> list[str]: + bases = [] + for raw in pairs: + try: + base = raw.split(":", 1)[1] if ":" in raw else raw + base = base.split("/", 1)[0] + bases.append(base.upper()) + except Exception: + continue + return list(dict.fromkeys(bases)) + + +@st.cache_data(ttl=60) +def fetch_market_snapshot(pairs: list[str]) -> dict[str, dict]: + symbols = extract_base_symbols(pairs) + ids = [COINGECKO_IDS[s] for s in symbols if s in COINGECKO_IDS] + if not ids: + return {} + url = "https://api.coingecko.com/api/v3/simple/price" + params = { + "ids": ",".join(ids), + "vs_currencies": "usd", + "include_24hr_change": "true", + "include_last_updated_at": "true", + } + try: + resp = requests.get(url, params=params, timeout=10) + resp.raise_for_status() + payload = resp.json() + simplified = {} + for symbol, cg_id in COINGECKO_IDS.items(): + if cg_id in payload: + entry = payload[cg_id] + simplified[symbol] = { + "price": entry.get("usd"), + "change": entry.get("usd_24h_change"), + "updated": entry.get("last_updated_at"), + } + return simplified + except Exception as exc: + st.warning(f"Unable to fetch live market snapshot: {exc}") + return {} + + # Refresh button to clear cache -if st.button("Refresh Data"): +if st.button("Refresh Data", use_container_width=True): fetch_pairs.clear() fetch_timeseries.clear() fetch_signals.clear() + fetch_market_snapshot.clear() pairs = fetch_pairs() pair = None @@ -121,6 +239,35 @@ def fetch_signals(pairs: list[str]) -> dict: signals_map = sig_payload.get("signals", {}) explanations = sig_payload.get("explanations", {}) +market_snapshot = fetch_market_snapshot(pairs) +if market_snapshot: + st.subheader("Live Market Snapshot") + cards = st.columns(min(4, len(market_snapshot))) + idx = 0 + for symbol, data in market_snapshot.items(): + col = cards[idx % len(cards)] + idx += 1 + with col: + price = data.get("price") + change = data.get("change") + delta = f"{change:+.2f}%" if change is not None else "n/a" + price_str = f"{price:,.2f} USD" if price is not None else "n/a" + color = "#22c55e" if change is not None and change >= 0 else "#ef4444" + st.markdown( + f""" +
+
{symbol}
+
+ {price_str} +
+
+ 24h: {delta} +
+
+ """, + unsafe_allow_html=True, + ) + st.subheader("Hot Signals") if pair in signals_map: s = signals_map[pair] @@ -181,4 +328,3 @@ def fetch_signals(pairs: list[str]) -> dict: st.divider() st.caption("Configure pairs & scheduler in .env. Add API keys for live data.") - From 7f598d7349797a53ca982b6ce545d09f4ea69455 Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 21:47:57 +0100 Subject: [PATCH 32/43] Add manual ingest endpoint and enhance database operations Introduced a new POST endpoint for manual data ingestion in the API, allowing background processing of data updates. Enhanced the upsert functionality in the database to handle JSON data types and improved conflict resolution strategies. Updated the UI to include a button for triggering manual ingests, providing users with better control over data updates. --- services/api/main.py | 22 +++++- services/common/db.py | 16 +++- services/common/signals.py | 9 ++- services/ui/app.py | 149 +++++++++++++++++++++++++++++++++++-- 4 files changed, 182 insertions(+), 14 deletions(-) diff --git a/services/api/main.py b/services/api/main.py index ce6a6500..1a6a3d91 100644 --- a/services/api/main.py +++ b/services/api/main.py @@ -1,6 +1,11 @@ -from fastapi import FastAPI, Query -from services_common.db import fetch_df -from services_common.signals import latest_signals_for_pairs, signal_explanations +from fastapi import FastAPI, Query, BackgroundTasks +from services_common.db import fetch_df, ensure_schema +from services_common.ingest import run_ingest_cycle +from services_common.signals import ( + latest_signals_for_pairs, + signal_explanations, + compute_all_signals, +) from services_common.config import load_config app = FastAPI(title="Crypto Risk API", version="0.2.0") @@ -20,6 +25,17 @@ def get_signals(pairs: str | None = Query(None)): data = latest_signals_for_pairs(pairs_list) return {"signals": data, "explanations": signal_explanations()} +def _run_manual_cycle(): + ensure_schema() + run_ingest_cycle() + compute_all_signals() + +@app.post("/ingest") +def manual_ingest(background_tasks: BackgroundTasks): + """Trigger a best-effort ingest cycle in the background.""" + background_tasks.add_task(_run_manual_cycle) + return {"status": "queued", "message": "Manual ingest started in background."} + @app.get("/timeseries/{metric}") def timeseries(metric: str, pair: str, limit: int = 500): table_map = { diff --git a/services/common/db.py b/services/common/db.py index d51e56fb..6c94140e 100644 --- a/services/common/db.py +++ b/services/common/db.py @@ -1,5 +1,5 @@ import os, psycopg2, pandas as pd -from psycopg2.extras import execute_values +from psycopg2.extras import execute_values, Json from services.common.config import load_config _cfg = load_config() @@ -24,14 +24,22 @@ def fetch_df(sql, params=None) -> pd.DataFrame: def upsert_many(table: str, rows: list[dict], conflict_cols: list[str], update_cols: list[str]): if not rows: return + def _coerce(value): + if isinstance(value, (dict, list)): + return Json(value) + return value cols = list(rows[0].keys()) - vals = [[r[c] for c in cols] for r in rows] + vals = [[_coerce(r[c]) for c in cols] for r in rows] on_conflict = ", ".join(conflict_cols) - updates = ", ".join([f"{c}=EXCLUDED.{c}" for c in update_cols]) + if update_cols: + updates = ", ".join([f"{c}=EXCLUDED.{c}" for c in update_cols]) + conflict_clause = f"ON CONFLICT ({on_conflict}) DO UPDATE SET {updates}" + else: + conflict_clause = f"ON CONFLICT ({on_conflict}) DO NOTHING" sql = f""" INSERT INTO {table} ({",".join(cols)}) VALUES %s - ON CONFLICT ({on_conflict}) DO UPDATE SET {updates} + {conflict_clause} """ with _conn() as conn, conn.cursor() as cur: execute_values(cur, sql, vals) diff --git a/services/common/signals.py b/services/common/signals.py index 1ea94736..6db2afc7 100644 --- a/services/common/signals.py +++ b/services/common/signals.py @@ -1,5 +1,5 @@ import pandas as pd -from services.common.db import fetch_df, upsert_many +from services.common.db import fetch_df, upsert_many, execute def _percentile(series: pd.Series, value: float): if series.empty: @@ -95,7 +95,12 @@ def compute_all_signals(): "long_prob": s["long_prob"], "short_prob": s["short_prob"], "summary": s["summary"] } for s in out] if rows: - upsert_many("signals", rows, ["id"], []) + sql = """ + INSERT INTO signals (ts, pair, regime, bias, long_prob, short_prob, summary) + VALUES (%(ts)s, %(pair)s, %(regime)s, %(bias)s, %(long_prob)s, %(short_prob)s, %(summary)s) + """ + for row in rows: + execute(sql, row) def latest_signals_for_pairs(pairs: list[str]): placeholders = ",".join(["%s"]*len(pairs)) diff --git a/services/ui/app.py b/services/ui/app.py index ccf3b7f3..53268d14 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -153,6 +153,17 @@ def fetch_signals(pairs: list[str]) -> dict: st.error(f"Error fetching signals: {e}") return {} +def trigger_manual_ingest() -> tuple[bool, str]: + """Call the API endpoint to trigger a one-off ingest cycle.""" + try: + resp = requests.post(f"{API_BASE}/ingest", timeout=10) + resp.raise_for_status() + payload = resp.json() if resp.headers.get("content-type","").startswith("application/json") else {} + message = payload.get("message") or "Manual ingest triggered. Give it a few seconds to populate." + return True, message + except Exception as exc: + return False, str(exc) + COINGECKO_IDS = { "BTC": "bitcoin", @@ -167,6 +178,19 @@ def fetch_signals(pairs: list[str]) -> dict: "LINK": "chainlink", } +COINCAP_IDS = { + "BTC": "bitcoin", + "ETH": "ethereum", + "SOL": "solana", + "BNB": "binance-coin", + "XRP": "ripple", + "ADA": "cardano", + "DOGE": "dogecoin", + "AVAX": "avalanche", + "DOT": "polkadot", + "LINK": "chainlink", +} + def extract_base_symbols(pairs: list[str]) -> list[str]: bases = [] @@ -211,13 +235,79 @@ def fetch_market_snapshot(pairs: list[str]) -> dict[str, dict]: st.warning(f"Unable to fetch live market snapshot: {exc}") return {} +@st.cache_data(ttl=600) +def fetch_fear_greed() -> Optional[dict]: + try: + resp = requests.get("https://api.alternative.me/fng/?limit=1", timeout=10) + resp.raise_for_status() + data = resp.json() + if not data.get("data"): + return None + entry = data["data"][0] + return { + "value": float(entry.get("value", 0)), + "classification": entry.get("value_classification", "n/a"), + "updated": entry.get("timestamp"), + } + except Exception: + return None + +@st.cache_data(ttl=180) +def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: + symbols = extract_base_symbols(pairs) + ids = [COINCAP_IDS[s] for s in symbols if s in COINCAP_IDS] + if not ids: + return {} + try: + resp = requests.get( + "https://api.coincap.io/v2/assets", + params={"ids": ",".join(ids)}, + timeout=10, + ) + resp.raise_for_status() + data = resp.json().get("data", []) + out: dict[str, dict] = {} + for asset in data: + symbol = asset.get("symbol") + if not symbol: + continue + try: + out[symbol.upper()] = { + "price": float(asset.get("priceUsd") or 0), + "volume": float(asset.get("volumeUsd24Hr") or 0), + "change": float(asset.get("changePercent24Hr") or 0), + "market_cap": float(asset.get("marketCapUsd") or 0), + } + except (TypeError, ValueError): + continue + return out + except Exception: + return {} # Refresh button to clear cache -if st.button("Refresh Data", use_container_width=True): - fetch_pairs.clear() - fetch_timeseries.clear() - fetch_signals.clear() - fetch_market_snapshot.clear() +controls = st.columns([1, 1, 3]) +with controls[0]: + if st.button("Refresh Data", use_container_width=True): + fetch_pairs.clear() + fetch_timeseries.clear() + fetch_signals.clear() + fetch_market_snapshot.clear() + fetch_fear_greed.clear() + fetch_asset_flows.clear() +with controls[1]: + if st.button("Manual Data Pull", use_container_width=True): + with st.spinner("Triggering worker ingest…"): + ok, msg = trigger_manual_ingest() + if ok: + fetch_pairs.clear() + fetch_timeseries.clear() + fetch_signals.clear() + fetch_market_snapshot.clear() + fetch_fear_greed.clear() + fetch_asset_flows.clear() + st.success(msg) + else: + st.error(f"Manual ingest failed: {msg}") pairs = fetch_pairs() pair = None @@ -268,6 +358,55 @@ def fetch_market_snapshot(pairs: list[str]) -> dict[str, dict]: unsafe_allow_html=True, ) +macro = st.columns(2) +with macro[0]: + fg = fetch_fear_greed() + if fg: + fg_color = "#22c55e" if fg["value"] >= 50 else "#f97316" if fg["value"] >= 25 else "#ef4444" + st.markdown( + f""" +
+
Fear & Greed Index
+
+ {fg["value"]:.0f} +
+
{fg["classification"]}
+
Source: alternative.me
+
+ """, + unsafe_allow_html=True, + ) + else: + st.info("Fear & Greed data is temporarily unavailable.") + +with macro[1]: + flows = fetch_asset_flows(pairs) + if flows: + sorted_rows = sorted(flows.items(), key=lambda kv: kv[1]["volume"], reverse=True) + top_rows = sorted_rows[:3] + items = [] + for symbol, info in top_rows: + vol = info["volume"] + change = info["change"] + price = info["price"] + items.append( + f""" +
+
{symbol}
+
{price:,.2f} USD
+
+ 24h Vol: {vol/1_000_000:,.1f}M • Change: {change:+.2f}% +
+
+ """ + ) + st.markdown( + "
" + "".join(items) + "
Source: coincap.io
", + unsafe_allow_html=True, + ) + else: + st.info("CoinCap asset metrics unavailable right now.") + st.subheader("Hot Signals") if pair in signals_map: s = signals_map[pair] From 586c1cbf3d47999af04e1a5434504ccacd257ad0 Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 21:57:20 +0100 Subject: [PATCH 33/43] Update deployment guide and configuration for default trading pairs Revised the RAILWAY_DEPLOY.md to clarify the default trading pairs for the SYMBOLS variable, now including the top 30 perpetual pairs. Updated the GitHub Actions workflow to reflect the new default trading pairs. Enhanced the config.py to load a comprehensive list of default symbols. Improved the UI to utilize cached time series data for better performance and added new analytics modules for user insights. --- .github/workflows/crypto-worker.yml | 2 +- RAILWAY_DEPLOY.md | 6 +-- services/common/config.py | 12 ++++- services/ui/app.py | 84 ++++++++++++++++++++++++++--- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 8a75a554..2148fa98 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -36,6 +36,6 @@ jobs: # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL # Add it to GitHub Secrets as DATABASE_URL DATABASE_URL: ${{ secrets.DATABASE_URL }} - SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT + SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,binance:SOL/USDT,binance:BNB/USDT,binance:XRP/USDT,binance:DOGE/USDT,binance:ADA/USDT,binance:AVAX/USDT,binance:TRX/USDT,binance:DOT/USDT,binance:LINK/USDT,binance:MATIC/USDT,binance:UNI/USDT,binance:APT/USDT,binance:ARB/USDT,binance:ATOM/USDT,binance:OP/USDT,binance:SEI/USDT,binance:NEAR/USDT,binance:INJ/USDT,bybit:BTC/USDT,bybit:ETH/USDT,bybit:SOL/USDT,bybit:XRP/USDT,bybit:DOGE/USDT,bybit:ADA/USDT,bybit:LINK/USDT,bybit:MATIC/USDT,bybit:NEAR/USDT,bybit:APT/USDT SCHEDULE_MINUTES: 5 run: python services/worker/run_worker.py diff --git a/RAILWAY_DEPLOY.md b/RAILWAY_DEPLOY.md index 778e91b9..841459d5 100644 --- a/RAILWAY_DEPLOY.md +++ b/RAILWAY_DEPLOY.md @@ -42,7 +42,7 @@ This guide provides detailed instructions for deploying the Crypto Risk Dashboar - Go to "Variables" tab - Click "New Variable" - `DATABASE_URL`: Click "Reference Variable" → Select your PostgreSQL service → Select `DATABASE_URL` - - `SYMBOLS`: `binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT` + - `SYMBOLS`: Default includes top 30 perp pairs (e.g., `binance:BTC/USDT,...,bybit:APT/USDT`). Override if you want a smaller slice. 5. **Generate Public URL**: - Go to "Settings" → "Networking" @@ -106,7 +106,7 @@ The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). ### API Service - `DATABASE_URL`: PostgreSQL connection string (referenced from database service) -- `SYMBOLS`: Comma-separated trading pairs (e.g., `binance:BTC/USDT,binance:ETH/USDT`) +- `SYMBOLS`: Comma-separated trading pairs (default top 30, e.g., `binance:BTC/USDT,...,bybit:APT/USDT`) - `PORT`: Automatically set by Railway (don't set manually) ### UI Service @@ -116,7 +116,7 @@ The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). ### Worker (GitHub Actions) - `DATABASE_URL`: PostgreSQL connection string (from GitHub Secrets) -- `SYMBOLS`: Trading pairs (set in workflow file) +- `SYMBOLS`: Trading pairs (set in workflow file; defaults to top 30) - `SCHEDULE_MINUTES`: Worker schedule interval (set in workflow file) ## Troubleshooting diff --git a/services/common/config.py b/services/common/config.py index 0f515f72..145d9d31 100644 --- a/services/common/config.py +++ b/services/common/config.py @@ -12,7 +12,17 @@ class Config: symbols: list[str] def load_config() -> Config: - symbols_env = os.getenv("SYMBOLS", "binance:BTC/USDT") + default_symbols = ( + "binance:BTC/USDT,binance:ETH/USDT,binance:SOL/USDT,binance:BNB/USDT," + "binance:XRP/USDT,binance:DOGE/USDT,binance:ADA/USDT,binance:AVAX/USDT," + "binance:TRX/USDT,binance:DOT/USDT,binance:LINK/USDT,binance:MATIC/USDT," + "binance:UNI/USDT,binance:APT/USDT,binance:ARB/USDT,binance:ATOM/USDT," + "binance:OP/USDT,binance:SEI/USDT,binance:NEAR/USDT,binance:INJ/USDT," + "bybit:BTC/USDT,bybit:ETH/USDT,bybit:SOL/USDT,bybit:XRP/USDT," + "bybit:DOGE/USDT,bybit:ADA/USDT,bybit:LINK/USDT,bybit:MATIC/USDT," + "bybit:NEAR/USDT,bybit:APT/USDT" + ) + symbols_env = os.getenv("SYMBOLS", default_symbols) symbols = [s.strip() for s in symbols_env.split(",") if s.strip()] return Config( pg_user=os.getenv("POSTGRES_USER","cryptouser"), diff --git a/services/ui/app.py b/services/ui/app.py index 53268d14..324b59ca 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -329,6 +329,15 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: signals_map = sig_payload.get("signals", {}) explanations = sig_payload.get("explanations", {}) +# Pre-load core time series once so analytics modules can reuse them +ts_cache: dict[str, Optional[pd.DataFrame]] = { + "candles": fetch_timeseries("candles", pair=pair, limit=limit), + "funding": fetch_timeseries("funding", pair=pair, limit=limit), + "oi": fetch_timeseries("oi", pair=pair, limit=limit), + "vol": fetch_timeseries("vol", pair=pair, limit=limit), + "sentiment": fetch_timeseries("sentiment", pair=pair, limit=limit), +} + market_snapshot = fetch_market_snapshot(pairs) if market_snapshot: st.subheader("Live Market Snapshot") @@ -386,7 +395,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: top_rows = sorted_rows[:3] items = [] for symbol, info in top_rows: - vol = info["volume"] + vol_usd = info["volume"] change = info["change"] price = info["price"] items.append( @@ -395,7 +404,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]:
{symbol}
{price:,.2f} USD
- 24h Vol: {vol/1_000_000:,.1f}M • Change: {change:+.2f}% + 24h Vol: {vol_usd/1_000_000:,.1f}M • Change: {change:+.2f}%
""" @@ -427,14 +436,14 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: data_tabs = st.tabs(["Candles", "Funding", "Open Interest", "Volatility", "Sentiment"]) with data_tabs[0]: - candles = fetch_timeseries("candles", pair=pair, limit=limit) + candles = ts_cache["candles"] if candles is not None and not candles.empty: st.line_chart(candles[["open", "high", "low", "close"]]) else: st.write("No candles data available.") with data_tabs[1]: - funding = fetch_timeseries("funding", pair=pair, limit=limit) + funding = ts_cache["funding"] if funding is not None and not funding.empty: col = "rate" if "rate" in funding.columns else funding.columns[-1] st.line_chart(funding[col]) @@ -442,7 +451,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: st.write("No funding rate data available.") with data_tabs[2]: - oi = fetch_timeseries("oi", pair=pair, limit=limit) + oi = ts_cache["oi"] if oi is not None and not oi.empty: col = "value_usd" if "value_usd" in oi.columns else oi.columns[-1] st.line_chart(oi[col]) @@ -450,7 +459,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: st.write("No open interest data available.") with data_tabs[3]: - vol = fetch_timeseries("vol", pair=pair, limit=limit) + vol = ts_cache["vol"] if vol is not None and not vol.empty: col = "atr" if "atr" in vol.columns else vol.columns[-1] st.line_chart(vol[col]) @@ -458,12 +467,73 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: st.write("No volatility data available.") with data_tabs[4]: - sentiment = fetch_timeseries("sentiment", pair=pair, limit=limit) + sentiment = ts_cache["sentiment"] if sentiment is not None and not sentiment.empty: col = "score_norm" if "score_norm" in sentiment.columns else sentiment.columns[-1] st.line_chart(sentiment[col]) else: st.write("No sentiment data available.") +st.subheader("Insight Modules") +module_options = { + "Funding vs Price Overlay": "overlay", + "Volatility Pulse (ATR stats)": "vol_stats", + "Sentiment Keyword Heatmap": "sentiment_keywords", +} +selected_modules = st.multiselect( + "Add-on analytics (select one or more)", + list(module_options.keys()), + default=["Funding vs Price Overlay"], +) + +for label in selected_modules: + module_key = module_options[label] + with st.container(): + st.markdown(f"#### {label}") + if module_key == "overlay": + candles = ts_cache["candles"] + funding = ts_cache["funding"] + if candles is None or candles.empty or funding is None or funding.empty: + st.info("Need both candles and funding data to build this view.") + else: + merged = candles[["close"]].join(funding["rate"], how="inner") + merged = merged.dropna() + if merged.empty: + st.info("Not enough overlapping data for overlay.") + else: + scaled = merged.copy() + scaled["close_norm"] = (scaled["close"] / scaled["close"].iloc[0]) * 100 + scaled["rate_bps"] = merged["rate"] * 10000 + st.line_chart(scaled[["close_norm", "rate_bps"]]) + st.caption("Price normalized to 100 (left axis) vs funding rate in basis points (right axis).") + elif module_key == "vol_stats": + vol = ts_cache["vol"] + if vol is None or vol.empty: + st.info("Volatility data not available yet.") + else: + latest = vol.iloc[-1] + series = vol["atr"] if "atr" in vol.columns else vol.iloc[:, 0] + stats = series.describe() + metric_value = float(latest.get("atr", series.iloc[-1])) + st.metric("Latest ATR", f"{metric_value:,.2f}") + summary = stats.to_frame(name="ATR").T + st.dataframe(summary) + st.caption("Descriptive statistics of ATR values for the selected window.") + elif module_key == "sentiment_keywords": + sentiment = ts_cache["sentiment"] + if sentiment is None or sentiment.empty or "keywords" not in sentiment.columns: + st.info("Sentiment keyword data is not available.") + else: + latest = sentiment.dropna(subset=["keywords"]).iloc[-1] + keywords = latest["keywords"] + if isinstance(keywords, dict) and keywords: + df_kw = pd.DataFrame( + {"keyword": list(keywords.keys()), "count": list(keywords.values())} + ).sort_values("count", ascending=False) + st.bar_chart(df_kw.set_index("keyword")) + else: + st.info("Latest sentiment entry does not include keyword counts.") + st.caption("Counts derived from latest CryptoPanic headlines scrape.") + st.divider() st.caption("Configure pairs & scheduler in .env. Add API keys for live data.") From 504a67356b930c4a80c3c3e280999aa69861483f Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 22:46:28 +0100 Subject: [PATCH 34/43] Enhance UI with new global market overview and Telegram alert options Updated the RAILWAY_DEPLOY.md to include optional Telegram bot configuration for alerts. Enhanced the Streamlit UI to display a global market overview with key metrics such as market cap, volume, and BTC dominance. Added new data fetching functions and improved visualizations for sentiment and funding analysis. Updated the layout for better user experience and added functionality for live liquidation feeds. --- RAILWAY_DEPLOY.md | 2 + services/common/notifications.py | 69 ++++++ services/ui/app.py | 380 ++++++++++++++++++++++--------- services/worker/run_worker.py | 5 +- services_common/__init__.py | 2 +- 5 files changed, 347 insertions(+), 111 deletions(-) create mode 100644 services/common/notifications.py diff --git a/RAILWAY_DEPLOY.md b/RAILWAY_DEPLOY.md index 841459d5..234aa720 100644 --- a/RAILWAY_DEPLOY.md +++ b/RAILWAY_DEPLOY.md @@ -118,6 +118,8 @@ The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). - `DATABASE_URL`: PostgreSQL connection string (from GitHub Secrets) - `SYMBOLS`: Trading pairs (set in workflow file; defaults to top 30) - `SCHEDULE_MINUTES`: Worker schedule interval (set in workflow file) +- `TELEGRAM_BOT_TOKEN`: *(optional)* Telegram bot token for alerting top signals +- `TELEGRAM_CHAT_ID`: *(optional)* Destination chat/channel ID for Telegram notifications ## Troubleshooting diff --git a/services/common/notifications.py b/services/common/notifications.py new file mode 100644 index 00000000..1e188d22 --- /dev/null +++ b/services/common/notifications.py @@ -0,0 +1,69 @@ +import os +from typing import Optional + +import requests +from services.common.db import fetch_df + + +def _telegram_credentials() -> Optional[tuple[str, str]]: + token = os.getenv("TELEGRAM_BOT_TOKEN") + chat_id = os.getenv("TELEGRAM_CHAT_ID") + if token and chat_id: + return token, chat_id + return None + + +def maybe_notify_top_signals(limit: int = 3) -> None: + creds = _telegram_credentials() + if not creds: + return + token, chat_id = creds + data = fetch_df( + """ + WITH ordered AS ( + SELECT + pair, + ts, + regime, + bias, + long_prob, + short_prob, + summary, + ROW_NUMBER() OVER (PARTITION BY pair ORDER BY ts DESC) AS rn + FROM signals + ) + SELECT pair, regime, bias, long_prob, short_prob, summary + FROM ordered + WHERE rn = 1 + ORDER BY GREATEST(long_prob, short_prob) DESC + LIMIT %(limit)s + """, + {"limit": limit}, + ) + if data.empty: + return + + lines = ["🔥 *Top Signal Update* 🔥"] + for _, row in data.iterrows(): + strength = max(float(row.get("long_prob", 0)), float(row.get("short_prob", 0))) + direction = "Long" if float(row.get("long_prob", 0)) >= float(row.get("short_prob", 0)) else "Short" + lines.append( + f"*{row['pair']}* → `{row['regime']}` · Bias: *{row['bias']}* " + f"· Top: *{direction}* ({strength*100:.1f}%)" + ) + summary = row.get("summary") + if isinstance(summary, str) and summary: + lines.append(f"_Summary_: {summary}") + lines.append("") + + message = "\n".join(lines).strip() + try: + resp = requests.post( + f"https://api.telegram.org/bot{token}/sendMessage", + json={"chat_id": chat_id, "text": message, "parse_mode": "Markdown"}, + timeout=10, + ) + resp.raise_for_status() + except Exception as exc: + # Fail silently so worker continues even if Telegram is misconfigured. + print(f"[notifications] Telegram send failed: {exc}") diff --git a/services/ui/app.py b/services/ui/app.py index 324b59ca..af0dffed 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -2,8 +2,10 @@ import requests import pandas as pd import streamlit as st +import streamlit.components.v1 as components from urllib.parse import urlencode from typing import Optional +import plotly.graph_objects as go DEFAULT_CANDIDATES = [ "https://crypto-risk-api-production.up.railway.app", @@ -63,16 +65,20 @@ def resolve_api_base() -> Optional[str]: box-shadow: 0 18px 35px rgba(15, 23, 42, 0.3); margin-bottom: 1.5rem; } - button[data-baseweb="button"] { + button[data-baseweb="button"], + div.stButton > button { border-radius: 999px !important; - border: none; + border: 1px solid rgba(99, 102, 241, 0.45); font-weight: 600; - background: linear-gradient(90deg, #6366f1, #8b5cf6); - color: white; - box-shadow: 0 12px 24px rgba(99, 102, 241, 0.35); + background: linear-gradient(135deg, #f8fafc 0%, #6366f1 50%, #312e81 100%); + color: #0f172a; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.5), 0 12px 24px rgba(79, 70, 229, 0.35); + text-shadow: none; } - button[data-baseweb="button"]:hover { - opacity: 0.88; + button[data-baseweb="button"]:hover, + div.stButton > button:hover { + filter: brightness(1.05); + box-shadow: inset 0 1px 0 rgba(255,255,255,0.7), 0 14px 26px rgba(79,70,229,0.45); } .stTabs [data-baseweb="tab"] { border-radius: 999px !important; @@ -233,7 +239,12 @@ def fetch_market_snapshot(pairs: list[str]) -> dict[str, dict]: return simplified except Exception as exc: st.warning(f"Unable to fetch live market snapshot: {exc}") - return {} + return {} + + +def build_aggr_trade_url(pair: str) -> str: + target = pair.split(":", 1)[-1].replace("/", "").upper() + return f"https://aggr.trade/?pair={target}" @st.cache_data(ttl=600) def fetch_fear_greed() -> Optional[dict]: @@ -284,6 +295,30 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: except Exception: return {} +@st.cache_data(ttl=600) +def fetch_alt_global() -> Optional[dict]: + try: + resp = requests.get("https://api.alternative.me/v2/global/?convert=USD", timeout=10) + resp.raise_for_status() + payload = resp.json() + data = payload.get("data") + if not isinstance(data, dict): + return None + quotes = data.get("quotes") or {} + usd = quotes.get("USD") if isinstance(quotes, dict) else {} + return { + "market_cap": float(usd.get("total_market_cap", 0) or 0), + "volume": float(usd.get("total_volume_24h", 0) or 0), + "market_cap_change": float( + usd.get("total_market_cap_yesterday_percentage_change", 0) or 0 + ), + "active_cryptos": int(data.get("active_cryptocurrencies", 0) or 0), + "active_markets": int(data.get("active_markets", 0) or 0), + "btc_dominance": float(data.get("bitcoin_percentage_of_market_cap", 0) or 0), + } + except Exception: + return None + # Refresh button to clear cache controls = st.columns([1, 1, 3]) with controls[0]: @@ -294,6 +329,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: fetch_market_snapshot.clear() fetch_fear_greed.clear() fetch_asset_flows.clear() + fetch_alt_global.clear() with controls[1]: if st.button("Manual Data Pull", use_container_width=True): with st.spinner("Triggering worker ingest…"): @@ -305,6 +341,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: fetch_market_snapshot.clear() fetch_fear_greed.clear() fetch_asset_flows.clear() + fetch_alt_global.clear() st.success(msg) else: st.error(f"Manual ingest failed: {msg}") @@ -312,7 +349,7 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: pairs = fetch_pairs() pair = None if pairs: - pair = st.selectbox("Select trading pair", pairs, index=0) + pair = st.selectbox("Select trading pair", pairs, index=0, format_func=lambda x: x.replace(":", " · ")) else: st.warning("No trading pairs available from API.") @@ -367,24 +404,38 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: unsafe_allow_html=True, ) -macro = st.columns(2) +macro = st.columns(3) with macro[0]: fg = fetch_fear_greed() if fg: - fg_color = "#22c55e" if fg["value"] >= 50 else "#f97316" if fg["value"] >= 25 else "#ef4444" - st.markdown( - f""" -
-
Fear & Greed Index
-
- {fg["value"]:.0f} -
-
{fg["classification"]}
-
Source: alternative.me
-
- """, - unsafe_allow_html=True, + fg_val = float(fg["value"]) + gauge = go.Figure( + go.Indicator( + mode="gauge+number", + value=fg_val, + number={"suffix": " / 100", "font": {"color": "#f1f5ff"}}, + title={"text": f"Fear & Greed · {fg['classification']}", "font": {"color": "#e0e7ff"}}, + gauge={ + "axis": {"range": [0, 100], "tickwidth": 1, "tickcolor": "#94a3b8"}, + "bar": {"color": "#a855f7"}, + "bgcolor": "#0f172a", + "borderwidth": 1, + "bordercolor": "#6366f1", + "steps": [ + {"range": [0, 25], "color": "#7f1d1d"}, + {"range": [25, 50], "color": "#b45309"}, + {"range": [50, 75], "color": "#1f2937"}, + {"range": [75, 100], "color": "#065f46"}, + ], + }, + ) ) + gauge.update_layout( + paper_bgcolor="rgba(15, 23, 42, 0.65)", + font={"color": "#cdd4ff"}, + margin=dict(l=10, r=10, t=50, b=10), + ) + st.plotly_chart(gauge, use_container_width=True) else: st.info("Fear & Greed data is temporarily unavailable.") @@ -416,6 +467,34 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: else: st.info("CoinCap asset metrics unavailable right now.") +with macro[2]: + global_data = fetch_alt_global() + if global_data: + mc = global_data["market_cap"] / 1_000_000_000 if global_data["market_cap"] else 0 + vol = global_data["volume"] / 1_000_000_000 if global_data["volume"] else 0 + dom = global_data["btc_dominance"] + change = global_data["market_cap_change"] + st.markdown( + f""" +
+
Global Market Overview
+
+ MC: {mc:,.1f}B · 24h Vol: {vol:,.1f}B +
+
+ BTC Dominance: {dom:.1f}% · Change: {change:+.2f}% +
+
+ Active Assets: {global_data['active_cryptos']} · Markets: {global_data['active_markets']} +
+
Source: alternative.me
+
+ """, + unsafe_allow_html=True, + ) + else: + st.info("Alternative.me global metrics unavailable right now.") + st.subheader("Hot Signals") if pair in signals_map: s = signals_map[pair] @@ -433,107 +512,190 @@ def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: else: st.write("No signal for selected pair yet.") -data_tabs = st.tabs(["Candles", "Funding", "Open Interest", "Volatility", "Sentiment"]) +visual_tabs = st.tabs(["Price Action", "Funding & OI", "Sentiment", "aggr.trade"]) -with data_tabs[0]: +with visual_tabs[0]: candles = ts_cache["candles"] if candles is not None and not candles.empty: - st.line_chart(candles[["open", "high", "low", "close"]]) + fig = go.Figure( + data=[ + go.Candlestick( + x=candles.index, + open=candles["open"], + high=candles["high"], + low=candles["low"], + close=candles["close"], + name="Price", + ) + ] + ) + fig.update_layout( + margin=dict(l=0, r=0, t=35, b=0), + template="plotly_dark", + title=f"{pair} · Candlestick", + ) + st.plotly_chart(fig, use_container_width=True) else: - st.write("No candles data available.") + st.info("No candles data available.") -with data_tabs[1]: +with visual_tabs[1]: funding = ts_cache["funding"] - if funding is not None and not funding.empty: + oi = ts_cache["oi"] + if funding is not None and not funding.empty and oi is not None and not oi.empty: col = "rate" if "rate" in funding.columns else funding.columns[-1] - st.line_chart(funding[col]) + oi_col = "value_usd" if "value_usd" in oi.columns else oi.columns[-1] + merged = funding[[col]].join(oi[[oi_col]], how="inner") + merged = merged.dropna() + if merged.empty: + st.info("Not enough overlapping funding/OI data.") + else: + fig = go.Figure() + fig.add_trace( + go.Bar( + x=merged.index, + y=merged[col] * 10000, + name="Funding (bps)", + marker_color="#22c55e", + opacity=0.6, + ) + ) + fig.add_trace( + go.Scatter( + x=merged.index, + y=merged[oi_col] / 1_000_000, + name="Open Interest (M USD)", + mode="lines", + line=dict(color="#38bdf8", width=2), + yaxis="y2", + ) + ) + fig.update_layout( + template="plotly_dark", + margin=dict(l=0, r=0, t=35, b=0), + yaxis=dict(title="Funding (bps)"), + yaxis2=dict(title="OI (Millions USD)", overlaying="y", side="right"), + title=f"{pair} · Funding vs Open Interest", + ) + st.plotly_chart(fig, use_container_width=True) else: - st.write("No funding rate data available.") + st.info("Funding or open interest data unavailable.") -with data_tabs[2]: - oi = ts_cache["oi"] - if oi is not None and not oi.empty: - col = "value_usd" if "value_usd" in oi.columns else oi.columns[-1] - st.line_chart(oi[col]) +with visual_tabs[2]: + sentiment = ts_cache["sentiment"] + if sentiment is not None and not sentiment.empty: + metric = "score_norm" if "score_norm" in sentiment.columns else sentiment.columns[-1] + fig = go.Figure() + fig.add_trace( + go.Scatter( + x=sentiment.index, + y=sentiment[metric], + name="Sentiment", + line=dict(color="#f97316"), + fill="tozeroy", + ) + ) + fig.update_layout( + template="plotly_dark", + margin=dict(l=0, r=0, t=35, b=0), + title=f"{pair} · Sentiment Trend", + ) + st.plotly_chart(fig, use_container_width=True) + else: + st.info("No sentiment data available.") + +with visual_tabs[3]: + st.write("Live liquidation feed via aggr.trade") + components.iframe(build_aggr_trade_url(pair), height=640, scrolling=True) + +st.subheader("Insight Modules") +analysis_tabs = st.tabs(["Funding Overlay", "Volatility Pulse", "Sentiment Radar"]) + +with analysis_tabs[0]: + candles = ts_cache["candles"] + funding = ts_cache["funding"] + if candles is None or candles.empty or funding is None or funding.empty: + st.info("Need both candle and funding data for overlay.") else: - st.write("No open interest data available.") + merged = candles[["close"]].join(funding["rate" if "rate" in funding.columns else funding.columns[-1]], how="inner") + merged = merged.dropna() + if merged.empty: + st.info("Not enough overlapping data for overlay.") + else: + price_norm = (merged["close"] / merged["close"].iloc[0]) * 100 + funding_bps = merged.iloc[:, 1] * 10000 + fig = go.Figure() + fig.add_trace( + go.Scatter(x=merged.index, y=price_norm, name="Price (norm 100)", line=dict(color="#a855f7")) + ) + fig.add_trace( + go.Scatter( + x=merged.index, + y=funding_bps, + name="Funding (bps)", + line=dict(color="#f59e0b", dash="dot"), + yaxis="y2", + ) + ) + fig.update_layout( + template="plotly_dark", + yaxis=dict(title="Price (index)"), + yaxis2=dict(title="Funding (bps)", overlaying="y", side="right"), + margin=dict(l=0, r=0, t=35, b=0), + ) + st.plotly_chart(fig, use_container_width=True) -with data_tabs[3]: +with analysis_tabs[1]: vol = ts_cache["vol"] - if vol is not None and not vol.empty: - col = "atr" if "atr" in vol.columns else vol.columns[-1] - st.line_chart(vol[col]) + if vol is None or vol.empty: + st.info("Volatility data not available yet.") else: - st.write("No volatility data available.") + series = vol["atr"] if "atr" in vol.columns else vol.iloc[:, 0] + metric_value = float(series.iloc[-1]) + fig = go.Figure( + go.Indicator( + mode="gauge+number", + value=metric_value, + title={"text": "ATR Snapshot"}, + gauge={ + "axis": {"range": [0, series.max() * 1.2 if series.max() else 1]}, + "bar": {"color": "#38bdf8"}, + "bgcolor": "#0f172a", + "steps": [ + {"range": [0, series.median()], "color": "#1e3a8a"}, + {"range": [series.median(), series.max()], "color": "#0f766e"}, + ], + }, + ) + ) + fig.update_layout(template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0)) + st.plotly_chart(fig, use_container_width=True) + stats = series.describe().to_frame(name="ATR").T + st.dataframe(stats) -with data_tabs[4]: +with analysis_tabs[2]: sentiment = ts_cache["sentiment"] - if sentiment is not None and not sentiment.empty: - col = "score_norm" if "score_norm" in sentiment.columns else sentiment.columns[-1] - st.line_chart(sentiment[col]) + if sentiment is None or sentiment.empty or "keywords" not in sentiment.columns: + st.info("Sentiment keyword data is not available.") else: - st.write("No sentiment data available.") - -st.subheader("Insight Modules") -module_options = { - "Funding vs Price Overlay": "overlay", - "Volatility Pulse (ATR stats)": "vol_stats", - "Sentiment Keyword Heatmap": "sentiment_keywords", -} -selected_modules = st.multiselect( - "Add-on analytics (select one or more)", - list(module_options.keys()), - default=["Funding vs Price Overlay"], -) - -for label in selected_modules: - module_key = module_options[label] - with st.container(): - st.markdown(f"#### {label}") - if module_key == "overlay": - candles = ts_cache["candles"] - funding = ts_cache["funding"] - if candles is None or candles.empty or funding is None or funding.empty: - st.info("Need both candles and funding data to build this view.") - else: - merged = candles[["close"]].join(funding["rate"], how="inner") - merged = merged.dropna() - if merged.empty: - st.info("Not enough overlapping data for overlay.") - else: - scaled = merged.copy() - scaled["close_norm"] = (scaled["close"] / scaled["close"].iloc[0]) * 100 - scaled["rate_bps"] = merged["rate"] * 10000 - st.line_chart(scaled[["close_norm", "rate_bps"]]) - st.caption("Price normalized to 100 (left axis) vs funding rate in basis points (right axis).") - elif module_key == "vol_stats": - vol = ts_cache["vol"] - if vol is None or vol.empty: - st.info("Volatility data not available yet.") - else: - latest = vol.iloc[-1] - series = vol["atr"] if "atr" in vol.columns else vol.iloc[:, 0] - stats = series.describe() - metric_value = float(latest.get("atr", series.iloc[-1])) - st.metric("Latest ATR", f"{metric_value:,.2f}") - summary = stats.to_frame(name="ATR").T - st.dataframe(summary) - st.caption("Descriptive statistics of ATR values for the selected window.") - elif module_key == "sentiment_keywords": - sentiment = ts_cache["sentiment"] - if sentiment is None or sentiment.empty or "keywords" not in sentiment.columns: - st.info("Sentiment keyword data is not available.") - else: - latest = sentiment.dropna(subset=["keywords"]).iloc[-1] - keywords = latest["keywords"] - if isinstance(keywords, dict) and keywords: - df_kw = pd.DataFrame( - {"keyword": list(keywords.keys()), "count": list(keywords.values())} - ).sort_values("count", ascending=False) - st.bar_chart(df_kw.set_index("keyword")) - else: - st.info("Latest sentiment entry does not include keyword counts.") - st.caption("Counts derived from latest CryptoPanic headlines scrape.") + latest = sentiment.dropna(subset=["keywords"]).iloc[-1] + keywords = latest["keywords"] + if isinstance(keywords, dict) and keywords: + df_kw = ( + pd.DataFrame({"keyword": list(keywords.keys()), "count": list(keywords.values())}) + .sort_values("count", ascending=False) + .head(15) + ) + fig = go.Figure( + go.Bar( + x=df_kw["keyword"], + y=df_kw["count"], + marker_color="#f472b6", + ) + ) + fig.update_layout(template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0)) + st.plotly_chart(fig, use_container_width=True) + else: + st.info("Latest sentiment entry does not include keyword counts.") st.divider() -st.caption("Configure pairs & scheduler in .env. Add API keys for live data.") +st.caption("Configure pairs, Telegram alerts, and scheduler in .env. Add API keys for live data.") diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 0a83a0fd..6ffa2705 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -23,6 +23,7 @@ from services.common.db import ensure_schema from services.common.ingest import run_ingest_cycle from services.common.signals import compute_all_signals + from services.common.notifications import maybe_notify_top_signals print("[DEBUG] Successfully imported from services.common", file=sys.stderr) except ImportError as e: print(f"[DEBUG] Import error: {e}", file=sys.stderr) @@ -36,7 +37,9 @@ def main(): run_ingest_cycle() print("[worker] Computing signals...") compute_all_signals() + print("[worker] Sending notifications (if configured)...") + maybe_notify_top_signals() print("[worker] Worker cycle completed successfully!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/services_common/__init__.py b/services_common/__init__.py index b56a9485..709b12ec 100644 --- a/services_common/__init__.py +++ b/services_common/__init__.py @@ -14,7 +14,7 @@ # Import services.common modules and make them available as services_common.* # Do this eagerly so modules are available immediately _common_modules = [ - 'config', 'db', 'ingest', 'signals', 'schema' + 'config', 'db', 'ingest', 'signals', 'schema', 'notifications' ] for mod_name in _common_modules: From 53fae88eb8891e667338d5c0930025c708d5d918 Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 23:10:52 +0100 Subject: [PATCH 35/43] Refactor configuration loading and enhance asset fetching logic Updated the configuration loading in config.py to handle default symbols more effectively, allowing for user-defined symbols while maintaining a comprehensive default list. Improved the asset fetching logic in app.py to include fallback mechanisms for missing data and optimized the parameters for API requests. Adjusted UI layout for better display of market data. --- services/common/config.py | 13 ++++++++++--- services/ui/app.py | 35 +++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/services/common/config.py b/services/common/config.py index 145d9d31..ac2c4c8f 100644 --- a/services/common/config.py +++ b/services/common/config.py @@ -12,7 +12,7 @@ class Config: symbols: list[str] def load_config() -> Config: - default_symbols = ( + default_symbols_str = ( "binance:BTC/USDT,binance:ETH/USDT,binance:SOL/USDT,binance:BNB/USDT," "binance:XRP/USDT,binance:DOGE/USDT,binance:ADA/USDT,binance:AVAX/USDT," "binance:TRX/USDT,binance:DOT/USDT,binance:LINK/USDT,binance:MATIC/USDT," @@ -22,8 +22,15 @@ def load_config() -> Config: "bybit:DOGE/USDT,bybit:ADA/USDT,bybit:LINK/USDT,bybit:MATIC/USDT," "bybit:NEAR/USDT,bybit:APT/USDT" ) - symbols_env = os.getenv("SYMBOLS", default_symbols) - symbols = [s.strip() for s in symbols_env.split(",") if s.strip()] + default_symbols = [s.strip() for s in default_symbols_str.split(",") if s.strip()] + symbols_env = os.getenv("SYMBOLS") + if symbols_env: + symbols = [s.strip() for s in symbols_env.split(",") if s.strip()] + if len(symbols) < len(default_symbols): + merged = symbols + default_symbols + symbols = list(dict.fromkeys(merged)) + else: + symbols = default_symbols return Config( pg_user=os.getenv("POSTGRES_USER","cryptouser"), pg_pass=os.getenv("POSTGRES_PASSWORD","cryptopass"), diff --git a/services/ui/app.py b/services/ui/app.py index af0dffed..65cdf686 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -193,8 +193,18 @@ def trigger_manual_ingest() -> tuple[bool, str]: "ADA": "cardano", "DOGE": "dogecoin", "AVAX": "avalanche", + "TRX": "tron", "DOT": "polkadot", "LINK": "chainlink", + "MATIC": "polygon", + "UNI": "uniswap", + "APT": "aptos", + "ARB": "arbitrum", + "ATOM": "cosmos", + "OP": "optimism", + "SEI": "sei-network", + "NEAR": "near", + "INJ": "injective-protocol", } @@ -267,16 +277,20 @@ def fetch_fear_greed() -> Optional[dict]: def fetch_asset_flows(pairs: list[str]) -> dict[str, dict]: symbols = extract_base_symbols(pairs) ids = [COINCAP_IDS[s] for s in symbols if s in COINCAP_IDS] - if not ids: - return {} try: - resp = requests.get( - "https://api.coincap.io/v2/assets", - params={"ids": ",".join(ids)}, - timeout=10, - ) + params = {"limit": 5} + if ids: + params = {"ids": ",".join(ids)} + resp = requests.get("https://api.coincap.io/v2/assets", params=params, timeout=10) resp.raise_for_status() data = resp.json().get("data", []) + if not data and not ids: + return {} + if not data and ids: + # fallback to global top assets if specific ids missing + resp = requests.get("https://api.coincap.io/v2/assets", params={"limit": 5}, timeout=10) + resp.raise_for_status() + data = resp.json().get("data", []) out: dict[str, dict] = {} for asset in data: symbol = asset.get("symbol") @@ -404,7 +418,7 @@ def fetch_alt_global() -> Optional[dict]: unsafe_allow_html=True, ) -macro = st.columns(3) +macro = st.columns([1.2, 1, 1]) with macro[0]: fg = fetch_fear_greed() if fg: @@ -433,9 +447,10 @@ def fetch_alt_global() -> Optional[dict]: gauge.update_layout( paper_bgcolor="rgba(15, 23, 42, 0.65)", font={"color": "#cdd4ff"}, - margin=dict(l=10, r=10, t=50, b=10), + height=260, + margin=dict(l=10, r=10, t=40, b=0), ) - st.plotly_chart(gauge, use_container_width=True) + st.plotly_chart(gauge, use_container_width=True, config={"displayModeBar": False}) else: st.info("Fear & Greed data is temporarily unavailable.") From 9411b825f630f62afc21097bf739b9d010d1d700 Mon Sep 17 00:00:00 2001 From: skymike Date: Sun, 2 Nov 2025 23:43:36 +0100 Subject: [PATCH 36/43] Update worker schedule to 15 minutes and introduce signal profile options Modified the GitHub Actions workflow to adjust the worker service schedule from every 5 minutes to every 15 minutes. Updated the configuration files to reflect this change and added a new optional `SIGNAL_PROFILE` variable for controlling signal strictness. Enhanced the API to support different profiles in signal computation, improving the flexibility of the signal generation process. Updated the UI to allow users to select their preferred signal profile, enhancing user experience and customization. --- .github/workflows/crypto-worker.yml | 4 +- RAILWAY_DEPLOY.md | 9 +- services/api/main.py | 20 +- services/common/signals.py | 224 +++++++++++++++------ services/ui/app.py | 298 ++++++++++++++++++++++++---- services/worker/run_worker.py | 2 +- timescale.env | 3 +- 7 files changed, 455 insertions(+), 105 deletions(-) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 2148fa98..6733c25d 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -2,7 +2,7 @@ name: Crypto Data Worker on: schedule: - - cron: '*/5 * * * *' # Every 5 minutes + - cron: '*/15 * * * *' # Every 15 minutes workflow_dispatch: jobs: @@ -37,5 +37,5 @@ jobs: # Add it to GitHub Secrets as DATABASE_URL DATABASE_URL: ${{ secrets.DATABASE_URL }} SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,binance:SOL/USDT,binance:BNB/USDT,binance:XRP/USDT,binance:DOGE/USDT,binance:ADA/USDT,binance:AVAX/USDT,binance:TRX/USDT,binance:DOT/USDT,binance:LINK/USDT,binance:MATIC/USDT,binance:UNI/USDT,binance:APT/USDT,binance:ARB/USDT,binance:ATOM/USDT,binance:OP/USDT,binance:SEI/USDT,binance:NEAR/USDT,binance:INJ/USDT,bybit:BTC/USDT,bybit:ETH/USDT,bybit:SOL/USDT,bybit:XRP/USDT,bybit:DOGE/USDT,bybit:ADA/USDT,bybit:LINK/USDT,bybit:MATIC/USDT,bybit:NEAR/USDT,bybit:APT/USDT - SCHEDULE_MINUTES: 5 + SCHEDULE_MINUTES: 15 run: python services/worker/run_worker.py diff --git a/RAILWAY_DEPLOY.md b/RAILWAY_DEPLOY.md index 234aa720..4685caac 100644 --- a/RAILWAY_DEPLOY.md +++ b/RAILWAY_DEPLOY.md @@ -11,7 +11,7 @@ This guide provides detailed instructions for deploying the Crypto Risk Dashboar ## Architecture - **Railway Services**: API, UI, and PostgreSQL database -- **GitHub Actions**: Worker service (runs every 5 minutes) +- **GitHub Actions**: Worker service (runs every 15 minutes) ## Deployment Steps @@ -84,7 +84,7 @@ The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). 3. **Verify Workflow**: - Go to Actions tab in GitHub - - The "Crypto Data Worker" workflow should run automatically every 5 minutes + - The "Crypto Data Worker" workflow should run automatically every 15 minutes - You can manually trigger it using "workflow_dispatch" ### 6. Verify Deployment @@ -120,6 +120,7 @@ The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). - `SCHEDULE_MINUTES`: Worker schedule interval (set in workflow file) - `TELEGRAM_BOT_TOKEN`: *(optional)* Telegram bot token for alerting top signals - `TELEGRAM_CHAT_ID`: *(optional)* Destination chat/channel ID for Telegram notifications +- `SIGNAL_PROFILE`: *(optional)* Default worker profile (`conservative`, `balanced`, or `aggressive`) controlling signal strictness ## Troubleshooting @@ -168,8 +169,8 @@ railway up - Perfect for testing and small deployments - May need to upgrade for production traffic - **GitHub Actions**: Free tier includes 2,000 minutes/month - - Worker runs every 5 minutes = ~8,640 runs/month - - Each run takes ~1-2 minutes = ~17,280 minutes/month + - Worker runs every 15 minutes = ~2,880 runs/month + - Each run takes ~1-2 minutes = ~2,880–5,760 minutes/month - May need to optimize or reduce frequency for free tier ## Updating Services diff --git a/services/api/main.py b/services/api/main.py index 1a6a3d91..90ca9e81 100644 --- a/services/api/main.py +++ b/services/api/main.py @@ -1,8 +1,9 @@ +import os from fastapi import FastAPI, Query, BackgroundTasks from services_common.db import fetch_df, ensure_schema from services_common.ingest import run_ingest_cycle from services_common.signals import ( - latest_signals_for_pairs, + compute_market_stress, signal_explanations, compute_all_signals, ) @@ -20,15 +21,24 @@ def pairs(): return {"pairs": cfg.symbols} @app.get("/signals") -def get_signals(pairs: str | None = Query(None)): +def get_signals(pairs: str | None = Query(None), profile: str = Query("balanced")): pairs_list = [p.strip() for p in pairs.split(",")] if pairs else cfg.symbols - data = latest_signals_for_pairs(pairs_list) - return {"signals": data, "explanations": signal_explanations()} + data = {} + resolved_profile = None + for p in pairs_list: + signal = compute_market_stress(p, profile) + resolved_profile = resolved_profile or signal.get("profile") + data[p] = signal + return { + "signals": data, + "profile": resolved_profile or profile, + "explanations": signal_explanations(profile), + } def _run_manual_cycle(): ensure_schema() run_ingest_cycle() - compute_all_signals() + compute_all_signals(os.getenv("SIGNAL_PROFILE")) @app.post("/ingest") def manual_ingest(background_tasks: BackgroundTasks): diff --git a/services/common/signals.py b/services/common/signals.py index 6db2afc7..ee698d06 100644 --- a/services/common/signals.py +++ b/services/common/signals.py @@ -1,27 +1,79 @@ +import os import pandas as pd -from services.common.db import fetch_df, upsert_many, execute +from services.common.db import fetch_df, execute + +PROFILE_RULES = { + "aggressive": { + "oi_high": 65, + "oi_low": 30, + "funding_neg": -0.00002, + "funding_pos": 0.00002, + "slope_long": 0.00008, + "slope_short": -0.00008, + "sent_spike": 1.2, + }, + "balanced": { + "oi_high": 80, + "oi_low": 40, + "funding_neg": -0.0001, + "funding_pos": 0.00005, + "slope_long": 0.00015, + "slope_short": -0.00015, + "sent_spike": 1.5, + }, + "conservative": { + "oi_high": 90, + "oi_low": 45, + "funding_neg": -0.0002, + "funding_pos": 0.00012, + "slope_long": 0.00025, + "slope_short": -0.00025, + "sent_spike": 1.8, + }, +} + +DEFAULT_PROFILE = os.getenv("SIGNAL_PROFILE", "balanced").lower() +if DEFAULT_PROFILE not in PROFILE_RULES: + DEFAULT_PROFILE = "balanced" def _percentile(series: pd.Series, value: float): if series.empty: return None return (series < value).mean() * 100.0 -def compute_market_stress(pair: str): - oi = fetch_df(""" +def _resolve_profile(profile: str | None) -> str: + key = (profile or DEFAULT_PROFILE).lower() + return key if key in PROFILE_RULES else DEFAULT_PROFILE + + +def compute_market_stress(pair: str, profile: str | None = None): + profile_key = _resolve_profile(profile) + rules = PROFILE_RULES[profile_key] + + oi = fetch_df( + """ SELECT ts, value_usd FROM open_interest WHERE pair=%(pair)s AND ts > now() - interval '30 days' ORDER BY ts - """, {"pair": pair}) - fr = fetch_df(""" + """, + {"pair": pair}, + ) + fr = fetch_df( + """ SELECT ts, rate FROM funding_rates WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts - """, {"pair": pair}) - sent = fetch_df(""" + """, + {"pair": pair}, + ) + sent = fetch_df( + """ SELECT ts, mentions, score_norm, keywords FROM sentiment WHERE pair=%(pair)s AND ts > now() - interval '14 days' ORDER BY ts - """, {"pair": pair}) + """, + {"pair": pair}, + ) regime = "Unknown" bias = "Neutral" @@ -29,48 +81,109 @@ def compute_market_stress(pair: str): short_prob = 0.5 summary = "Insufficient data." - if not oi.empty and not fr.empty and not sent.empty: - latest_oi = oi["value_usd"].iloc[-1] - oi_pct = _percentile(oi["value_usd"], latest_oi) - latest_funding = fr["rate"].iloc[-1] - sent["liq_kw"] = sent["keywords"].apply(lambda d: (d.get("liquidation",0) if isinstance(d, dict) else 0) + (d.get("margin call",0) if isinstance(d, dict) else 0)) - last_week = sent.tail(max(1, len(sent)//2)) - first_week = sent.head(len(sent)-len(last_week)) if len(sent)>1 else sent.head(1) + latest_oi = oi["value_usd"].iloc[-1] if not oi.empty else None + oi_pct = _percentile(oi["value_usd"], latest_oi) if latest_oi is not None else None + latest_funding = fr["rate"].iloc[-1] if not fr.empty else None + + sent_spike = None + if not sent.empty: + sent["liq_kw"] = sent["keywords"].apply( + lambda d: (d.get("liquidation", 0) if isinstance(d, dict) else 0) + + (d.get("margin call", 0) if isinstance(d, dict) else 0) + ) + last_week = sent.tail(max(1, len(sent) // 2)) + first_week = sent.head(len(sent) - len(last_week)) if len(sent) > 1 else sent.head(1) base = max(1, first_week["liq_kw"].sum()) - spike_ratio = (last_week["liq_kw"].sum()) / base + sent_spike = (last_week["liq_kw"].sum()) / base + + candles = fetch_df( + """ + SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 60 + """, + {"pair": pair}, + ).sort_values("ts") + slope = 0.0 + if not candles.empty: + y = candles["close"].pct_change().fillna(0).tail(12) + slope = y.mean() - if oi_pct is not None and oi_pct >= 90 and latest_funding < 0 and spike_ratio >= 2.0: + data_points = sum( + [ + 1 if latest_oi is not None else 0, + 1 if latest_funding is not None else 0, + 1 if not candles.empty else 0, + ] + ) + + if data_points >= 1: + if ( + oi_pct is not None + and latest_funding is not None + and oi_pct >= rules["oi_high"] + and latest_funding <= rules["funding_neg"] + and (sent_spike is None or sent_spike >= rules["sent_spike"]) + ): regime = "Risky / High Liquidation Risk" bias = "Short" - long_prob = 0.25 - short_prob = 0.75 - summary = "OI in 90th pct+, funding negative, and liquidation mentions up ≥200%. Consider caution or short bias." + long_prob = 0.2 + short_prob = 0.8 + summary = ( + f"[{profile_key.capitalize()}] OI in {rules['oi_high']}th pct+, funding ≤ " + f"{rules['funding_neg']*10000:.1f} bps, and stress chatter elevated." + ) else: - candles = fetch_df(""" - SELECT ts, close FROM candles WHERE pair=%(pair)s ORDER BY ts DESC LIMIT 50 - """, {"pair": pair}).sort_values("ts") - slope = 0.0 - if not candles.empty: - y = candles["close"].pct_change().fillna(0).tail(10) - slope = y.mean() - if slope > 0 and latest_funding >= 0: + long_tailwind = False + short_headwind = False + + if slope > rules["slope_long"]: + long_tailwind = True + if slope < rules["slope_short"]: + short_headwind = True + + if latest_funding is not None: + if latest_funding > rules["funding_pos"]: + long_tailwind = True + if latest_funding < rules["funding_neg"]: + short_headwind = True + + if oi_pct is not None: + if oi_pct >= rules["oi_high"]: + short_headwind = True + if oi_pct <= rules["oi_low"]: + long_tailwind = True + + if long_tailwind and not short_headwind: regime = "Constructive" bias = "Long" - long_prob = 0.65 - short_prob = 0.35 - summary = "Upward momentum with non-negative funding suggests a modest long bias." - elif slope < 0 and latest_funding <= 0: + long_prob = 0.7 + short_prob = 0.3 + summary = ( + f"[{profile_key.capitalize()}] Momentum/funding tailwinds favour longs; monitor for follow-through." + ) + elif short_headwind and not long_tailwind: regime = "Weak" bias = "Short" - long_prob = 0.35 - short_prob = 0.65 - summary = "Downward momentum with non-positive funding suggests a modest short bias." + long_prob = 0.3 + short_prob = 0.7 + summary = ( + f"[{profile_key.capitalize()}] Elevated OI or negative funding tilts short; watch for squeeze risk." + ) + elif long_tailwind and short_headwind: + regime = "Cross Currents" + bias = "Flat" + long_prob = 0.5 + short_prob = 0.5 + summary = ( + f"[{profile_key.capitalize()}] Drivers conflict (momentum vs positioning); stay nimble." + ) else: regime = "Balanced / Choppy" bias = "Flat" long_prob = 0.5 short_prob = 0.5 - summary = "Mixed signals; prefer mean-reversion or wait for clarity." + summary = ( + f"[{profile_key.capitalize()}] No clear edge from funding, momentum, or OI; favour range setups." + ) return { "pair": pair, @@ -78,21 +191,23 @@ def compute_market_stress(pair: str): "bias": bias, "long_prob": float(long_prob), "short_prob": float(short_prob), - "summary": summary + "summary": summary, + "profile": profile_key, } -def compute_all_signals(): +def compute_all_signals(profile: str | None = None): + profile_key = _resolve_profile(profile) pairs_df = fetch_df("SELECT DISTINCT pair FROM candles") pairs = list(pairs_df["pair"]) if "pair" in pairs_df else [] out = [] for p in pairs: - s = compute_market_stress(p) + s = compute_market_stress(p, profile_key) out.append(s) from datetime import datetime, timezone now = datetime.now(timezone.utc) rows = [{ "ts": now, "pair": s["pair"], "regime": s["regime"], "bias": s["bias"], - "long_prob": s["long_prob"], "short_prob": s["short_prob"], "summary": s["summary"] + "long_prob": s["long_prob"], "short_prob": s["short_prob"], "summary": s["summary"], } for s in out] if rows: sql = """ @@ -102,19 +217,16 @@ def compute_all_signals(): for row in rows: execute(sql, row) -def latest_signals_for_pairs(pairs: list[str]): - placeholders = ",".join(["%s"]*len(pairs)) - q = f""" - SELECT DISTINCT ON (pair) pair, ts, regime, bias, long_prob, short_prob, summary - FROM signals - WHERE pair IN ({placeholders}) - ORDER BY pair, ts DESC - """ - df = fetch_df(q, pairs) - result = {} - for _, r in df.iterrows(): - result[r["pair"]] = dict(r) - return result - -def signal_explanations(): - return {"market_stress": "OI ≥ 90th pct + negative funding + liquidation keyword spike ≥ 200% → bearish risk."} +def signal_explanations(profile: str | None = None): + profile_key = _resolve_profile(profile) + rules = PROFILE_RULES[profile_key] + return { + "market_stress": ( + f"Risky trigger when open interest hits the {rules['oi_high']}th percentile, " + f"funding ≤ {rules['funding_neg']*10000:.1f} bps, and liquidation chatter ≥ {rules['sent_spike']}× baseline." + ), + "momentum": ( + f"Long tailwind needs slope ≥ {rules['slope_long']*10000:.1f} bps/hr or funding ≥ {rules['funding_pos']*10000:.1f} bps. " + f"Short tailwind kicks in below {rules['slope_short']*10000:.1f} bps/hr or if funding ≤ {rules['funding_neg']*10000:.1f} bps." + ), + } diff --git a/services/ui/app.py b/services/ui/app.py index 65cdf686..a547d7e5 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -2,11 +2,17 @@ import requests import pandas as pd import streamlit as st -import streamlit.components.v1 as components from urllib.parse import urlencode -from typing import Optional +from typing import Optional, List, Tuple import plotly.graph_objects as go +PROFILE_LABELS = { + "Aggressive (fast triggers)": "aggressive", + "Balanced (default)": "balanced", + "Conservative (high confidence)": "conservative", +} +DEFAULT_PROFILE_KEY = "balanced" + DEFAULT_CANDIDATES = [ "https://crypto-risk-api-production.up.railway.app", "https://crypto-risk-api.onrender.com", @@ -65,20 +71,61 @@ def resolve_api_base() -> Optional[str]: box-shadow: 0 18px 35px rgba(15, 23, 42, 0.3); margin-bottom: 1.5rem; } + .snapshot-card { + padding: 0.9rem 1.1rem; + border-radius: 16px; + } + .snapshot-symbol { + font-size: 0.85rem; + color: #cdd4ff; + text-transform: uppercase; + letter-spacing: 0.08em; + } + .snapshot-price { + font-size: 1.2rem; + font-weight: 700; + margin: 0.25rem 0 0.35rem; + } + .snapshot-change { + font-size: 0.85rem; + } + .signal-driver { + background: rgba(15, 23, 42, 0.65); + border: 1px solid rgba(99, 102, 241, 0.25); + border-radius: 18px; + padding: 1rem 1.25rem; + margin-bottom: 1rem; + } + .signal-driver-title { + font-size: 0.85rem; + letter-spacing: 0.08em; + color: #94a3b8; + text-transform: uppercase; + } + .signal-driver-value { + font-size: 1.4rem; + font-weight: 700; + color: #e2e8f0; + margin: 0.2rem 0; + } + .signal-driver-desc { + font-size: 0.9rem; + color: #cbd5f5; + } button[data-baseweb="button"], div.stButton > button { border-radius: 999px !important; - border: 1px solid rgba(99, 102, 241, 0.45); + border: 1px solid rgba(99, 102, 241, 0.35); font-weight: 600; - background: linear-gradient(135deg, #f8fafc 0%, #6366f1 50%, #312e81 100%); - color: #0f172a; - box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.5), 0 12px 24px rgba(79, 70, 229, 0.35); + background: #4338ca; + color: #f8fafc; + box-shadow: 0 10px 20px rgba(67, 56, 202, 0.35); text-shadow: none; } button[data-baseweb="button"]:hover, div.stButton > button:hover { - filter: brightness(1.05); - box-shadow: inset 0 1px 0 rgba(255,255,255,0.7), 0 14px 26px rgba(79,70,229,0.45); + background: #4f46e5; + box-shadow: 0 12px 24px rgba(79, 70, 229, 0.45); } .stTabs [data-baseweb="tab"] { border-radius: 999px !important; @@ -149,10 +196,14 @@ def fetch_timeseries(metric: str, pair: str, limit: int = 500) -> Optional[pd.Da return None @st.cache_data(ttl=120) -def fetch_signals(pairs: list[str]) -> dict: +def fetch_signals(pairs: list[str], profile: str) -> dict: try: - qs = "?pairs=" + ",".join(pairs) if pairs else "" - r = requests.get(f"{API_BASE}/signals{qs}", timeout=20) + params = {} + if pairs: + params["pairs"] = ",".join(pairs) + if profile: + params["profile"] = profile + r = requests.get(f"{API_BASE}/signals", params=params, timeout=20) r.raise_for_status() return r.json() except Exception as e: @@ -256,6 +307,74 @@ def build_aggr_trade_url(pair: str) -> str: target = pair.split(":", 1)[-1].replace("/", "").upper() return f"https://aggr.trade/?pair={target}" +def summarize_signal_drivers(cache: dict[str, Optional[pd.DataFrame]]) -> List[Tuple[str, str, str]]: + drivers: List[Tuple[str, str, str]] = [] + + oi = cache.get("oi") + if oi is not None and not oi.empty: + latest_oi = oi.iloc[-1] + latest_val = latest_oi.get("value_usd", latest_oi.iloc[-1] if hasattr(latest_oi, "iloc") else None) + if pd.notnull(latest_val): + pct = (oi["value_usd"] < latest_val).mean() * 100 if "value_usd" in oi.columns else None + desc = "Higher than most of the last 30 days." if pct and pct >= 70 else "Close to typical positioning." + if pct and pct <= 35: + desc = "Lighter positioning than usual." + drivers.append( + ( + "Open Interest", + f"{latest_val/1_000_000:,.1f}M USD" if latest_val else "n/a", + f"Approx. {pct:.0f}th percentile · {desc}" if pct is not None else desc, + ) + ) + + funding = cache.get("funding") + if funding is not None and not funding.empty: + col = "rate" if "rate" in funding.columns else funding.columns[-1] + latest_rate = funding[col].iloc[-1] + avg_rate = funding[col].tail(24).mean() + sentiment = "longs paying" if latest_rate > 0 else "shorts paying" if latest_rate < 0 else "neutral" + drivers.append( + ( + "Funding", + f"{latest_rate*10000:+.1f} bps", + f"{sentiment}; 24h avg {avg_rate*10000:+.1f} bps.", + ) + ) + + candles = cache.get("candles") + if candles is not None and not candles.empty: + closes = candles["close"] + slope = closes.pct_change().rolling(window=12, min_periods=6).mean().iloc[-1] + slope_bps = slope * 10000 if pd.notnull(slope) else 0 + if pd.notnull(slope): + direction = "Upside pressure" if slope > 0 else "Downside pressure" if slope < 0 else "Flat momentum" + drivers.append( + ( + "Momentum", + f"{slope_bps:+.1f} bps/hr", + f"{direction} based on the last ~12 hours of closes.", + ) + ) + + sentiment_df = cache.get("sentiment") + if sentiment_df is not None and not sentiment_df.empty and "keywords" in sentiment_df.columns: + latest_kw = sentiment_df["keywords"].dropna().iloc[-1] if not sentiment_df["keywords"].dropna().empty else {} + if isinstance(latest_kw, dict): + fear_terms = ["liquidation", "margin call", "crash", "dump"] + bull_terms = ["rally", "surge", "pump", "bull"] + fear_count = sum(latest_kw.get(term, 0) for term in fear_terms) + bull_count = sum(latest_kw.get(term, 0) for term in bull_terms) + tone = "Bearish chatter dominates." if fear_count > bull_count else "Bullish chatter dominates." if bull_count > fear_count else "Chatter balanced." + drivers.append( + ( + "Headline Tone", + f"Fear {fear_count} vs Bull {bull_count}", + tone, + ) + ) + + return drivers + @st.cache_data(ttl=600) def fetch_fear_greed() -> Optional[dict]: try: @@ -334,7 +453,10 @@ def fetch_alt_global() -> Optional[dict]: return None # Refresh button to clear cache -controls = st.columns([1, 1, 3]) +if "selected_profile_key" not in st.session_state: + st.session_state["selected_profile_key"] = DEFAULT_PROFILE_KEY + +controls = st.columns([1, 1, 1.4]) with controls[0]: if st.button("Refresh Data", use_container_width=True): fetch_pairs.clear() @@ -361,24 +483,38 @@ def fetch_alt_global() -> Optional[dict]: st.error(f"Manual ingest failed: {msg}") pairs = fetch_pairs() -pair = None -if pairs: - pair = st.selectbox("Select trading pair", pairs, index=0, format_func=lambda x: x.replace(":", " · ")) -else: +if not pairs: st.warning("No trading pairs available from API.") - -# If no pair selected, stop further processing -if not pair: - st.info("Select a pair to see data.") st.stop() -limit = 1000 +with controls[2]: + profile_labels = list(PROFILE_LABELS.keys()) + current_key = st.session_state.get("selected_profile_key", DEFAULT_PROFILE_KEY) + current_label = next((label for label, key in PROFILE_LABELS.items() if key == current_key), profile_labels[1]) + selected_label = st.selectbox( + "Signal profile", + profile_labels, + index=profile_labels.index(current_label), + help="Choose how strict the signal engine should be.", + ) + new_profile_key = PROFILE_LABELS[selected_label] + if new_profile_key != current_key: + st.session_state["selected_profile_key"] = new_profile_key + fetch_signals.clear() -st.subheader(f"Data for Pair: {pair}") +profile_key = st.session_state.get("selected_profile_key", DEFAULT_PROFILE_KEY) + +if "selected_pair" not in st.session_state or st.session_state["selected_pair"] not in pairs: + st.session_state["selected_pair"] = pairs[0] -sig_payload = fetch_signals([pair]) or {} +pair = st.session_state["selected_pair"] + +limit = 1000 + +sig_payload = fetch_signals([pair], profile_key) or {} signals_map = sig_payload.get("signals", {}) explanations = sig_payload.get("explanations", {}) +active_profile = sig_payload.get("profile", profile_key) # Pre-load core time series once so analytics modules can reuse them ts_cache: dict[str, Optional[pd.DataFrame]] = { @@ -405,12 +541,10 @@ def fetch_alt_global() -> Optional[dict]: color = "#22c55e" if change is not None and change >= 0 else "#ef4444" st.markdown( f""" -
-
{symbol}
-
- {price_str} -
-
+
+
{symbol}
+
{price_str}
+
24h: {delta}
@@ -418,7 +552,7 @@ def fetch_alt_global() -> Optional[dict]: unsafe_allow_html=True, ) -macro = st.columns([1.2, 1, 1]) +macro = st.columns([0.85, 1, 1]) with macro[0]: fg = fetch_fear_greed() if fg: @@ -427,7 +561,7 @@ def fetch_alt_global() -> Optional[dict]: go.Indicator( mode="gauge+number", value=fg_val, - number={"suffix": " / 100", "font": {"color": "#f1f5ff"}}, + number={"suffix": " / 100", "font": {"color": "#f1f5ff", "size": 32}}, title={"text": f"Fear & Greed · {fg['classification']}", "font": {"color": "#e0e7ff"}}, gauge={ "axis": {"range": [0, 100], "tickwidth": 1, "tickcolor": "#94a3b8"}, @@ -447,8 +581,8 @@ def fetch_alt_global() -> Optional[dict]: gauge.update_layout( paper_bgcolor="rgba(15, 23, 42, 0.65)", font={"color": "#cdd4ff"}, - height=260, - margin=dict(l=10, r=10, t=40, b=0), + height=220, + margin=dict(l=10, r=10, t=30, b=0), ) st.plotly_chart(gauge, use_container_width=True, config={"displayModeBar": False}) else: @@ -511,6 +645,12 @@ def fetch_alt_global() -> Optional[dict]: st.info("Alternative.me global metrics unavailable right now.") st.subheader("Hot Signals") +st.markdown( + "Bias scores blend open interest, funding, short-term momentum, and headline tone." +) +st.caption(f"Profile: {active_profile.capitalize() if active_profile else profile_key.capitalize()} · Adjust via the selector above to change strictness.") +drivers = summarize_signal_drivers(ts_cache) + if pair in signals_map: s = signals_map[pair] c1, c2, c3, c4 = st.columns(4) @@ -522,11 +662,67 @@ def fetch_alt_global() -> Optional[dict]: st.metric("Long Prob. %", round(100 * float(s.get("long_prob", 0)), 1)) with c4: st.metric("Short Prob. %", round(100 * float(s.get("short_prob", 0)), 1)) - if s.get("summary"): - st.info(s["summary"]) + summary_text = s.get("summary") + if summary_text: + bias_lower = s.get("bias", "").lower() + callout = st.info + if bias_lower == "long": + callout = st.success + elif bias_lower == "short": + callout = st.warning + callout(summary_text) else: st.write("No signal for selected pair yet.") +if drivers: + st.markdown("**Signal Drivers**") + cols_per_row = 2 + for i in range(0, len(drivers), cols_per_row): + row = drivers[i : i + cols_per_row] + columns = st.columns(len(row)) + for (title, value, desc), col in zip(row, columns): + with col: + st.markdown( + f""" +
+
{title}
+
{value}
+
{desc}
+
+ """, + unsafe_allow_html=True, + ) +else: + st.markdown( + "*Waiting on more data to break down the drivers. Run the worker or refresh once new samples arrive.*" + ) + +if explanations: + with st.expander("How this profile scores signals", expanded=False): + for key, text in explanations.items(): + label = key.replace("_", " ").title() + st.markdown(f"**{label}:** {text}") + +with st.container(): + selected = st.selectbox( + "Select trading pair", + pairs, + index=pairs.index(pair) if pair in pairs else 0, + format_func=lambda x: x.replace(":", " · "), + key="selected_pair", + ) + if selected != pair: + st.experimental_rerun() + +pair = st.session_state.get("selected_pair", pair) + +# Ensure we still have a pair after potential selection change +if not pair: + st.info("Select a pair to see data.") + st.stop() + +st.subheader(f"Data for Pair: {pair}") + visual_tabs = st.tabs(["Price Action", "Funding & OI", "Sentiment", "aggr.trade"]) with visual_tabs[0]: @@ -620,7 +816,9 @@ def fetch_alt_global() -> Optional[dict]: with visual_tabs[3]: st.write("Live liquidation feed via aggr.trade") - components.iframe(build_aggr_trade_url(pair), height=640, scrolling=True) + aggr_url = build_aggr_trade_url(pair) + st.link_button("Open aggr.trade in new tab", aggr_url) + st.caption("aggr.trade does not allow in-app embedding, so use the button above to view the live heatmap.") st.subheader("Insight Modules") analysis_tabs = st.tabs(["Funding Overlay", "Volatility Pulse", "Sentiment Radar"]) @@ -658,6 +856,15 @@ def fetch_alt_global() -> Optional[dict]: margin=dict(l=0, r=0, t=35, b=0), ) st.plotly_chart(fig, use_container_width=True) + st.markdown( + """ + **How to read this:** + - *Price (norm 100)* scales the first data point to 100 so you can focus on directional drift rather than absolute price. + - *Funding (bps)* shows whether longs or shorts are paying; persistent positive funding implies long crowding, negatives imply short crowding. + - When funding rises while price stalls or drops, be cautious of long squeezes; the inverse can precede short squeezes. + - Look for divergences: price grinding higher while funding cools is healthier than price pumping on aggressive positive funding. + """ + ) with analysis_tabs[1]: vol = ts_cache["vol"] @@ -686,6 +893,16 @@ def fetch_alt_global() -> Optional[dict]: st.plotly_chart(fig, use_container_width=True) stats = series.describe().to_frame(name="ATR").T st.dataframe(stats) + st.markdown( + """ + **How to read this:** + - *Latest ATR* shows the current absolute true range (volatility proxy). + - *mean / std* help gauge if today's movement is above its recent norm. + - A rising *max* or widening *std* often hints at breakout-like conditions. + - If the latest ATR is near the lower quartile, conditions are typically calmer (range-trading bias). + - When ATR presses into the upper quartile, tighten risk or look for momentum setups. + """ + ) with analysis_tabs[2]: sentiment = ts_cache["sentiment"] @@ -711,6 +928,15 @@ def fetch_alt_global() -> Optional[dict]: st.plotly_chart(fig, use_container_width=True) else: st.info("Latest sentiment entry does not include keyword counts.") + st.markdown( + """ + **How to read this:** + - Bars show the latest counts of sentiment keywords captured from CryptoPanic headlines. + - Liquidity-stress words (e.g., *liquidation*, *margin call*) signal risk-off chatter; bullish words (e.g., *rally*, *surge*) hint at optimism. + - Use the mix to contextualise signal bias: a short setup is stronger when bearish terms dominate, and vice versa. + - Sudden spikes in any keyword bucket often precede volatility bursts—combine with the Funding overlay for higher conviction. + """ + ) st.divider() st.caption("Configure pairs, Telegram alerts, and scheduler in .env. Add API keys for live data.") diff --git a/services/worker/run_worker.py b/services/worker/run_worker.py index 6ffa2705..2e79ef19 100644 --- a/services/worker/run_worker.py +++ b/services/worker/run_worker.py @@ -36,7 +36,7 @@ def main(): print("[worker] Running ingest cycle...") run_ingest_cycle() print("[worker] Computing signals...") - compute_all_signals() + compute_all_signals(os.getenv("SIGNAL_PROFILE")) print("[worker] Sending notifications (if configured)...") maybe_notify_top_signals() print("[worker] Worker cycle completed successfully!") diff --git a/timescale.env b/timescale.env index 5683c363..90aab0d4 100644 --- a/timescale.env +++ b/timescale.env @@ -1,5 +1,6 @@ # For local/testing only. Keep this PRIVATE and out of git. DATABASE_URL=postgres://tsdbadmin:hl7bjpkhq8ziestj@glmv89ewih.enc0lseujb.tsdb.cloud.timescale.com:34312/tsdb?sslmode=require -SCHEDULE_MINUTES=5 +SCHEDULE_MINUTES=15 SYMBOLS=binance:BTC/USDT,binance:ETH/USDT,bybit:SOL/USDT API_CANDIDATES=http://localhost:8000 +SIGNAL_PROFILE=balanced From 843ea737686ab0e9af31cd4813b77773588dc68b Mon Sep 17 00:00:00 2001 From: skymike <313426+skymike@users.noreply.github.com> Date: Sun, 2 Nov 2025 23:45:05 +0100 Subject: [PATCH 37/43] Update crypto-worker.yml From 6f24764206ba3a98625060c68df96b10796fd0e1 Mon Sep 17 00:00:00 2001 From: skymike Date: Mon, 3 Nov 2025 02:41:10 +0100 Subject: [PATCH 38/43] Enhance UI with live time display and improved market snapshot features Updated the Streamlit UI to include a live time display in Amsterdam timezone, enhancing user experience. Refactored the market snapshot section to utilize a more structured layout with a DataFrame for better data presentation. Improved the styling of signal metrics and added new visualizations for market data, including sentiment trends and volume analysis. Enhanced the overall responsiveness and clarity of the dashboard components. --- services/ui/app.py | 795 +++++++++++++++++++++++++-------------------- 1 file changed, 449 insertions(+), 346 deletions(-) diff --git a/services/ui/app.py b/services/ui/app.py index a547d7e5..00fff2ed 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -1,6 +1,9 @@ import os -import requests +from datetime import datetime +from zoneinfo import ZoneInfo + import pandas as pd +import requests import streamlit as st from urllib.parse import urlencode from typing import Optional, List, Tuple @@ -12,6 +15,7 @@ "Conservative (high confidence)": "conservative", } DEFAULT_PROFILE_KEY = "balanced" +LOCAL_TZ = ZoneInfo("Europe/Amsterdam") DEFAULT_CANDIDATES = [ "https://crypto-risk-api-production.up.railway.app", @@ -112,6 +116,25 @@ def resolve_api_base() -> Optional[str]: font-size: 0.9rem; color: #cbd5f5; } + .signal-pill { + background: rgba(15, 23, 42, 0.6); + border: 1px solid rgba(148, 163, 184, 0.2); + border-radius: 16px; + padding: 0.75rem 1rem; + margin-bottom: 0.5rem; + } + .signal-pill-title { + font-size: 0.75rem; + letter-spacing: 0.08em; + text-transform: uppercase; + color: #94a3b8; + } + .signal-pill-value { + font-size: 1.6rem; + font-weight: 700; + color: #e2e8f0; + line-height: 1.1; + } button[data-baseweb="button"], div.stButton > button { border-radius: 999px !important; @@ -153,6 +176,10 @@ def resolve_api_base() -> Optional[str]: st.title("Crypto Risk Dashboard") st.caption("Graphs, Meters, and Hot Signals") +st.caption( + f"Live time: {datetime.now(LOCAL_TZ).strftime('%d %b %Y · %H:%M:%S')} (GMT+1 Amsterdam)" +) + if not API_BASE: st.error("Could not locate the API automatically.") manual = st.text_input("Enter your API base URL (e.g., https://crypto-risk-api-production.up.railway.app)") @@ -189,7 +216,7 @@ def fetch_timeseries(metric: str, pair: str, limit: int = 500) -> Optional[pd.Da df = pd.DataFrame(rows) if "ts" in df.columns: df["ts"] = pd.to_datetime(df["ts"], utc=True) - df = df.set_index("ts") + df = df.set_index("ts").tz_convert(LOCAL_TZ) return df except Exception as e: st.error(f"Error fetching timeseries {metric}: {e}") @@ -526,182 +553,192 @@ def fetch_alt_global() -> Optional[dict]: } market_snapshot = fetch_market_snapshot(pairs) -if market_snapshot: - st.subheader("Live Market Snapshot") - cards = st.columns(min(4, len(market_snapshot))) - idx = 0 - for symbol, data in market_snapshot.items(): - col = cards[idx % len(cards)] - idx += 1 - with col: - price = data.get("price") - change = data.get("change") - delta = f"{change:+.2f}%" if change is not None else "n/a" - price_str = f"{price:,.2f} USD" if price is not None else "n/a" - color = "#22c55e" if change is not None and change >= 0 else "#ef4444" - st.markdown( - f""" -
-
{symbol}
-
{price_str}
-
- 24h: {delta} -
-
- """, - unsafe_allow_html=True, - ) - -macro = st.columns([0.85, 1, 1]) -with macro[0]: - fg = fetch_fear_greed() - if fg: - fg_val = float(fg["value"]) - gauge = go.Figure( - go.Indicator( - mode="gauge+number", - value=fg_val, - number={"suffix": " / 100", "font": {"color": "#f1f5ff", "size": 32}}, - title={"text": f"Fear & Greed · {fg['classification']}", "font": {"color": "#e0e7ff"}}, - gauge={ - "axis": {"range": [0, 100], "tickwidth": 1, "tickcolor": "#94a3b8"}, - "bar": {"color": "#a855f7"}, - "bgcolor": "#0f172a", - "borderwidth": 1, - "bordercolor": "#6366f1", - "steps": [ - {"range": [0, 25], "color": "#7f1d1d"}, - {"range": [25, 50], "color": "#b45309"}, - {"range": [50, 75], "color": "#1f2937"}, - {"range": [75, 100], "color": "#065f46"}, - ], - }, +with st.expander("Live Market Snapshot", expanded=True): + if market_snapshot: + snapshot_rows = [] + for symbol, data in market_snapshot.items(): + snapshot_rows.append( + { + "Symbol": symbol, + "Price (USD)": data.get("price"), + "24h Δ %": data.get("change"), + "Updated": datetime.now(LOCAL_TZ).strftime("%d %b %Y %H:%M"), + } ) + snapshot_df = pd.DataFrame(snapshot_rows).set_index("Symbol") + styled = snapshot_df.style.format({ + "Price (USD)": "{:.2f}", + "24h Δ %": "{:+.2f}%", + }).background_gradient( + subset=["24h Δ %"], cmap="RdYlGn" ) - gauge.update_layout( - paper_bgcolor="rgba(15, 23, 42, 0.65)", - font={"color": "#cdd4ff"}, - height=220, - margin=dict(l=10, r=10, t=30, b=0), - ) - st.plotly_chart(gauge, use_container_width=True, config={"displayModeBar": False}) + st.dataframe(styled, use_container_width=True, height=220) else: - st.info("Fear & Greed data is temporarily unavailable.") - -with macro[1]: - flows = fetch_asset_flows(pairs) - if flows: - sorted_rows = sorted(flows.items(), key=lambda kv: kv[1]["volume"], reverse=True) - top_rows = sorted_rows[:3] - items = [] - for symbol, info in top_rows: - vol_usd = info["volume"] - change = info["change"] - price = info["price"] - items.append( + st.info("No market snapshot data available yet. Refresh once the worker ingests more data.") + +with st.expander("Macro Context", expanded=True): + macro = st.columns([0.85, 1, 1]) + with macro[0]: + fg = fetch_fear_greed() + if fg: + fg_val = float(fg["value"]) + gauge = go.Figure( + go.Indicator( + mode="gauge+number", + value=fg_val, + number={"suffix": " / 100", "font": {"color": "#f1f5ff", "size": 32}}, + title={"text": f"Fear & Greed · {fg['classification']}", "font": {"color": "#e0e7ff"}}, + gauge={ + "axis": {"range": [0, 100], "tickwidth": 1, "tickcolor": "#94a3b8"}, + "bar": {"color": "#a855f7"}, + "bgcolor": "#0f172a", + "borderwidth": 1, + "bordercolor": "#6366f1", + "steps": [ + {"range": [0, 25], "color": "#7f1d1d"}, + {"range": [25, 50], "color": "#b45309"}, + {"range": [50, 75], "color": "#1f2937"}, + {"range": [75, 100], "color": "#065f46"}, + ], + }, + ) + ) + gauge.update_layout( + paper_bgcolor="rgba(15, 23, 42, 0.65)", + font={"color": "#cdd4ff"}, + height=220, + margin=dict(l=10, r=10, t=30, b=0), + ) + st.plotly_chart(gauge, use_container_width=True, config={"displayModeBar": False}) + else: + st.info("Fear & Greed data is temporarily unavailable.") + + with macro[1]: + flows = fetch_asset_flows(pairs) + if flows: + sorted_rows = sorted(flows.items(), key=lambda kv: kv[1]["volume"], reverse=True) + top_rows = sorted_rows[:3] + items = [] + for symbol, info in top_rows: + vol_usd = info["volume"] + change = info["change"] + price = info["price"] + items.append( + f""" +
+
{symbol}
+
{price:,.2f} USD
+
+ 24h Vol: {vol_usd/1_000_000:,.1f}M • Change: {change:+.2f}% +
+
+ """ + ) + st.markdown( + "
" + "".join(items) + "
Source: coincap.io
", + unsafe_allow_html=True, + ) + else: + st.info("CoinCap asset metrics unavailable right now.") + + with macro[2]: + global_data = fetch_alt_global() + if global_data: + mc = global_data["market_cap"] / 1_000_000_000 if global_data["market_cap"] else 0 + vol = global_data["volume"] / 1_000_000_000 if global_data["volume"] else 0 + dom = global_data["btc_dominance"] + change = global_data["market_cap_change"] + st.markdown( f""" -
-
{symbol}
-
{price:,.2f} USD
-
- 24h Vol: {vol_usd/1_000_000:,.1f}M • Change: {change:+.2f}% +
+
Global Market Overview
+
+ MC: {mc:,.1f}B · 24h Vol: {vol:,.1f}B +
+
+ BTC Dominance: {dom:.1f}% · Change: {change:+.2f}%
+
+ Active Assets: {global_data['active_cryptos']} · Markets: {global_data['active_markets']} +
+
Source: alternative.me
- """ + """, + unsafe_allow_html=True, ) - st.markdown( - "
" + "".join(items) + "
Source: coincap.io
", - unsafe_allow_html=True, - ) - else: - st.info("CoinCap asset metrics unavailable right now.") - -with macro[2]: - global_data = fetch_alt_global() - if global_data: - mc = global_data["market_cap"] / 1_000_000_000 if global_data["market_cap"] else 0 - vol = global_data["volume"] / 1_000_000_000 if global_data["volume"] else 0 - dom = global_data["btc_dominance"] - change = global_data["market_cap_change"] - st.markdown( - f""" -
-
Global Market Overview
-
- MC: {mc:,.1f}B · 24h Vol: {vol:,.1f}B -
-
- BTC Dominance: {dom:.1f}% · Change: {change:+.2f}% -
-
- Active Assets: {global_data['active_cryptos']} · Markets: {global_data['active_markets']} -
-
Source: alternative.me
-
- """, - unsafe_allow_html=True, - ) - else: - st.info("Alternative.me global metrics unavailable right now.") + else: + st.info("Alternative.me global metrics unavailable right now.") -st.subheader("Hot Signals") -st.markdown( - "Bias scores blend open interest, funding, short-term momentum, and headline tone." -) -st.caption(f"Profile: {active_profile.capitalize() if active_profile else profile_key.capitalize()} · Adjust via the selector above to change strictness.") -drivers = summarize_signal_drivers(ts_cache) - -if pair in signals_map: - s = signals_map[pair] - c1, c2, c3, c4 = st.columns(4) - with c1: - st.metric("Market Regime", s.get("regime", "Unknown")) - with c2: - st.metric("Bias", s.get("bias", "Neutral")) - with c3: - st.metric("Long Prob. %", round(100 * float(s.get("long_prob", 0)), 1)) - with c4: - st.metric("Short Prob. %", round(100 * float(s.get("short_prob", 0)), 1)) - summary_text = s.get("summary") - if summary_text: - bias_lower = s.get("bias", "").lower() - callout = st.info - if bias_lower == "long": - callout = st.success - elif bias_lower == "short": - callout = st.warning - callout(summary_text) -else: - st.write("No signal for selected pair yet.") - -if drivers: - st.markdown("**Signal Drivers**") - cols_per_row = 2 - for i in range(0, len(drivers), cols_per_row): - row = drivers[i : i + cols_per_row] - columns = st.columns(len(row)) - for (title, value, desc), col in zip(row, columns): +with st.expander("Hot Signals", expanded=True): + st.subheader("Hot Signals") + st.markdown( + "Bias scores blend open interest, funding, short-term momentum, and headline tone." + ) + st.caption( + f"Profile: {active_profile.capitalize() if active_profile else profile_key.capitalize()} · Adjust via the selector above to change strictness." + ) + drivers = summarize_signal_drivers(ts_cache) + + if pair in signals_map: + s = signals_map[pair] + metric_cols = st.columns(4) + metrics = [ + ("Market Regime", s.get("regime", "Unknown")), + ("Bias", s.get("bias", "Neutral")), + ("Long Prob. %", f"{round(100 * float(s.get('long_prob', 0)), 1):.1f}"), + ("Short Prob. %", f"{round(100 * float(s.get('short_prob', 0)), 1):.1f}"), + ] + for (title, value), col in zip(metrics, metric_cols): with col: st.markdown( f""" -
-
{title}
-
{value}
-
{desc}
+
+
{title}
+
{value}
""", unsafe_allow_html=True, ) -else: - st.markdown( - "*Waiting on more data to break down the drivers. Run the worker or refresh once new samples arrive.*" - ) + summary_text = s.get("summary") + if summary_text: + bias_lower = s.get("bias", "").lower() + callout = st.info + if bias_lower == "long": + callout = st.success + elif bias_lower == "short": + callout = st.warning + callout(summary_text) + else: + st.write("No signal for selected pair yet.") + + if drivers: + st.markdown("**Signal Drivers**") + cols_per_row = 2 + for i in range(0, len(drivers), cols_per_row): + row = drivers[i : i + cols_per_row] + columns = st.columns(len(row)) + for (title, value, desc), col in zip(row, columns): + with col: + st.markdown( + f""" +
+
{title}
+
{value}
+
{desc}
+
+ """, + unsafe_allow_html=True, + ) + else: + st.markdown( + "*Waiting on more data to break down the drivers. Run the worker or refresh once new samples arrive.*" + ) -if explanations: - with st.expander("How this profile scores signals", expanded=False): - for key, text in explanations.items(): - label = key.replace("_", " ").title() - st.markdown(f"**{label}:** {text}") + if explanations: + with st.expander("How this profile scores signals", expanded=False): + for key, text in explanations.items(): + label = key.replace("_", " ").title() + st.markdown(f"**{label}:** {text}") with st.container(): selected = st.selectbox( @@ -721,222 +758,288 @@ def fetch_alt_global() -> Optional[dict]: st.info("Select a pair to see data.") st.stop() -st.subheader(f"Data for Pair: {pair}") +with st.expander(f"Data for Pair: {pair}", expanded=True): + st.subheader(f"Data for Pair: {pair}") + st.markdown("Timeseries are displayed in GMT+1 (Amsterdam). Use the tabs below to inspect different market lenses.") -visual_tabs = st.tabs(["Price Action", "Funding & OI", "Sentiment", "aggr.trade"]) + visual_tabs = st.tabs(["Price Action", "Funding & OI", "Sentiment", "Returns", "Volume & Liquidity", "aggr.trade"]) -with visual_tabs[0]: - candles = ts_cache["candles"] - if candles is not None and not candles.empty: - fig = go.Figure( - data=[ - go.Candlestick( - x=candles.index, - open=candles["open"], - high=candles["high"], - low=candles["low"], - close=candles["close"], - name="Price", + with visual_tabs[0]: + candles = ts_cache["candles"] + if candles is not None and not candles.empty: + fig = go.Figure( + data=[ + go.Candlestick( + x=candles.index, + open=candles["open"], + high=candles["high"], + low=candles["low"], + close=candles["close"], + name="Price", + ) + ] + ) + fig.update_layout( + margin=dict(l=0, r=0, t=35, b=0), + template="plotly_dark", + title=f"{pair} · Candlestick", + ) + fig.update_xaxes(title="Time (GMT+1)", rangeslider_visible=False, tickformat="%d %b %H:%M") + st.plotly_chart(fig, use_container_width=True) + else: + st.info("No candles data available.") + + with visual_tabs[1]: + funding = ts_cache["funding"] + oi = ts_cache["oi"] + if funding is not None and not funding.empty and oi is not None and not oi.empty: + col = "rate" if "rate" in funding.columns else funding.columns[-1] + oi_col = "value_usd" if "value_usd" in oi.columns else oi.columns[-1] + merged = funding[[col]].join(oi[[oi_col]], how="inner") + merged = merged.dropna() + if merged.empty: + st.info("Not enough overlapping funding/OI data.") + else: + fig = go.Figure() + fig.add_trace( + go.Bar( + x=merged.index, + y=merged[col] * 10000, + name="Funding (bps)", + marker_color="#22c55e", + opacity=0.6, + ) ) - ] - ) - fig.update_layout( - margin=dict(l=0, r=0, t=35, b=0), - template="plotly_dark", - title=f"{pair} · Candlestick", - ) - st.plotly_chart(fig, use_container_width=True) - else: - st.info("No candles data available.") - -with visual_tabs[1]: - funding = ts_cache["funding"] - oi = ts_cache["oi"] - if funding is not None and not funding.empty and oi is not None and not oi.empty: - col = "rate" if "rate" in funding.columns else funding.columns[-1] - oi_col = "value_usd" if "value_usd" in oi.columns else oi.columns[-1] - merged = funding[[col]].join(oi[[oi_col]], how="inner") - merged = merged.dropna() - if merged.empty: - st.info("Not enough overlapping funding/OI data.") + fig.add_trace( + go.Scatter( + x=merged.index, + y=merged[oi_col] / 1_000_000, + name="Open Interest (M USD)", + mode="lines", + line=dict(color="#38bdf8", width=2), + yaxis="y2", + ) + ) + fig.update_layout( + template="plotly_dark", + margin=dict(l=0, r=0, t=35, b=0), + yaxis=dict(title="Funding (bps)"), + yaxis2=dict(title="OI (Millions USD)", overlaying="y", side="right"), + title=f"{pair} · Funding vs Open Interest", + ) + fig.update_xaxes(title="Time (GMT+1)", tickformat="%d %b %H:%M") + st.plotly_chart(fig, use_container_width=True) else: + st.info("Funding or open interest data unavailable.") + + with visual_tabs[2]: + sentiment = ts_cache["sentiment"] + if sentiment is not None and not sentiment.empty: + metric = "score_norm" if "score_norm" in sentiment.columns else sentiment.columns[-1] fig = go.Figure() - fig.add_trace( - go.Bar( - x=merged.index, - y=merged[col] * 10000, - name="Funding (bps)", - marker_color="#22c55e", - opacity=0.6, - ) - ) fig.add_trace( go.Scatter( - x=merged.index, - y=merged[oi_col] / 1_000_000, - name="Open Interest (M USD)", - mode="lines", - line=dict(color="#38bdf8", width=2), - yaxis="y2", + x=sentiment.index, + y=sentiment[metric], + name="Sentiment", + line=dict(color="#f97316"), + fill="tozeroy", ) ) fig.update_layout( template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0), - yaxis=dict(title="Funding (bps)"), - yaxis2=dict(title="OI (Millions USD)", overlaying="y", side="right"), - title=f"{pair} · Funding vs Open Interest", + title=f"{pair} · Sentiment Trend", ) + fig.update_xaxes(title="Time (GMT+1)", tickformat="%d %b %H:%M") st.plotly_chart(fig, use_container_width=True) - else: - st.info("Funding or open interest data unavailable.") - -with visual_tabs[2]: - sentiment = ts_cache["sentiment"] - if sentiment is not None and not sentiment.empty: - metric = "score_norm" if "score_norm" in sentiment.columns else sentiment.columns[-1] - fig = go.Figure() - fig.add_trace( - go.Scatter( - x=sentiment.index, - y=sentiment[metric], - name="Sentiment", - line=dict(color="#f97316"), - fill="tozeroy", - ) - ) - fig.update_layout( - template="plotly_dark", - margin=dict(l=0, r=0, t=35, b=0), - title=f"{pair} · Sentiment Trend", - ) - st.plotly_chart(fig, use_container_width=True) - else: - st.info("No sentiment data available.") - -with visual_tabs[3]: - st.write("Live liquidation feed via aggr.trade") - aggr_url = build_aggr_trade_url(pair) - st.link_button("Open aggr.trade in new tab", aggr_url) - st.caption("aggr.trade does not allow in-app embedding, so use the button above to view the live heatmap.") - -st.subheader("Insight Modules") -analysis_tabs = st.tabs(["Funding Overlay", "Volatility Pulse", "Sentiment Radar"]) - -with analysis_tabs[0]: - candles = ts_cache["candles"] - funding = ts_cache["funding"] - if candles is None or candles.empty or funding is None or funding.empty: - st.info("Need both candle and funding data for overlay.") - else: - merged = candles[["close"]].join(funding["rate" if "rate" in funding.columns else funding.columns[-1]], how="inner") - merged = merged.dropna() - if merged.empty: - st.info("Not enough overlapping data for overlay.") else: - price_norm = (merged["close"] / merged["close"].iloc[0]) * 100 - funding_bps = merged.iloc[:, 1] * 10000 + st.info("No sentiment data available.") + + with visual_tabs[3]: + candles = ts_cache["candles"] + if candles is not None and not candles.empty: + returns = candles["close"].pct_change().dropna() + if not returns.empty: + fig = go.Figure( + go.Histogram(x=returns * 100, nbinsx=40, marker_color="#8b5cf6") + ) + fig.update_layout( + template="plotly_dark", + margin=dict(l=0, r=0, t=35, b=0), + title=f"{pair} · Hourly Return Distribution", + xaxis_title="Return (%)", + yaxis_title="Frequency", + ) + st.plotly_chart(fig, use_container_width=True) + st.caption("Helps gauge tail risks and skew in the latest window.") + else: + st.info("Not enough data to compute returns yet.") + else: + st.info("No candles data available.") + + with visual_tabs[4]: + candles = ts_cache["candles"] + oi = ts_cache["oi"] + if candles is not None and not candles.empty: + volume_series = candles["volume"] if "volume" in candles.columns else pd.Series(0, index=candles.index) fig = go.Figure() fig.add_trace( - go.Scatter(x=merged.index, y=price_norm, name="Price (norm 100)", line=dict(color="#a855f7")) - ) - fig.add_trace( - go.Scatter( - x=merged.index, - y=funding_bps, - name="Funding (bps)", - line=dict(color="#f59e0b", dash="dot"), - yaxis="y2", + go.Bar( + x=candles.index, + y=volume_series, + name="Volume", + marker_color="#0ea5e9", + opacity=0.6, ) ) + if oi is not None and not oi.empty: + fig.add_trace( + go.Scatter( + x=oi.index, + y=oi.get("value_usd", oi.iloc[:, 0]) / 1_000_000, + name="Open Interest (M USD)", + line=dict(color="#facc15", width=2), + yaxis="y2", + ) + ) fig.update_layout( template="plotly_dark", - yaxis=dict(title="Price (index)"), - yaxis2=dict(title="Funding (bps)", overlaying="y", side="right"), margin=dict(l=0, r=0, t=35, b=0), + yaxis=dict(title="Volume"), + yaxis2=dict(title="OI (Millions USD)", overlaying="y", side="right"), + title=f"{pair} · Volume & Liquidity", ) + fig.update_xaxes(title="Time (GMT+1)", tickformat="%d %b %H:%M") st.plotly_chart(fig, use_container_width=True) + else: + st.info("Volume data unavailable.") + + with visual_tabs[5]: + st.write("Live liquidation feed via aggr.trade") + aggr_url = build_aggr_trade_url(pair) + st.link_button("Open aggr.trade in new tab", aggr_url) + st.caption("aggr.trade does not allow in-app embedding, so use the button above to view the live heatmap.") + +with st.expander("Insight Modules", expanded=True): + st.subheader("Insight Modules") + analysis_tabs = st.tabs(["Funding Overlay", "Volatility Pulse", "Sentiment Radar"]) + + with analysis_tabs[0]: + candles = ts_cache["candles"] + funding = ts_cache["funding"] + if candles is None or candles.empty or funding is None or funding.empty: + st.info("Need both candle and funding data for overlay.") + else: + merged = candles[["close"]].join(funding["rate" if "rate" in funding.columns else funding.columns[-1]], how="inner") + merged = merged.dropna() + if merged.empty: + st.info("Not enough overlapping data for overlay.") + else: + price_norm = (merged["close"] / merged["close"].iloc[0]) * 100 + funding_bps = merged.iloc[:, 1] * 10000 + fig = go.Figure() + fig.add_trace( + go.Scatter(x=merged.index, y=price_norm, name="Price (norm 100)", line=dict(color="#a855f7")) + ) + fig.add_trace( + go.Scatter( + x=merged.index, + y=funding_bps, + name="Funding (bps)", + line=dict(color="#f59e0b", dash="dot"), + yaxis="y2", + ) + ) + fig.update_layout( + template="plotly_dark", + yaxis=dict(title="Price (index)"), + yaxis2=dict(title="Funding (bps)", overlaying="y", side="right"), + margin=dict(l=0, r=0, t=35, b=0), + ) + fig.update_xaxes(title="Time (GMT+1)", tickformat="%d %b %H:%M") + st.plotly_chart(fig, use_container_width=True) + st.markdown( + """ + **How to read this:** + - *Price (norm 100)* scales the first data point to 100 so you can focus on directional drift rather than absolute price. + - *Funding (bps)* shows whether longs or shorts are paying; persistent positive funding implies long crowding, negatives imply short crowding. + - When funding rises while price stalls or drops, be cautious of long squeezes; the inverse can precede short squeezes. + - Look for divergences: price grinding higher while funding cools is healthier than price pumping on aggressive positive funding. + """ + ) + + with analysis_tabs[1]: + vol = ts_cache["vol"] + if vol is None or vol.empty: + st.info("Volatility data not available yet.") + else: + series = vol["atr"] if "atr" in vol.columns else vol.iloc[:, 0] + metric_value = float(series.iloc[-1]) + fig = go.Figure( + go.Indicator( + mode="gauge+number", + value=metric_value, + title={"text": "ATR Snapshot"}, + gauge={ + "axis": {"range": [0, series.max() * 1.2 if series.max() else 1]}, + "bar": {"color": "#38bdf8"}, + "bgcolor": "#0f172a", + "steps": [ + {"range": [0, series.median()], "color": "#1e3a8a"}, + {"range": [series.median(), series.max()], "color": "#0f766e"}, + ], + }, + ) + ) + fig.update_layout(template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0)) + st.plotly_chart(fig, use_container_width=True) + stats = series.describe().to_frame(name="ATR").T + st.dataframe(stats) st.markdown( """ **How to read this:** - - *Price (norm 100)* scales the first data point to 100 so you can focus on directional drift rather than absolute price. - - *Funding (bps)* shows whether longs or shorts are paying; persistent positive funding implies long crowding, negatives imply short crowding. - - When funding rises while price stalls or drops, be cautious of long squeezes; the inverse can precede short squeezes. - - Look for divergences: price grinding higher while funding cools is healthier than price pumping on aggressive positive funding. + - *Latest ATR* shows the current absolute true range (volatility proxy). + - *mean / std* help gauge if today's movement is above its recent norm. + - A rising *max* or widening *std* often hints at breakout-like conditions. + - If the latest ATR is near the lower quartile, conditions are typically calmer (range-trading bias). + - When ATR presses into the upper quartile, tighten risk or look for momentum setups. """ ) -with analysis_tabs[1]: - vol = ts_cache["vol"] - if vol is None or vol.empty: - st.info("Volatility data not available yet.") - else: - series = vol["atr"] if "atr" in vol.columns else vol.iloc[:, 0] - metric_value = float(series.iloc[-1]) - fig = go.Figure( - go.Indicator( - mode="gauge+number", - value=metric_value, - title={"text": "ATR Snapshot"}, - gauge={ - "axis": {"range": [0, series.max() * 1.2 if series.max() else 1]}, - "bar": {"color": "#38bdf8"}, - "bgcolor": "#0f172a", - "steps": [ - {"range": [0, series.median()], "color": "#1e3a8a"}, - {"range": [series.median(), series.max()], "color": "#0f766e"}, - ], - }, - ) - ) - fig.update_layout(template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0)) - st.plotly_chart(fig, use_container_width=True) - stats = series.describe().to_frame(name="ATR").T - st.dataframe(stats) - st.markdown( - """ - **How to read this:** - - *Latest ATR* shows the current absolute true range (volatility proxy). - - *mean / std* help gauge if today's movement is above its recent norm. - - A rising *max* or widening *std* often hints at breakout-like conditions. - - If the latest ATR is near the lower quartile, conditions are typically calmer (range-trading bias). - - When ATR presses into the upper quartile, tighten risk or look for momentum setups. - """ - ) - -with analysis_tabs[2]: - sentiment = ts_cache["sentiment"] - if sentiment is None or sentiment.empty or "keywords" not in sentiment.columns: - st.info("Sentiment keyword data is not available.") - else: - latest = sentiment.dropna(subset=["keywords"]).iloc[-1] - keywords = latest["keywords"] - if isinstance(keywords, dict) and keywords: - df_kw = ( - pd.DataFrame({"keyword": list(keywords.keys()), "count": list(keywords.values())}) - .sort_values("count", ascending=False) - .head(15) - ) - fig = go.Figure( - go.Bar( - x=df_kw["keyword"], - y=df_kw["count"], - marker_color="#f472b6", + with analysis_tabs[2]: + sentiment = ts_cache["sentiment"] + if sentiment is None or sentiment.empty or "keywords" not in sentiment.columns: + st.info("Sentiment keyword data is not available.") + else: + latest = sentiment.dropna(subset=["keywords"]).iloc[-1] + keywords = latest["keywords"] + if isinstance(keywords, dict) and keywords: + df_kw = ( + pd.DataFrame({"keyword": list(keywords.keys()), "count": list(keywords.values())}) + .sort_values("count", ascending=False) + .head(15) ) + fig = go.Figure( + go.Bar( + x=df_kw["keyword"], + y=df_kw["count"], + marker_color="#f472b6", + ) + ) + fig.update_layout(template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0)) + st.plotly_chart(fig, use_container_width=True) + else: + st.info("Latest sentiment entry does not include keyword counts.") + st.markdown( + """ + **How to read this:** + - Bars show the latest counts of sentiment keywords captured from CryptoPanic headlines. + - Liquidity-stress words (e.g., *liquidation*, *margin call*) signal risk-off chatter; bullish words (e.g., *rally*, *surge*) hint at optimism. + - Use the mix to contextualise signal bias: a short setup is stronger when bearish terms dominate, and vice versa. + - Sudden spikes in any keyword bucket often precede volatility bursts—combine with the Funding overlay for higher conviction. + """ ) - fig.update_layout(template="plotly_dark", margin=dict(l=0, r=0, t=35, b=0)) - st.plotly_chart(fig, use_container_width=True) - else: - st.info("Latest sentiment entry does not include keyword counts.") - st.markdown( - """ - **How to read this:** - - Bars show the latest counts of sentiment keywords captured from CryptoPanic headlines. - - Liquidity-stress words (e.g., *liquidation*, *margin call*) signal risk-off chatter; bullish words (e.g., *rally*, *surge*) hint at optimism. - - Use the mix to contextualise signal bias: a short setup is stronger when bearish terms dominate, and vice versa. - - Sudden spikes in any keyword bucket often precede volatility bursts—combine with the Funding overlay for higher conviction. - """ - ) st.divider() st.caption("Configure pairs, Telegram alerts, and scheduler in .env. Add API keys for live data.") From 0645ed8b92be81d79a65da6c72d3af33222682fa Mon Sep 17 00:00:00 2001 From: skymike Date: Mon, 3 Nov 2025 02:54:06 +0100 Subject: [PATCH 39/43] Refactor scoring notes display in Hot Signals section Updated the presentation of scoring notes in the Hot Signals section of the UI. Changed the layout to use bullet points for better readability and clarity, enhancing the overall user experience. --- services/ui/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/ui/app.py b/services/ui/app.py index 00fff2ed..b2c4bd07 100644 --- a/services/ui/app.py +++ b/services/ui/app.py @@ -735,10 +735,10 @@ def fetch_alt_global() -> Optional[dict]: ) if explanations: - with st.expander("How this profile scores signals", expanded=False): - for key, text in explanations.items(): - label = key.replace("_", " ").title() - st.markdown(f"**{label}:** {text}") + st.markdown("**Scoring Notes**") + for key, note in explanations.items(): + label = key.replace("_", " ").title() + st.markdown(f"- **{label}:** {note}") with st.container(): selected = st.selectbox( From 73a5689c12c8887c111016d9552178a7beeae310 Mon Sep 17 00:00:00 2001 From: skymike Date: Mon, 10 Nov 2025 00:27:23 +0100 Subject: [PATCH 40/43] chore: snapshot current state before merging Open-Trader fork --- .github/workflows/crypto-worker.yml | 72 +++--- .gitignore | 14 +- RAILWAY_DEPLOY.md | 340 +++++++++++++-------------- railway.json | 2 +- railway.toml | 18 +- services/__init__.py | 4 +- services/api/Dockerfile | 44 ++-- services/api/railway.json | 26 +- services/common/__init__.py | 4 +- services/common/requirements.txt | 16 +- services/common/schema_timescale.sql | 18 +- services/ui/railway.json | 26 +- services/worker/requirements.txt | 42 ++-- services_common/__init__.py | 76 +++--- 14 files changed, 351 insertions(+), 351 deletions(-) diff --git a/.github/workflows/crypto-worker.yml b/.github/workflows/crypto-worker.yml index 6733c25d..523a26e0 100644 --- a/.github/workflows/crypto-worker.yml +++ b/.github/workflows/crypto-worker.yml @@ -1,41 +1,41 @@ -name: Crypto Data Worker - +name: Crypto Data Worker + on: schedule: - cron: '*/15 * * * *' # Every 15 minutes - workflow_dispatch: - -jobs: - worker: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Cache Python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/api/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - pip install -r services/worker/requirements.txt - pip install -r services/api/requirements.txt - - - name: Run Data Worker - env: - # DATABASE_URL should be set from Railway PostgreSQL service - # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL - # Add it to GitHub Secrets as DATABASE_URL - DATABASE_URL: ${{ secrets.DATABASE_URL }} + workflow_dispatch: + +jobs: + worker: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Cache Python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('services/worker/requirements.txt', 'services/api/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r services/worker/requirements.txt + pip install -r services/api/requirements.txt + + - name: Run Data Worker + env: + # DATABASE_URL should be set from Railway PostgreSQL service + # Go to Railway dashboard → PostgreSQL → Variables → Copy DATABASE_URL + # Add it to GitHub Secrets as DATABASE_URL + DATABASE_URL: ${{ secrets.DATABASE_URL }} SYMBOLS: binance:BTC/USDT,binance:ETH/USDT,binance:SOL/USDT,binance:BNB/USDT,binance:XRP/USDT,binance:DOGE/USDT,binance:ADA/USDT,binance:AVAX/USDT,binance:TRX/USDT,binance:DOT/USDT,binance:LINK/USDT,binance:MATIC/USDT,binance:UNI/USDT,binance:APT/USDT,binance:ARB/USDT,binance:ATOM/USDT,binance:OP/USDT,binance:SEI/USDT,binance:NEAR/USDT,binance:INJ/USDT,bybit:BTC/USDT,bybit:ETH/USDT,bybit:SOL/USDT,bybit:XRP/USDT,bybit:DOGE/USDT,bybit:ADA/USDT,bybit:LINK/USDT,bybit:MATIC/USDT,bybit:NEAR/USDT,bybit:APT/USDT SCHEDULE_MINUTES: 15 - run: python services/worker/run_worker.py + run: python services/worker/run_worker.py diff --git a/.gitignore b/.gitignore index 91bf069a..6c517101 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ -__pycache__/ -*.pyc -.env -db_data/ -.idea/ -.vscode/ -.DS_Store +__pycache__/ +*.pyc +.env +db_data/ +.idea/ +.vscode/ +.DS_Store diff --git a/RAILWAY_DEPLOY.md b/RAILWAY_DEPLOY.md index 4685caac..8595f911 100644 --- a/RAILWAY_DEPLOY.md +++ b/RAILWAY_DEPLOY.md @@ -1,119 +1,119 @@ -# Railway Deployment Guide - -This guide provides detailed instructions for deploying the Crypto Risk Dashboard to Railway. - -## Prerequisites - -- GitHub account with this repository -- Railway account (sign up at [railway.app](https://railway.app)) -- Railway CLI (optional, for local testing) - -## Architecture - -- **Railway Services**: API, UI, and PostgreSQL database +# Railway Deployment Guide + +This guide provides detailed instructions for deploying the Crypto Risk Dashboard to Railway. + +## Prerequisites + +- GitHub account with this repository +- Railway account (sign up at [railway.app](https://railway.app)) +- Railway CLI (optional, for local testing) + +## Architecture + +- **Railway Services**: API, UI, and PostgreSQL database - **GitHub Actions**: Worker service (runs every 15 minutes) - -## Deployment Steps - -### 1. Create Railway Project - -1. Go to [railway.app](https://railway.app) and sign in with GitHub -2. Click "New Project" -3. Select "Empty Project" or "Deploy from GitHub repo" -4. Connect your GitHub repository - -### 2. Create PostgreSQL Database - -1. In your Railway project, click "+ New" -2. Select "Database" → "Add PostgreSQL" -3. Railway will automatically provision a PostgreSQL instance -4. Note the service name (e.g., "Postgres") - -### 3. Deploy API Service - -1. Click "+ New" → "GitHub Repo" -2. Select your repository -3. In the service settings: - - **Root Directory**: Set to `services/api` - - **Build Command**: Leave empty (uses Dockerfile) - - Railway will detect the Dockerfile automatically - -4. **Add Environment Variables**: - - Go to "Variables" tab - - Click "New Variable" - - `DATABASE_URL`: Click "Reference Variable" → Select your PostgreSQL service → Select `DATABASE_URL` + +## Deployment Steps + +### 1. Create Railway Project + +1. Go to [railway.app](https://railway.app) and sign in with GitHub +2. Click "New Project" +3. Select "Empty Project" or "Deploy from GitHub repo" +4. Connect your GitHub repository + +### 2. Create PostgreSQL Database + +1. In your Railway project, click "+ New" +2. Select "Database" → "Add PostgreSQL" +3. Railway will automatically provision a PostgreSQL instance +4. Note the service name (e.g., "Postgres") + +### 3. Deploy API Service + +1. Click "+ New" → "GitHub Repo" +2. Select your repository +3. In the service settings: + - **Root Directory**: Set to `services/api` + - **Build Command**: Leave empty (uses Dockerfile) + - Railway will detect the Dockerfile automatically + +4. **Add Environment Variables**: + - Go to "Variables" tab + - Click "New Variable" + - `DATABASE_URL`: Click "Reference Variable" → Select your PostgreSQL service → Select `DATABASE_URL` - `SYMBOLS`: Default includes top 30 perp pairs (e.g., `binance:BTC/USDT,...,bybit:APT/USDT`). Override if you want a smaller slice. - -5. **Generate Public URL**: - - Go to "Settings" → "Networking" - - Click "Generate Domain" - - Note your API URL (e.g., `https://crypto-risk-api-production.up.railway.app`) - -### 4. Deploy UI Service - -1. Click "+ New" → "GitHub Repo" -2. Select the same repository -3. In the service settings: - - **Root Directory**: Set to `services/ui` - - **Build Command**: Leave empty (uses Dockerfile) - -4. **Add Environment Variables**: - - `API_CANDIDATES`: Your API URL from step 3 (e.g., `https://crypto-risk-api-production.up.railway.app,http://api:8000,http://localhost:8000`) - - `DATABASE_URL`: Reference from PostgreSQL service (same as API service) - -5. **Generate Public URL**: - - Generate a domain for UI service - - Note your UI URL - -### 5. Configure GitHub Actions Worker - -The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). - -1. **Get Database Connection String**: - - In Railway → Your PostgreSQL service → "Variables" tab - - Copy the `DATABASE_URL` value - - It should look like: `postgresql://user:password@host:port/dbname` - -2. **Add GitHub Secret**: - - Go to your GitHub repository - - Settings → Secrets and variables → Actions - - Click "New repository secret" - - Name: `DATABASE_URL` - - Value: Paste your Railway PostgreSQL `DATABASE_URL` - - Click "Add secret" - -3. **Verify Workflow**: - - Go to Actions tab in GitHub + +5. **Generate Public URL**: + - Go to "Settings" → "Networking" + - Click "Generate Domain" + - Note your API URL (e.g., `https://crypto-risk-api-production.up.railway.app`) + +### 4. Deploy UI Service + +1. Click "+ New" → "GitHub Repo" +2. Select the same repository +3. In the service settings: + - **Root Directory**: Set to `services/ui` + - **Build Command**: Leave empty (uses Dockerfile) + +4. **Add Environment Variables**: + - `API_CANDIDATES`: Your API URL from step 3 (e.g., `https://crypto-risk-api-production.up.railway.app,http://api:8000,http://localhost:8000`) + - `DATABASE_URL`: Reference from PostgreSQL service (same as API service) + +5. **Generate Public URL**: + - Generate a domain for UI service + - Note your UI URL + +### 5. Configure GitHub Actions Worker + +The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). + +1. **Get Database Connection String**: + - In Railway → Your PostgreSQL service → "Variables" tab + - Copy the `DATABASE_URL` value + - It should look like: `postgresql://user:password@host:port/dbname` + +2. **Add GitHub Secret**: + - Go to your GitHub repository + - Settings → Secrets and variables → Actions + - Click "New repository secret" + - Name: `DATABASE_URL` + - Value: Paste your Railway PostgreSQL `DATABASE_URL` + - Click "Add secret" + +3. **Verify Workflow**: + - Go to Actions tab in GitHub - The "Crypto Data Worker" workflow should run automatically every 15 minutes - - You can manually trigger it using "workflow_dispatch" - -### 6. Verify Deployment - -1. **Check API Health**: - - Visit your API URL: `https://your-api-url.railway.app/health` - - Should return: `{"ok": true}` - -2. **Check UI**: - - Visit your UI URL - - Dashboard should load and show pair selection - -3. **Check Worker**: - - Go to GitHub Actions - - Check recent runs are successful - - Wait a few minutes for initial data ingestion - -## Environment Variables Reference - -### API Service -- `DATABASE_URL`: PostgreSQL connection string (referenced from database service) + - You can manually trigger it using "workflow_dispatch" + +### 6. Verify Deployment + +1. **Check API Health**: + - Visit your API URL: `https://your-api-url.railway.app/health` + - Should return: `{"ok": true}` + +2. **Check UI**: + - Visit your UI URL + - Dashboard should load and show pair selection + +3. **Check Worker**: + - Go to GitHub Actions + - Check recent runs are successful + - Wait a few minutes for initial data ingestion + +## Environment Variables Reference + +### API Service +- `DATABASE_URL`: PostgreSQL connection string (referenced from database service) - `SYMBOLS`: Comma-separated trading pairs (default top 30, e.g., `binance:BTC/USDT,...,bybit:APT/USDT`) -- `PORT`: Automatically set by Railway (don't set manually) - -### UI Service -- `API_CANDIDATES`: Comma-separated API URLs to try (include your Railway API URL) -- `DATABASE_URL`: PostgreSQL connection string (referenced from database service) -- `PORT`: Automatically set by Railway (don't set manually) - +- `PORT`: Automatically set by Railway (don't set manually) + +### UI Service +- `API_CANDIDATES`: Comma-separated API URLs to try (include your Railway API URL) +- `DATABASE_URL`: PostgreSQL connection string (referenced from database service) +- `PORT`: Automatically set by Railway (don't set manually) + ### Worker (GitHub Actions) - `DATABASE_URL`: PostgreSQL connection string (from GitHub Secrets) - `SYMBOLS`: Trading pairs (set in workflow file; defaults to top 30) @@ -121,65 +121,65 @@ The worker runs via GitHub Actions to keep costs low (GitHub Actions free tier). - `TELEGRAM_BOT_TOKEN`: *(optional)* Telegram bot token for alerting top signals - `TELEGRAM_CHAT_ID`: *(optional)* Destination chat/channel ID for Telegram notifications - `SIGNAL_PROFILE`: *(optional)* Default worker profile (`conservative`, `balanced`, or `aggressive`) controlling signal strictness - -## Troubleshooting - -### API not connecting to database -- Verify `DATABASE_URL` is correctly referenced from PostgreSQL service -- Check PostgreSQL service is running in Railway dashboard -- Ensure database has been initialized (first worker run will do this) - -### UI can't find API -- Verify `API_CANDIDATES` includes your Railway API URL -- Check API service is running and health endpoint works -- Ensure API URL is publicly accessible (not internal Railway hostname) - -### Worker not running -- Check GitHub Actions workflow is enabled -- Verify `DATABASE_URL` secret is set correctly -- Check workflow logs for errors -- Ensure repository has Actions enabled - -### Service not building -- Check Dockerfile exists in service directory -- Verify Root Directory is set correctly (`services/api` or `services/ui`) -- Check build logs in Railway dashboard - -## Railway CLI (Optional) - -You can also deploy using Railway CLI: - -```bash -# Install Railway CLI -npm i -g @railway/cli - -# Login -railway login - -# Link project -railway link - -# Deploy -railway up -``` - -## Cost Considerations - -- **Railway Free Tier**: $5 credit per month - - Perfect for testing and small deployments - - May need to upgrade for production traffic + +## Troubleshooting + +### API not connecting to database +- Verify `DATABASE_URL` is correctly referenced from PostgreSQL service +- Check PostgreSQL service is running in Railway dashboard +- Ensure database has been initialized (first worker run will do this) + +### UI can't find API +- Verify `API_CANDIDATES` includes your Railway API URL +- Check API service is running and health endpoint works +- Ensure API URL is publicly accessible (not internal Railway hostname) + +### Worker not running +- Check GitHub Actions workflow is enabled +- Verify `DATABASE_URL` secret is set correctly +- Check workflow logs for errors +- Ensure repository has Actions enabled + +### Service not building +- Check Dockerfile exists in service directory +- Verify Root Directory is set correctly (`services/api` or `services/ui`) +- Check build logs in Railway dashboard + +## Railway CLI (Optional) + +You can also deploy using Railway CLI: + +```bash +# Install Railway CLI +npm i -g @railway/cli + +# Login +railway login + +# Link project +railway link + +# Deploy +railway up +``` + +## Cost Considerations + +- **Railway Free Tier**: $5 credit per month + - Perfect for testing and small deployments + - May need to upgrade for production traffic - **GitHub Actions**: Free tier includes 2,000 minutes/month - Worker runs every 15 minutes = ~2,880 runs/month - Each run takes ~1-2 minutes = ~2,880–5,760 minutes/month - May need to optimize or reduce frequency for free tier - -## Updating Services - -Railway auto-deploys on git push to your main branch. To update: - -1. Make changes to your code -2. Commit and push to main branch -3. Railway will automatically rebuild and deploy - -You can also manually trigger deployments from Railway dashboard. - + +## Updating Services + +Railway auto-deploys on git push to your main branch. To update: + +1. Make changes to your code +2. Commit and push to main branch +3. Railway will automatically rebuild and deploy + +You can also manually trigger deployments from Railway dashboard. + diff --git a/railway.json b/railway.json index 7a1c29b0..34d91d14 100644 --- a/railway.json +++ b/railway.json @@ -10,4 +10,4 @@ "restartPolicyMaxRetries": 10 } } - + diff --git a/railway.toml b/railway.toml index 6f86e95e..3b8da614 100644 --- a/railway.toml +++ b/railway.toml @@ -1,9 +1,9 @@ -# Railway configuration for Crypto Risk Dashboard -# This file defines services for Railway deployment - -[build] -builder = "DOCKERFILE" - -# Service configuration is typically done via Railway dashboard -# or through railway.json in each service directory - +# Railway configuration for Crypto Risk Dashboard +# This file defines services for Railway deployment + +[build] +builder = "DOCKERFILE" + +# Service configuration is typically done via Railway dashboard +# or through railway.json in each service directory + diff --git a/services/__init__.py b/services/__init__.py index 6d31e90a..ae770d64 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -1,2 +1,2 @@ -# Services package - +# Services package + diff --git a/services/api/Dockerfile b/services/api/Dockerfile index eafcb7d3..ffc4eedb 100644 --- a/services/api/Dockerfile +++ b/services/api/Dockerfile @@ -1,22 +1,22 @@ -FROM python:3.11-slim -WORKDIR /app - -# Copy services package structure (needed for services.common imports) -# Note: Railway uses repo root as build context when Root Directory is set -COPY services/__init__.py /app/services/__init__.py -COPY services/common /app/services/common - -# Copy the root-level services_common namespace package (not the old shim) -# This makes services.common importable as services_common -COPY services_common /app/services_common - -# Copy API service files (excluding the old services_common shim directory) -COPY services/api/requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt - -# Copy API service files, but exclude the old services_common shim -COPY services/api/main.py /app/main.py -COPY services/api/railway.json /app/railway.json - -# Use PORT env var (Railway) or default to 8000 -CMD sh -c "uvicorn main:app --host 0.0.0.0 --port ${PORT:-8000}" +FROM python:3.11-slim +WORKDIR /app + +# Copy services package structure (needed for services.common imports) +# Note: Railway uses repo root as build context when Root Directory is set +COPY services/__init__.py /app/services/__init__.py +COPY services/common /app/services/common + +# Copy the root-level services_common namespace package (not the old shim) +# This makes services.common importable as services_common +COPY services_common /app/services_common + +# Copy API service files (excluding the old services_common shim directory) +COPY services/api/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +# Copy API service files, but exclude the old services_common shim +COPY services/api/main.py /app/main.py +COPY services/api/railway.json /app/railway.json + +# Use PORT env var (Railway) or default to 8000 +CMD sh -c "uvicorn main:app --host 0.0.0.0 --port ${PORT:-8000}" diff --git a/services/api/railway.json b/services/api/railway.json index a5cec656..d6358aa6 100644 --- a/services/api/railway.json +++ b/services/api/railway.json @@ -1,13 +1,13 @@ -{ - "$schema": "https://railway.app/railway.schema.json", - "build": { - "builder": "DOCKERFILE", - "dockerfilePath": "services/api/Dockerfile" - }, - "deploy": { - "startCommand": "uvicorn main:app --host 0.0.0.0 --port $PORT", - "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10 - } -} - +{ + "$schema": "https://railway.app/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "services/api/Dockerfile" + }, + "deploy": { + "startCommand": "uvicorn main:app --host 0.0.0.0 --port $PORT", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 10 + } +} + diff --git a/services/common/__init__.py b/services/common/__init__.py index 851daabf..76f3653b 100644 --- a/services/common/__init__.py +++ b/services/common/__init__.py @@ -1,2 +1,2 @@ -# services.common package - +# services.common package + diff --git a/services/common/requirements.txt b/services/common/requirements.txt index e1365d15..5cf09589 100644 --- a/services/common/requirements.txt +++ b/services/common/requirements.txt @@ -1,8 +1,8 @@ -# Core dependencies for services/common module -psycopg2-binary==2.9.9 -pandas==2.2.3 -numpy==2.1.2 -requests==2.32.3 -ccxt==4.4.6 -python-dotenv==1.0.1 - +# Core dependencies for services/common module +psycopg2-binary==2.9.9 +pandas==2.2.3 +numpy==2.1.2 +requests==2.32.3 +ccxt==4.4.6 +python-dotenv==1.0.1 + diff --git a/services/common/schema_timescale.sql b/services/common/schema_timescale.sql index 1b770a9b..a7717919 100644 --- a/services/common/schema_timescale.sql +++ b/services/common/schema_timescale.sql @@ -23,11 +23,11 @@ WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', ts) AS bucket, pair, - first(open, ts) AS o, - max(high) AS h, - min(low) AS l, - last(close, ts) AS c, - sum(volume) AS v + first(open, ts) AS o, + max(high) AS h, + min(low) AS l, + last(close, ts) AS c, + sum(volume) AS v FROM candles GROUP BY 1, 2; @@ -37,7 +37,7 @@ SELECT add_continuous_aggregate_policy( start_offset => INTERVAL '3 days', end_offset => INTERVAL '5 minutes', schedule_interval => INTERVAL '15 minutes' -); - --- Monitor hypertable chunk sizes and adjust chunk_time_interval as needed --- Example: ALTER TABLE candles SET (timescaledb.chunk_time_interval = INTERVAL '1 day'); +); + +-- Monitor hypertable chunk sizes and adjust chunk_time_interval as needed +-- Example: ALTER TABLE candles SET (timescaledb.chunk_time_interval = INTERVAL '1 day'); diff --git a/services/ui/railway.json b/services/ui/railway.json index c062f872..3cce12e0 100644 --- a/services/ui/railway.json +++ b/services/ui/railway.json @@ -1,13 +1,13 @@ -{ - "$schema": "https://railway.app/railway.schema.json", - "build": { - "builder": "DOCKERFILE", - "dockerfilePath": "services/ui/Dockerfile" - }, - "deploy": { - "startCommand": "streamlit run app.py --server.port=$PORT --server.address=0.0.0.0", - "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10 - } -} - +{ + "$schema": "https://railway.app/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "services/ui/Dockerfile" + }, + "deploy": { + "startCommand": "streamlit run app.py --server.port=$PORT --server.address=0.0.0.0", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 10 + } +} + diff --git a/services/worker/requirements.txt b/services/worker/requirements.txt index 9d2300aa..c720b48b 100644 --- a/services/worker/requirements.txt +++ b/services/worker/requirements.txt @@ -1,21 +1,21 @@ -fastapi==0.115.2 -uvicorn[standard]==0.30.6 -pydantic==2.9.2 -python-dotenv==1.0.1 -psycopg2-binary==2.9.9 -SQLAlchemy==2.0.36 -alembic==1.13.2 -redis==5.0.8 -celery==5.4.0 -pandas==2.2.3 -numpy==2.1.2 -plotly==5.24.1 -matplotlib==3.9.2 -ccxt==4.4.6 -scikit-learn==1.5.2 -textblob==0.18.0.post0 -nltk==3.9.1 -requests==2.32.3 -beautifulsoup4==4.12.3 -streamlit==1.39.0 -streamlit_echarts==0.4.0 +fastapi==0.115.2 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +python-dotenv==1.0.1 +psycopg2-binary==2.9.9 +SQLAlchemy==2.0.36 +alembic==1.13.2 +redis==5.0.8 +celery==5.4.0 +pandas==2.2.3 +numpy==2.1.2 +plotly==5.24.1 +matplotlib==3.9.2 +ccxt==4.4.6 +scikit-learn==1.5.2 +textblob==0.18.0.post0 +nltk==3.9.1 +requests==2.32.3 +beautifulsoup4==4.12.3 +streamlit==1.39.0 +streamlit_echarts==0.4.0 diff --git a/services_common/__init__.py b/services_common/__init__.py index 709b12ec..1b420178 100644 --- a/services_common/__init__.py +++ b/services_common/__init__.py @@ -1,41 +1,41 @@ -""" -services_common namespace package -This makes services/common importable as services_common -""" -import sys -from pathlib import Path -import importlib - -# Ensure repo root is in path -_repo_root = Path(__file__).resolve().parent -if str(_repo_root) not in sys.path: - sys.path.insert(0, str(_repo_root)) - -# Import services.common modules and make them available as services_common.* -# Do this eagerly so modules are available immediately +""" +services_common namespace package +This makes services/common importable as services_common +""" +import sys +from pathlib import Path +import importlib + +# Ensure repo root is in path +_repo_root = Path(__file__).resolve().parent +if str(_repo_root) not in sys.path: + sys.path.insert(0, str(_repo_root)) + +# Import services.common modules and make them available as services_common.* +# Do this eagerly so modules are available immediately _common_modules = [ 'config', 'db', 'ingest', 'signals', 'schema', 'notifications' ] - -for mod_name in _common_modules: - try: - mod = importlib.import_module(f'services.common.{mod_name}') - # Register in sys.modules for direct import access - sys.modules[f'services_common.{mod_name}'] = mod - except Exception as e: - # Log the error but continue - this helps with debugging - print(f"Warning: Could not import services.common.{mod_name}: {e}", file=sys.stderr) - -# Handle adapters package -try: - adapters = importlib.import_module('services.common.adapters') - sys.modules['services_common.adapters'] = adapters - for adapter in ['exchanges', 'open_interest', 'volatility', 'sentiment', 'headlines']: - try: - mod = importlib.import_module(f'services.common.adapters.{adapter}') - sys.modules[f'services_common.adapters.{adapter}'] = mod - except Exception as e: - print(f"Warning: Could not import services.common.adapters.{adapter}: {e}", file=sys.stderr) -except Exception as e: - print(f"Warning: Could not import services.common.adapters: {e}", file=sys.stderr) - + +for mod_name in _common_modules: + try: + mod = importlib.import_module(f'services.common.{mod_name}') + # Register in sys.modules for direct import access + sys.modules[f'services_common.{mod_name}'] = mod + except Exception as e: + # Log the error but continue - this helps with debugging + print(f"Warning: Could not import services.common.{mod_name}: {e}", file=sys.stderr) + +# Handle adapters package +try: + adapters = importlib.import_module('services.common.adapters') + sys.modules['services_common.adapters'] = adapters + for adapter in ['exchanges', 'open_interest', 'volatility', 'sentiment', 'headlines']: + try: + mod = importlib.import_module(f'services.common.adapters.{adapter}') + sys.modules[f'services_common.adapters.{adapter}'] = mod + except Exception as e: + print(f"Warning: Could not import services.common.adapters.{adapter}: {e}", file=sys.stderr) +except Exception as e: + print(f"Warning: Could not import services.common.adapters: {e}", file=sys.stderr) + From fd7db60b86a779c3631d338190e7154cf6da53f8 Mon Sep 17 00:00:00 2001 From: skymike Date: Mon, 10 Nov 2025 00:30:09 +0100 Subject: [PATCH 41/43] Add Python 3.12 compiled cache files Added .pyc files generated by Python 3.12 for various modules in the services directory. These files are typically not committed to version control and may have been added unintentionally. --- services/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 177 bytes services/api/__pycache__/main.cpython-312.pyc | Bin 0 -> 3532 bytes .../common/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 184 bytes .../common/__pycache__/config.cpython-312.pyc | Bin 0 -> 2589 bytes services/common/__pycache__/db.cpython-312.pyc | Bin 0 -> 4406 bytes .../__pycache__/notifications.cpython-312.pyc | Bin 0 -> 3207 bytes .../common/__pycache__/signals.cpython-312.pyc | Bin 0 -> 8375 bytes .../__pycache__/sentiment.cpython-312.pyc | Bin 0 -> 4277 bytes services/ui/__pycache__/app.cpython-312.pyc | Bin 0 -> 50875 bytes .../__pycache__/run_worker.cpython-312.pyc | Bin 0 -> 2400 bytes 10 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 services/__pycache__/__init__.cpython-312.pyc create mode 100644 services/api/__pycache__/main.cpython-312.pyc create mode 100644 services/common/__pycache__/__init__.cpython-312.pyc create mode 100644 services/common/__pycache__/config.cpython-312.pyc create mode 100644 services/common/__pycache__/db.cpython-312.pyc create mode 100644 services/common/__pycache__/notifications.cpython-312.pyc create mode 100644 services/common/__pycache__/signals.cpython-312.pyc create mode 100644 services/common/adapters/__pycache__/sentiment.cpython-312.pyc create mode 100644 services/ui/__pycache__/app.cpython-312.pyc create mode 100644 services/worker/__pycache__/run_worker.cpython-312.pyc diff --git a/services/__pycache__/__init__.cpython-312.pyc b/services/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4122653793fd3636c4d700a37781b9ca78d18935 GIT binary patch literal 177 zcmX@j%ge<81ZNyrGsS@PV-N=&d}aZPOlPQM&}8&m$xy@uo0*)En3|*SlAm0fo0?ZrtnZ##;!&EUpIlT~P?E1(lv$juo03?Zk(8fUl%fyR yQ7&-6`t8YiXthB5+zcS<*fV@hjuJEmhH4f`eR#iD#w;$Nk$P00kPsPrImI` zdUh#CECn#FfeJx@%0OE?sEep5&?<1yx~Jxt6a{+dg$mu4nJP$vo_eF~6h(dNn_Vs` zyYT=!Kl|RidGqFb-^^b^p>_o2kDvPV@iv71$p&75u~Q!Y79(^EsYt~JO0t$Su*oGk zlTY%dkQ6Zc#~Y$4B_-3B^f_a}Xfypuztb0ub~BI+n89Sw3?)O(KgsAY!^tp49MU+| zr?$QCzstD`COhF@_-VULP?PJv*FVrIKrvL_sACpB03v zj6M%`ZX8;FsUpNk7yKwnBY8z9)>kk7ENHOBZD`%H)IhU0X078GH%Udu66}yqXBMc> zS~cf$hUGZwtVOy(ne=)U8(C}@%&G35%-ov0Iadh`RfHkev$l~&XldKL0>Mvfsxz7T z@+6nh4K2Ob7ZD6O0qb(|x`fE_5?X5H63CjRh5Iq|V=m1tpfvY7gkp^=aZAj?6UD>t zP44Ze0J>{jHf#%?(vsx$TJ0LIqcuTAYhsBXL$5!_Y}NU$2Q#r9@@9Z#k3IN1{0w-D9gnZ0gfRICKTrBW zlci51vd)+W_kug&qoS2JbQ>C*=y?)@DJ0G+wgt*BXf|P>O!(ZRWF{%j({|_dQ-*HY zR8SRLfi%;soUx**sm6g(sj)6l3zFU9fPx0Vs+URR2uRtg)&^8%^u;&6y?OOFbHALs z*Ihk)`frgln|NdB){&b>ewwI8&irZS&vPHml`qx;q4HQQHc%0wwNUH<;>Esoyc2nL z^V-&tYUJd)R2w+<@xa*D%I}Kr6+cK+N5(2`@mg1WU0A>Jmapyu{dzl!#%tXNKkgph zyuK4Zw)Ju~ernez1f@C>MCrc|!l>&-ivjItv2%j(Sjsfvw+P}}$Zi79gAYb_6I^@# zv3X9hSxlU=zJ8%dGhR~)T`4|7bY%^Cq%*w5oUuhL;+n` zRD$qo*_9_!kw#LC2u&qh0ETmjkUS4lSk_z5KvhONLgdEOju5>s^zI0;JKVeecl?zD zC#&(%zY3?^H1&%B9^Xz+g2C7@=t8>$CW^`+tjQUjm4hQ5X z6i?3*ePKZ(vLep|dmq;_nH;fY*8|zv0t{q(QIlKV3{&Y^K`Ur#@z7;wub#eTE5z1R zHe-)A5SRoKyI@hD33q_9pt0*jQB`MKPKXki9EW{aY=^Q7;ZP%vt}kR1B*&PF8_2qZ z#GbzkgWJO3{rw~Ng%h>j{gq=^wgb}@VcHF3bi~$SJ&owV z&`sS=H<1JjFamM{`)la9H-*Zpf-0)g9dV5>p%TA>iNGEsjHpZSrw=rNp!_A_=SXe; z1-~Y?@LCc}xWuV}J6wyTTF3>OYEmB;YsRCD_Pah zEE>wdr42z#5yfW1TwY_Bv#kMp*tC5mXTYrse6&jamX>89%G!YPufYP;*O&_+*C`LG zgz+O4H9~Tv*ipzXXLHxHvKzu;ti8cG^33?9@$<9tQTf93)MdFi+Z257yW`X2G7Gi* zP5JP1tdp>s(y8gO@oD+oYqD+0W8*XDn}SOdmnUXH-#MMegak)UgENHTo%)ED&JmTp z5~Lg)Fl>Q?5af)43GWGkydW@Qba0G)H_gH z$Hr^hzWue%Sfwvf2_LPr9jyg>HZt46!SZ;W!=YF$wr^trcyYg2=S6>8`RtcL)ZVo* zbX(oJy4^Ec^`9!A+ldX_KKJg$cP>_ANA7j~q5t>&AAbE_fAxjS+kwf7FzLWPf%PvT zixT_DQE&=Q6drc)j3Ertgkg;^ilklM8wcK3rkTqQtMg8{9JZsqPf8m}$XJlZo)mhQ zAG^<-tIQ?J|1;vgV>wU%DS!^xLEN&Q^oQw*M+C>hE+$b-8|^FTj>YRn9Un< IRUCNy4-vIHQ~&?~ literal 0 HcmV?d00001 diff --git a/services/common/__pycache__/__init__.cpython-312.pyc b/services/common/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e68c3a6bab96778c3ab70468479e975a7bf989cc GIT binary patch literal 184 zcmX@j%ge<81ZNyrGi8DFV-N=&d}aZPOlPQM&}8&m$xy@uo0*)En3|*SlAm0fo0?ZrtnZ##;!&EUpIlT~P?E1(lv$juo03?Zk(8fUl%fyR zQK FECA?{FnRz0 literal 0 HcmV?d00001 diff --git a/services/common/__pycache__/config.cpython-312.pyc b/services/common/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78a24962997d4affb0d95a2960fecc78c76d6157 GIT binary patch literal 2589 zcmc&$&2JM&6rcU@t{um*<0Pg~4n@!ir!*#fMbJQH$8K;E?9_GwSgKZzcav_gyh#)3V%S4P^oIGs%q#~kWO6*si)p7f>gn&Gi$HE4uxAs+Bfh0 z=DpdszxjCklf%&gaC1K5c3eFOERKGV&-U(yTK;eJmuy8CL<9qg=}I%|3a$ zm{ZtaU<>qB98k1qb~8>Ryc=!7X=;tWU@>Pk`y!j83v`jyx=pW_aD`8Tc_d%r$%*_c zdRj%>D|&W__PS@!D`wm&I?NdJt6!3Nm*_Mj3hX`CCSQG19PdmDKSgS}N%M9`QqS-X zo|SW{Tv|F6NR0ZXBL1Mi4Vdv?XhjmS zHXs;Hw0ca2BNMI2Ie#MDmj84l-0I_>YMaI%Ys=rCh@NYWiMD-EJk&O6B;=2^hJ+*I zH6TALYfZcwW`kA3Y;bCr4FV0bi9W+_kY|`pG#GX>2?o%VTs6MNzzgaS;^P<31)`I2 zAlqggS7CeHH zF8rwb@Nv5aDlBjFRv2J;t?6<2-S>a+|1$RD*w5kB%cbSD z<=go!&&;(b7G-yD$-CxV59{v!YZJfo?(P1e$Nj?(`-i__^!|}Y?(w_b_o?ss2mE)o z)uGbx+VJg@Tki1{9*eTQYi0S6@Aa(ulB+1zP;-oKkOS^d9Uo*bLZGsr@uVC z;n80|tb2xTjy-YjTj9(0?y}2WvaQ+HeH({#*Kp0++Vl2CMt8ka?s9!5-;~!KcNg@o zp@!=L-Q_F0dOi=7!fWAmWn)h7J+8Y>nAuNk9XzAE-`#fYdgiotaur~;aL=FifL-UQ z=TAA%GeN0@z%Tkj2iSXVEFKK>g}m&yz>THX0uLJO)_<5J6_Qe0T~5L?>?Oh|@noJC z){*L8$QgJHo1|c?gIH7;ilY7iwm-qXCsfBjG*9iS04#PGK-(*Z>E26mt2564?u_~V E1tLdj4FCWD literal 0 HcmV?d00001 diff --git a/services/common/__pycache__/db.cpython-312.pyc b/services/common/__pycache__/db.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3c31fabe5344100ba91480ee7d65b0b9c53bbcc GIT binary patch literal 4406 zcma)9U2GFq7QXZ68GHPbIDZgAaY_nI0_LAZOKF?f0oqLiG@)HAx|W%l1ZN%Fy)$Wu zJ6=V*JS2iDj#!EGPYv2iqzWrh?MlnOHL!2{5*oIIQBb?uec88^R6?XLd+v!B;HC@!_ zk%4fH&N;e1m&2{h26C*s(At)|;5^l5U5~S}=Q6GLNex+*Y5i+zE7sHopVnTdr;G2` zg(Rla^hR5|TiaT@+T*9*>X0~-J(tmRlg(zZZn~7S=|sP3vIClmO(C1mw80EkB#%jF zG}G0q>T3FZkZVJ7CfmEmPH$hV#QBGp)M|s+y`$*KS8f6xY!St1)YsJh!26dj& zXbL6Kbn#Rx1tATezaB!H(Qc$sgYevGM8V&s;CbdW8l+ashmbA*lIoNg%Ymj#9>7`# zn|v{~B-UiK_fn=So4|>FNHOATlOa|t{Ll~#Op+YG;1Cjd+HoLo2X6nxxap70f1>w4aFVO1Z%=_f9Uq^p(o7?|f>l3u!^KJsR(23!Sg zP}iyHqLoYo57PI1SeBTS%)-AX{I=kRT2QHWFeP3LWCVGCpJJNUV+Hes39mC z7f_yE!BK-!Xp)F)#k+%(dCs7sXv{Un%S%I%DW5>WV}pU5+6u-8X2tNGI+AD z^Xxc%DZs)rc^_c==h0lHhhXXJon7s3 zb;HzsJd~qD+m@;AxoXK$z1`Yzs=X`pl4h-PoHa=%dOd333}geiV?mqQDkTj`LT%7Tz20{_x$qUTIc5I2^qr9-I%>UXjM7i8tqh z&BMRC&j*Wwn=Riq9Jz6FlA6#ao2O1rMZVd%>zjrnbHSq{{CtBn)iSkl_P~*^2ft3v z)}I&+d~b_xq-N_QBY{Pl3RKQl))x2zp4j~PwvV<=?)xM>yIxvEbTjuoS3S$tKuxOU z7I{__h9f_EkSFlGLbo8)7PN`TWc3pz&MIk-AFfBf=Erm5!N(6>XyZ}pQ4ngLUG2&X z-)rEsDR-pB^;Iyk#XG}rz|QROM|KOh4pc^I<~Hw-9N=!(2!!84w6_m%kyqWf5Ah)R z2hBxAUWPqNgT#de<-Y_5dVc`z6zZq)&?>C7t}7V_(x+gFLPMlYv_k(&%Jg(ajv$-h$YxVRaXs0uYLI;@eiig~uV;sk zPfcqBSdAASkAQk7F(GI&j2^JN7g%xRjk|2!0=w>CY`v9?PrPWUJ9`U4?h0~0@v2Z) z&a)7US%AnT5VSEDgB^>rm6|F7{RlCIA%DQ4XQOV&5pa>;M3uWltpZbGd5! zrdF;r4E;@x8_;`p90VglP0JZ2neK%z1wIQ8q>^WGJG_P|#N$25lp2p)ji#E0Zw&Q3 z;J9YGtu{$l&+Ar$Y6BZ-HcfZ?d0EZsN%-XB>|I#0=W1{w<;MahyH9O{B_?Da{A(Wr z8Agk&!1*2sq5FZVKOVdIyL*AAJMe|D_3)j*;SqLT42&HbJ#^_XftK)icxr1QJQv(E z5?QFMy-d1WU*kk{vgumO)s{bz?$+0MWAA)L)fI7EoQT}52th06YlL+6HCoC=4tdu< zVx1-FaEqFs{wr3KZq0XJ@Ge6C=3CZC#r%D%*jA5be3fl}X6A)p8_mt^5(s}#0QoJN zYZKkKcpfCT{9Icdf2)ci@;ZywyW6%fw_c)v+7+`Jlla3V*odpsN0b zgOmV-RYspqXHpK5y_k3wlc2P3#=ewjLQfolPX*O#SK;G9$5x5LBy+7AfJq)(SMVjW z>nEuS%fie38D$`)zJj}9NwQH(0Rc}?)OVSNR@7_7}n7&-6`tjC$>pyo$&%?G$!l1$McJ0(x{eFmQY%Z8EJQy6Nk_0^msxX%)XF56 znH@$FvkcTCXeFS4y1+ zk}+h%mkzKyZ@>56zIi`0Z~o%(H~}4#pWD^9MF75GALbCM#N%m7Tn7qJxFncnvn9!; zEYlWl3*)DGit$NnN|+WXW`!1LgLY_-TXs=R_QWlU@V<3gR7AxF9g0=q;#S4}zA)`v z-~x_E%+N%DBjRKRt6Ex4GCB@%Y$mG2i48%VMd=umG6zoy_B@`Uuj>GFCS%rOQuSHk z?o|2tmq}V`lAHmM&ESyL00tMgguwy`@Drpd5*!PTga*c=XD1`k$mDCm3F50^gXab! z(b1uRg;+G5*b)%K^gHxRz}gjp3DMiiDH<92_&bYF$5Ku#;Tmuk(cVzbOHT)^tA{!-QKIC%cF(X?xOqfVsO*z zyQ(g$1%G$Zdw6l=E4%yJ&QimUmBi9}1+lHL}LHM#W}QGJC^aCw39?RDIw zOj|?X(h~qGc5Hb&c9?wZGAYA8>oMv2Ov_6!*I=ger8>6lI_5v};!CyvPZqCS1OQnS zdzw?kyu%P?8|q$}l;SY#ij!@s8$fYU+FhkRl=fEX2E#$QK0{O*b95UkO*gDc^G%l- z4Gz2@ayv|kyc0K?77ZtEGHEj=sMTkIUF%zNJR&}y&DbKD%5=P!Uc zF0gMx7U?c=O4Ttn8aXFvNP%>ZO9Nr4tHxmhVS3~YMxKEfoRL*@WE+b0?HGayH3hfh zmsR;W(n&3yh-Q#>dAou>qapkvUN)7I(ZY76(BxEf;{5p8V5ln~O}-HfNnNiGgd(Gn z(a8zv?1hRS(qw3eGUyl9rJ-PWFu=lq($Baz9GV=LbTvU6gkEC=YSdPkw!$J+U6v{? zN_Cf{@3_W}866=hlRl|TC6H&S;*%ShphpVIk^B4gMsF$Qtej39Q#5@+2RVjZJt zBSM&Eh;ZG+%E}|lj@NGKA3yu0^cL%*tPHYlioGTM;Zx}>P5&v0in1VO*{RyiU}~MB zIlf!!dZbV=$M+nI=PaE8Vy^})=Zc26LKi(H5%83A=Jw8DoY7D9_R1Nx$Lv`>8cOtD z)^N_AF=P_|+WTeArN>kt?b}ONM(>kkbv~6O# zY%Pm(2j6_AL0*y~)#gYWX1yK| zic%e%)rliGAA^+(3=uL&P2+&Kl2{_J)~zEJ;Htkgr!womX{Zw}N_e_3B2qgroxn51 zp{NMP*c|}vVJ_HH)xcR1)H8&K^D&)uYIX?J2V$*|_*3%2Wf-l*X(5WWOtiXTP#^NO) zKl4Z7UhL0_-zPp#7A{^Y4*$6Lqv&7Tqt}KDFuCL?iJq(OWp~Nju)?nlUvn+xS4Y+c z*Tq}oYd_pLa#z3i>SyoXO_c#R#`#PB{VVUT9ax)L*KZ{^ju$#lZlE&Hc}_p!d55zs zg67uihpru34HTO?mh79ZmX*YZ`WLPPx9uf&L%X3y4jvq!mH-6J@gNkVj(NwUHsJgweJ!Q2Lj-9aj~^;A4V_hvQ;`_T}^ z*u7C_MMZVo)fTY4aZm+K0U6mJ+G2|}+M@Axiz17*0LOHp!K{nA*`R<>prE3lyA4vb z=MFj4qw*#T^b(wV=iGD7J@?$#`R@F^&1NPbef7_V=ohsF@k{)oCZz^mHV={nai2g0 zB4fmGAt#3^c&B2TDebU!N;j+{38X<k$f;0V$c;S6 zi+rdYRVb9ohnitMMKHufncDDHj;heUkEp`?gma6omOd&LhE0r#(a(@9h4u}bP&GqM zcvPOP9Mv$EF)gZnM>}jqb&L&>DFZSaX!dRM<%)+(x5A^y-STlHS~GOTk*w>8eD!GK>6OtVrNYONfBXlWzUuI6WOFDIVHu8#&M>lZ zG!zTP!wizG;Y6Hc*qIQI>1Bg`7Y91~gWtY>wYRUohjl|cvL5QA3@7VmLa`(hF3tt0 zAqZU7z<=`+Jnj<$F#-AR#FBGRBaoB8J%PD*C3ObDT+k83AmOJ5iru~WEf_-`Gd0c6 z$=WF<6!%lC52V?0cyPFJn*Op1n;#7`+~IIyYAO*w%ta&dP>gGvo|7%XX@(7h&!aJh ztp@&PEZ6{#c|tPSeoy!vL9)9gt1B~fcVNvzixxU(*}ps_T3WK&mQCOzWWzL@7z24( z&oX?HjfY_=mQu+9*MWo`aEIjdB*B*$ep83jq(Jgom34+-DS;f^COb$xA_rjP?A<-R zokRUM)M1fzF#Q--wiSmdczvk9XApdYr?sq^Waf&#VfSHzBM8Huh|Ms;Vo%s6AP3;j zVRwKhEUu@P#y?7|nFFFZkTbW@L+4OF70$E%I=r~Mj$Iw z;DR1|fG>GRAU`UW(-e>Wq$Gi=ggmX`VW++e^)|gToWe|k+urfx9z+|Wl*IAt7@&n zWfOESX+?)D-=LK;Xvr?<_tfzdL65A84x4JlR*r1%5UQ7fuk0*Tfgbx7hg#xGTQ{=v zcv&k+(5uqOk%BQ&d!jofLE;NEi# zI|Vp$6?^f5+?7;&C-4H>m5^7}zt@-|pIV#t3f_sjUBd9H>P~z3lBZxsQ^*eU)+>~i zN_*Q~{e(~7EKt?q-hS`BNp2G(k=Kuy%Yya(?Qs7`GK%Cy=7s>g>0+W#Je3GW-ymiW(5 zL%JOP6@mwN8kH9DY^6|Bx?G_(D>Rtx3atWq;883-xQlnI7hAkG;N96AFGlT!2=Bk6@BRbno@6S7?Z@;=yQ~XW(q(OZ;zK>bc%S<1^}9<3XJH zIl6P88x+vz>3CxHpr3A~6)8|hHy=AfqtG1JT#~huC^Qxykg0b{_1o-W zLp)P3MGXTnr67i_|1C{vKmIkQq<*lM6;n92fe0=@bKwNb1mg*IDnLU7Fq>czSFjJq zoKehJQtl?P zvKG5eHmYXth_G;+fX}PVC3x(JFNm$ELfz|G7lfdt$N~h43C9>J5A1x@wkTyUEuNQX zfoMe4k^;?x0KW>kUppXMV$s{lD0C3UgT?Vq$}$yVBhfe=4#i?P0FQFfILG6-Th^jz zn3pvXhWC>)i8vf7(Mq`XrSWVmG&PDs=TprP&`twp34)$U25@%3&O@QV z_$CN*u*X3t_v4nw*Zx-f>}XHs_Wk5ya%p6x^I3V@C+*K$j;~((EA6wEa|^fcCGRFP zBg>sRd((!7Ebk>D0_^Hmi-kB(51k24&aPv-)|JkLO0x&g?rjSp97}M;l@JcaQHJ+_-BiF591#7?$6Mo=8C{=Sk%1zGE zhv`dz$>{zq-_y84)(v9775TvfoPOR>EUb5YNxqpfj?h>8U>l&5IDFy9>FF?kC_oq2 zG5zV?59xIWto$!EZS7;J{q*QGtS{&qT^Im5JRag<EAkv$THWF>K&o4B=GV~P90WF;6u^aY%~rlcq}xN zNU}7BlH3^@;7ycIu=JSnj{yv5<;U4XGBVyKYu{i(lUQ3%(Jb6a-~)83fV)!v01Ux? zr~uM4^lS(KvZ9Z>on)94L$lz%wp3Xco8UOQE6K888YgReVbUZpW8e`g0&I?7%31ikZo|vAVOLdLV0|`1D zV?y9e6k+HwHZi5z8Gzmv+!DauxY~C{bp{)X11|??Y?^EH8?#<%G&Gz6&FMeFa4-C1XpZL(> zgTRfiCBOi=tVk-(C1tf>KH!>Vm`T|@7G>44b~MB>vZ1(cWu0YS9z3J%3iQ06H18r~E3Oa%z)-E${n$3U_j{@fug-n^c>_AkGha~+z${M=f( zbV#%|&G&p}a^@+m$(T12W=F=eP_}6^nDz4)zHks`*UM_Gp8r#0)z5e5T|`;=e6M8j z%y&x;_k5q^^vu5|mF=Iu_~OXXpBVqln4vOf7VT?}2GP;5eEq58;9~?_Cp)tzue|%_ zl56SmgSr*}Gu!d(;2WT)D06*Dzc_q9xEKTpFP0d1;q-!Di~Zi|@1K?`YnCR&O8;8r zVX^Y?21VL>$xV%>%DkXm_|Dz3ypix$-tSxNdl+7>|8Q`{^Cve~>pyraS9?6?d1b-$ z{Cww=hQEw{QvVlInH%?SF5Y~|Egyex?uW-#<~}+v)}4ChJ)ONaw03P+yf&PD^H%QK z$b$K~-IZz2Oe|N3zLqs#o9JuH`Htl5?F(ATSGQ~wef~9HtLSUZ`3|oh7rn18m_IA? zq*gi^06ytJ-a|k4(EOSqW%w!3%bu--WSfQth4EE=SrvK ztjjtN-R)d;O5W=GBY!XgL^5yCn#`LTy<0zjIqx8h?o23WD1TvfWv#W6qc-bkksQ9I z#$}UO)$$a-)Gjx!*u?!Oo;prQ&V5E0>-&UwYv^zR)YVs}{PYihT=x zQf2kRYcE{x%;hyti|A>|c>-B?>&kalna6$NiQeoh7qb^{N$!fJ8qwXnq7$JmoOK<| z9=(y*kj~f14Gm%Qz)Y;!zi>r@$G{7_CzD*dC6>25wYNxgL&hd4lo7G~&{O*%slG8& z_QGD1wfiM^{c_JrtJrYzsr#hVa4>UaC#BJ!0d3x_yxn6?!Mv_&{U#E8)|1R*`@pYGujHzb z9G>Uis-;soZ{u^{zNOc4zGfIf$z8i)H(K?1!l=_LG0Ki85ud-xEz}G|EIXsbk8lhF zDNXX2LrG}c-4vjF3hv&@wvazy5#^Rjm5h@SxUqu2R?u`4BY?BhIv$53N)mK%(}hDM zNNGLXH{lp%SK7K~A&GbEwaP_(Og9DhRa0Dn;Xm4trzsdp(W|uA| z>v=e37>rG1BYvh5apt8*9h~#0K&l}Bvhy(YH>GH@Z(y+J`VG2o;Ko(Dcq2?7#6S|8 z6rc;yLV(86LO=YoSz% zLkh23+KcbS7Pzf8{N_Sff+wr2jYT1X#OI`}hhT(>BX%5nA&My)fLyXB$%k1uJ_)}~ zi6+<~TxG<)vMwA`4-6M*81i8U@PRsxa`72s^cNEoeoN@KS!Xh5x};&XyD)h={BheNJw$^B2( z8o)=US~v!c*8lq41QMJcy8P3V2QHk@?%0WN_^<3lKZe3fccS8c0qc{*t1(R>Qr&gn zc1r_gEv6%#a=WF6+bsjol$8thruPlV4DZU_@-`kT&}_Tp%7A9yMRNdcw|?jQCggfZ z!3~UiAk|U;S3DbyL;&c_jx%xP#-(s~0s$vI9t8}+j{{u58(|XPYhoZ(`r5J+P8D$5 zd0=8YF#CRZY{9B+z`2C2KE3;=v=X`|F{t|4eHurqv1puWOX9B_{Se;MYD5YUhXJUh z1h0VJKoB1XNqpmp8#z4AVgR9HD}@1%QmO&hw$X!l#cwH_jD{yUj1ET`z*n>a>XJB& z-=-@%woToKbV{daf*sAa1FFkL6~e&BnI=%7U=6=!YoW5Q1C_(`1(sZKp%oTakhwDr z-{_R<1dhem49_v^r|{UZ&`OaqtO5I}$KJhBP zR>Rq<)7jIvvLmC}a6%lJ&fdD6y`6^G#`ZlDX1%?2WoY%R*!ry}Ua__J36~9A&Rz!A z@oaQP49{loq{VPLPY~ZDFHn$$$u7v?(JHkbT|FnZc0OqoTQ7d1$p-qf{bSikEIXAF zBi{wqG)Qa^#CSoli|WPl5%MC=E>iszHsK1jVKd5tF@vDb<*wfsM5=D0q8S zwlc-dj|e5aP%mSq5(ogdK2}cs#f5QXpCCOh9re6^5e<6CGyQ|Lm8|u9Jjpk}6HE+wGCEV4S{rCNg{%mz%Wg_P~wdU#&T^%`B z=e%9gxRzQ)O|#^q=db3q8uHK*^~a_MCfL%*Lo1D&ILn(U^6XN0x%s{t%s~#b}#cERBaMa+z42KQ+fY_fTH4_{{@MnTLAz7 literal 0 HcmV?d00001 diff --git a/services/common/adapters/__pycache__/sentiment.cpython-312.pyc b/services/common/adapters/__pycache__/sentiment.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4c33592a295bed0c00e0575f9022a7591c19d80 GIT binary patch literal 4277 zcmbssTWl29_0DT|_Pt)O9Rp6YU;5o?$ko7vYacV=08 zHA}5FQr0b!jgr%8W#KA#&wqtcx6IZ*#39lP->#PY9zm_`zks1zEcEKCp57G?$+z;udD zbAw#kG3ZD;2b~leK@#(8YLExa0(JrB0J|lxS8~ao zb))Y0*g+r2kntzUIV2Cr`NycJ?;hD~f0QxVV~VDmY_B5fCVwKUD{4kaMV+Q6rJT(v zl0XIup0t2RluSevQYn+ga#A;)!?}#4WJU@;H7jQ#N=BEl44OC*3)&fz!@xQgbzpK| zCa2}_h-|W_b0AOWvL-izWjSNABZ88NIZP87xaCn^HB+Gfn`01Ga^NfveF!Y8TgiYfbR#Uk%2UiUgwagYb*2sa=|+kDFbd^VkJ&; zTxN^3PGqf?U~Os@8zzcSVQfE;@C~*2q*Zn&d@VK_Yz$^c8u7m)sxF7o03Ax}eAPqEiJJcN^{}uIG`w z_Ie3lZLPJ?-tSKWYgarK28|xWGZ8q4H0lCxxD3xIJxW`)=qrF8krC-`P;{Z0MS8&2 zSPBeNlK(wk585NW$X>xdALb0FWa)rYu9pmeWaz(3!cNXvvQNS8oEHoh8E{w0UWABF z!%^@cl%e4EygKZ^))LJ&wTctu7#rFeiOn0Ppqps@=9qn+#53QVliZT~deFZ2qEDg> zpW!gPhD&O=%#Bb3@R}5SMLv((kRGu$Y)81Q=-1oqz7#V2zd*mF?X@WMBzqR=lSpbT z`p^3ZR&bumqfzE0%2UyB;ou=z7tcgAIio9S0Fhy=rXvUOSXNhG7cz<%X&(lUjOdDt zqmjI#pNR~E1D_VeWJFgZX;nV( z5yOwtFezXmt(i`;X*CBy2SQ9PrE8dYkICtZo{~*;kNEVuFhCr1FMP|-0{9tq6j2b6 z4Ri`ZG`Ku9L7ziEr=}?iU2wB#4E=%0Q?pe6EShCZI=07T^D36`4v^J|rbuM`@eWPK z=M+)a;&9Nknu!aNkcF*k@%o*{vSW{3-6L{F9?jwdh1f8;@}c_l(AwMTPN@(}wF9v_ ze>>T36M#qmLXXDLr=jlhTT{=^yzt>G3!(0>mPt7JvA==(e~*cypa}0GpaXz?J{%(> zaRC48zAaUIh;%1_S$H%}7FBplPg)VNTIQ_8!wbtrO9LYYz`8SW-=o(B(j92LgGk)Y4JzC+PAK}dTtu{o2;sCknoWI@e9 zH>Zol2(u|clucLPs3=<*)8w)kGGH`_NvHufoWk1(1-sL-a8&RE&?x{P@eTYknd7X} zn^e;Dubh2IRC5_P2}`(4=gKJF58U`L5uxOoRg%yeS>L{6J)yZ}<0Y-JI{XfZk@&Ao zg2#Z7{lgPl478OGR|31|`Q1xwI~S|^UnntmH*c*rw^qZOtIb>QySR|Ms;nwNy zi``}B>Y*F2-fsMMiLVBpnk-HffBM4`dp8)K<}dOyo64_Nf;&sxVq^1E?t}d${%)`V zdhL5Y-_%*|oY^|_#-;7^n>tH=0C&Ti9w6SerPNyuw@e?tczEXR^65%AdM6xT2*)eo z&QjmvmbM3o_H|A~7N~rt#ySGE# zw*%ePVAD*)MfVi97zo+JpVfyia!U=c>{1x*>YD5Mee2ZsX5RU&%crj-FD0+2m(+?scE`VO!N2c@aO;hE|GtX<1YA+LwHj)wwnVC%w^v)9z3*`} zxR;P4=mFsJeD&ZNw7Kii!x-{5!2!6~BJZ0Iwv~Izd*}T-Z}aW;wSUI%{u++J#U5)1 z;9;hETKbyM&31QRkiFUUJkWpI*~0?*^K(6IfIbOw1U9)2QOqabYU%4{KG{bRx|;)N z`s7jZjF1_ThlsPS3l(0_PGZ?|7M^|@=|I+6Vx*imVMSVHdw^#_fca zfRwzkNt8rgr|Zd)((7eO!oQIhN?$0^>b&$C-g=3#Dnm-SIxn-fl$c~!N+?JQq%>Qj z=Sgc)I*G z%Cj$5!uuB*_Rj~8j~@YWH~4hb-%thTYu-`~KMii@bAZQrod7uJR}UD}&`((hxcN%Q zt9#+aVYhZ8_iOpU?|_(wAAr?D%BcSs6lsITZg@>vqDYM>StYJPlOG1XUhEslw(bE^ zywz8M;%!@z(aB0yli(L&eenim8e3$}Gf&o4n_E-5afLm{;U8($U?T9}C6e@6tJ1J6 zq#`7>)a$hsi+T~(FSNZ$rt2jL@F*!+dM=XF;Lpvf>PTv3t;B$jfdoEI0C{H^wQR*o zXfCqM5LpJl4Rac)Mu>&fa!z1F&cVYlO$<{@02oJ0tb==bF&v+BR>CipTz~U~t8IHq zN2Y{{V>8VQ-bl5vb>7>$#6hrxG~L<2S7vApON;KV}!Oou*} zh0DOiFfch6CXZ+N4DKaV9{`ilbX>b7%NfW4ts-g}2xQ&DW4=4BO1YGL0FzUbl2j`K zSYjxO`T{k5fdXG3-`~;B1+?=4OMB^Y{sE6d2dQzt*>HT^f1lk=#jdu@waq@ggn;~b z`^WmtUCV@AdYAH2&a2E40yM|Yz4eiMnUMEe-Bi51cL@QSYq`<(5l8}gYx^zzFS{NR J5-YHS{{VUx36KB) literal 0 HcmV?d00001 diff --git a/services/ui/__pycache__/app.cpython-312.pyc b/services/ui/__pycache__/app.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..105ab7f55476685d36ef81d2ba58bf909201253f GIT binary patch literal 50875 zcmdqK34B}Ec_)gM00@vE!JVQ+aFa+$1Q(H7sLfopQ?f|e7A%Ad+pNW3`r-m`w^+s=2s^Wo1kGSX%Ed?{V)IQ*(i_BV7Ree$A-4}RVx zliiS=k#RCk?vz>STj7+C$gOfYrIk*_h|;QLX}MEnRkM6))-?95v1;+Ha_UCXt?4X} z+NmEgSPdf?){K!%Yi6{ZaU{!{jdU8P;k2CYWu?``X{y zt$DL@ZiqAR`CJBHz-97<18JP`d9}3&X*?}L_5lT#^}Nbjj2eo+5gm$k1MalLyQ8-W zUtBh4x|Q<{h4@G=WBFPVGDMSHE|+&Je^tJHN%Cb&eU8?3AW5ErPmre&J&_MTvbHpz zE0rE@U;CCTlHR@~EYI<^bxCs?g>U!T99-!wvs4f8qB@2bWzVam($*SQ z|3gXx+Z$3ua32_J!K1<6Z*GB>K})GqW)T7bWK7( zEqGSP7xMW7iYRt(1$NhS+qmsqEob5CxO%REYvgutODC5^jN8d=dqK|a;YdhD3YX{efYZrG2*TdWqT#s_c za6QhQz_pt@iR&ruQSLO?^Rn{ma%&?OA4eSGDy|oO-H{RwXOhAJFyQk62XVYA?y&@D zG_je;1T<9KjRhbuEoxaa>2;wB!9>C5gTLeUR+xe zeD^b7l}prQ-N*0a804B|QTp?(EyLNsEBhs0DU9+p!;@=sNpu2^u>B$LEf0C$^pN+v z9`b(oL*DOM|2-$Cf6{m{o_(C1@MK*WaVuc-?-=}_T?-|SmtY^QtzLrLImj&QBi7b6 ztv@1s?*H!)XV6L_XZThLT5_2ykL5}x?FRVPmt?Of;(1rY5G89QI9uEJHjgN=KA&WM z2IIA`_SThHz>J{;%&6u~!!v8^lvBhWSpubNJ@kH~dkl-`s^;&Tpv19M%)Oi9bPOw|4X0 z{GkiGWHO-?sigc$u8pW+|AC456E&K-N^N%G8_Pe~DLa9DLO$!G(Ax|AeqOD6?AmC)THd^H0cRjHfJp6XXpIf&6kyU{@Zx|67FBdn^*AWRqi$9dWD-q`gN9`PMFYp(J?1?q56Px4!v_W_m&D)!LcHyd>&XyA$|dwclrlfe3jRGyS_?g#9;!fLS&h5dObQzF&O@czs$b>oFC`@ zcfjpGtOL`(K)ruS^#aZTq=V7)-B*mjg9LldI*@Ajp--+86k^5GyKuoHl+D;8Tpgzy$8&e5-mvR&rbd^scq{7qp@pU!kU9DHiQjGsUNu#zt&p$6XQ{?(7_qNp6S@|&NclAfY z8tdmlpGjo2^$Yyxx&M5j8k`^lIg+f-tY2_tBhTNlJO<=RvYxVj0g(Fw|Gd;^Xa!^o z)}l{Yt9yhwd=SsaPYBbLGbzXXKjZnhA?pnCeUT&G=}UY}H{)j{serlq4Av=E`3e3i z{>=~{g@pJrt33&BvwFTPwIF?4_u}~#{>%Jc39hKOF^U7wtgVCtzwuBk_&qBrC=SF2r#Anst`H zhWV;UGGFg;?@Q1a-oN&J497^8C!fg=_^$v4AM&%%k`i^{M^s5QL-b>F>ULuEmu!Xm z3%(5;uI-{s$mZLUu5?dH&>>mJ{R4E1ky{^0+LiT1{zd*2Tjw&p7CqUTx+h7bZqk); zQocFDpOTKFv}pcSZ70LG zk5}da%(kzE8ElgOn4quU;BUnBb^bB#X(oyMy#LDAct2_Y&8_};ehYm~sjs6<)9Mmf z!~2K*$)0lmmD=O|-2WT>{%_Imf1+>fAmHzo{2OUv*+Dt(Vk;=?OZ-d8G;j2-78pgP zm5CZR?o|YPe5+qkpr$m5XMv7iLcjN{?IH4NxPOf+c<3!%JU^SQ$lCn;OW-as>#f9p zlW~018QY;tg1*#@= ze`PocPX^U0=`D1er0?<0!20nyq%!zd*Qp}|btHWstK&8P6&~wesxPTt{wn{P_!cEH zF$=~Rwa)Q#7+w3?S%`5$F-H;{7DW4I5-LY4S#dg=mAkT1VSB6<;34|axwaov5^uWJ zzTxMR(!(78If4WFkWE?^zD?X8;hKYS=SsXW-n)mid-zkd`x>q}?G)z|9pSrx z8$#=a64dde2=~PjO(ymx88srmifT$mk}-)?I9d) zo}W*FQ?!~)iVyT0Wlpbclb`2qMEMR;z6EnwCqY~qdn&1m{3+=?I*0W%hg<{x8d38O z2|GSHMWW~0;Uw#I)Yv5T4>ETP%a@$DHz(v{YolLFHiwhUbzC+>Z@R*#ns>2Z}}6l%;<@7j6dQs`5F>Q0HzS&>LS{}N`=OrrizC%2O8ao@lU?84t} z{H2mruU-*SX(KQTuHTbnoaxo1oaxnMIzD;@Ed^u6F2(8(QS!fnomljkkf0Zrgn%N^ z!k(m3utmb*I97dn4i{|!@cUF!aPR*DQgB~<+-o@^ptfJ46O0j3IGGg!xeml@LB3|J z$0X0S-;b7q&3E+&k4W{c)w5Ukn%WyqvT8$Xt)J8iNqOTfMn{lz zC5D`B32U)$^50Cl7K`iu{wpP^_=*R*`jw1SD}r_P-S&8Iq3?d3O!LPw-lehXUv`V%a;*N*#cNlFSk?0Zkv`YBY z{1N4jCgFovA*@{uun{L+=ifRejr(6pPej{EEo|uL@i?qka~IHeY!41AgE685>t+5j z==L)3@YAkSPsCdS-S(sHZvJu#y&{cT;JT?rR}O0}H7-YGq4f_i$^!lml2Zt3W9ts7 z57uuZcT$XJ&lgwoZ>%=2^4~@+C-Ha60m}|NB4w;K$J)FS)d+bH%Yk}MlMV~3qJ2<- z{nr=qq=4=EinU)zUHc&Er#-aW%UW3{=firzYU*YC##HB-msP;($0V$b!7POZ3nfzO z(x~TgiON2?=FIv#l6vHrWB1y(=*JUi@2u2cZo=5_w+AzdnUn|B(Mf zD#NJ#`+(IS@!#ibr8yM-od3S?XLy>_69vz!`R{)cKQ}Qadh*QLd61|l-e2yixF-tA zB=ImskNe2-dFwM$Pm%ArwRIz(O{&*=2fawi15iGhCF~Ar=qK3ll(b=^=i2QsF8O3*|pG>W^P^mHo=Q>TwWn-ka^^n;Qe+M z{#$a{Y~I!1yR1AbyOiwBla=yKsva1x z=B6lNbyE@^`BC{F{OB;y;_svP1@D3rF?Tf<(vK#Z&X(w zN%2N;^*brv$ge(@^39Xto9baK)*cY%cGt)wTF&O>y^ax{(pI;NKjIp2Bdr_5-nq-} z=J*u|el&WP%`!{d#pd&fqdd?ejAI4JNQkK-aA(e~S%HkX~}A_lwL<>7^k zHZPvnD1F7Pv0Ucr0h`Bb_6m-{L0<6G_)KllQs!!oAFz!%y*0i~?Fr@0)kBWKAv3BR zaB%1kU*pT_924B5d|m5^#|v*G+enSd*D&Puj(S?^>g>Y!sMlR9I6N0>ZKICbQNhiP z*{SoEu~Ca)b2u;A#x1tdQD0?>A{Mtx;9VRqSnTc*pOJE*UewjHyS~2Omlu8DblYvt zA-BiN9!FIDHV+@s5Fp)S-bk9jd&dM9cADcKat&7E?}P8)azmz*d83PFY@;ORiPp?CY{zD3iiq4SU?Kh_3UpooDD%qZAByE*6vv9_o7sclEjN%jD|3d&b;lCB8rQm{9L$TeYov*;{&D zYk~5ud^xUmJdJ&zo{DpK7jt&wAu|VI5u34LD!J+-&B&x|0>JRb=4>M7U@0gjFdOUR z2ewLMtxU=%P3Dq#+)T&Qnj;q^LvI#ZnM!rc+0#r3pEojL&p|L~I_Q+JgYf4e?FUe}sd@ZMW zkK1{XH@BWTVkTlXpE-TpJRrD7%)INOLvXuBc$e4g7R(o1?n^GS-R8n!Iq*)8C89-{ zzP8q$&WJ3MN$K|1?v5iJt-YN+AIrCQ*XTedo=6%9164)T9`L_WhSo@rIvw7K(!+Zr zrjE|8)-%U@6G}(YKn`F=5v7geYP1n8%J(r&lS!#~A0l#lBt79)1p}=sk~YE%gW$;W z%Z%_mMB5-Z_7Mtj6UhSmrco22v5JLSJTAwd=Pz-YlKtG6eQkVZ++RF*I%KR~GFpO0 zOUPJ1b@-mf_~H@&&|>bEu(4p;_-{pHh}r#ix!g^L?% zgPFDO%arOukhI<~+wsNyVT0+~iJ24rvPDBl*pPMY@XX=aN56b*S%XZ=T3L3%`!WTx zt3chey$ll23<~Zs7uRAIY@Ea9tR19ppis5lA=sU~*#`ChcDjWYbHzaOz^;J-@%g0KsK?tnI+~(YuV8a|Mr{yf-o!FP4cpB_ zjriMvzovw;E1NNwIYvg^g4edPw1o|c=W6YMQbOSs!yW9mRW~$mH#ee*CNx!VX>Lv! z59Q?`z-tE_PRQ>TbH8&;sBUN+y_}fI4NqIVYZ$$327)>`v%sot+P&S}uoD$G>~5l( z8`qSnB`}Y9TFj00qnG1Q90dV6T!X05(kSpF_-e*i{C&Usa;;~`#rlOAkJzDxRUIM-#75Jw zQyR_cdOhPK{cdN%WDK}n-dc~t$73Xx-SG|)UP`lU=eQ;bndrHfzwE7LC^O&|Fpx2@ z9l;JR7>AIP_hPzgfm3z{$a+itZV7^``!p&*oG0&+2>*DsthbkVD0$u%^RD{(_!|+n zG>AqT1g53lvLim6)ot2`Y_7o+jjn)$RM0U9g|?Oxprr~4eSHWn?_7b)$ni^to>GMW#V#g_?awu=7u?AZe`vs&Cz&&7;u z5MwoS*^$UxR*ZIMA~#LM$cm;NJ9gV`@tKprKCpYhHeioGUnLbGQJGj3rQ*(A+s(~T zSsIDc5o=CBYBtr9JaxgU@QH!gGUNt7UI&z7hD3lS=voy`15G>mD20kZTp`z?Wq>J5 za(x&0^sai0pb?3BOOrGL3FExIwtkO!)ogoh{pIHqn_tP#Q_p7tT{1BS_MRZlsI$pu zZ&hz>g2Ptp;hntQ%X9n7y}}s3vOfvnVA{)Sgh|wyq=~oh;u;6|HE0-X`UKYWjznge zh)zj#q$E=ElSsTo%==^tV8fa@i+0s%>*o_Wa%`lGg{K(lzPbdRz86-o>@k00`brN8w$UNab~74+4Qsm@3qkW?x7Xam6bg?|e;gWxnJhujqjIt0sN&33 z{pPB(=C{9Vt~%UOb)u!J$7eEEA3D*yt-%~OWBInUyT_cI*@eo;yyQhKL*jzeHU|9? z1~u56oZ}YXqn$3aZN_>;Sh-%U<}HI3X!f7h8dl>=htcLpN3uYus$E>msjAT`l z3NM+$*k?FXO2xmqvq%efgb&0K3Eh@3jvR=vlrNbB$IY%a)Ksmtv|R3-(VB z-v<@H&*Qau$2CJ*y(*XmMk6W`a}hZ&8R|rnfu6O~#~Kmntoqyd z^IXCOwmFqXUHPuD;M&B@#LJic!}Do@lDgZbMdQ9{Rak3S(ryfDH!f;R!^Rx4Dtze~ zJVx^J%w@kOP~H?MYYrKA2DCdxBb@?uY16N1XS9Kwn)$N%oIrN%qSg}5Di0Vdeypwd zM_S-le(c$Vwm;jN)tV!FD@WHRSH4wJ+`3Qs)=oL4_o;CuO9(Nrh>E%$Q9B(Y4zHc; zkJ5*XaCHBH0;bkP1K)&vLcz)7dv7pvMUAoSlqC#xz(nPQ5;F`_7w|V`FdL4ZfsmS@ z?+F#Bx}}a`tQv-|BqlovBZ0B%_0Fb|2`w4Mb7_OIGk}wt35`cSE?XZ*Ct^mjNiC`|*eUNZf%&o`o` z@=5K4mQf(#Gy5l>V_)efeL-(5sft=HX zgShw%b;QqMr*^T_qzMY;z;#HbTAqAIj~Q0Ig;-6^?^@A zZSqF6$4|Dm9`Eb5GO8AKQKh?asYw^;09t%V5yjK6)nk)K$fq_nqqcFU+r|kUbf@43 zguO_3h|!T|B7UMaLy-Z*M?iq7n;Cna2cchZQ46?IGmkxqvU#tp9^JIEM;4Q*_K?y@iUv1U5XUX2$+FFILA7ts;4=8_7w4r^E z@&|j=NI$rYJVG-r0I%SNy^CM5Aw*cfc#b(L`y6DH2NG~{+M-+iiEj^pBPke7$ptmz zPfEJ;#`syXEh2A>8-ZV`u&t!v)baCFli*+v;Y7;yI1!`Cn2{eeSvjfl#!ULjxCyw7 zYNb?kz!=XcQuVBg$wYi7P^7k7AR=e*6nc^JY3n?q$|bbp_Dw}34famkh({oC>?>SF zh0(mAKOJ_a+zwX)?GZN-Vg$IFbky-Sg~ur;QH)3*G=+>dMAFC*Nki(^S-kxj{yd+@ zg`}fiZ2}HXr`^+Mg*TPGHgt1nZtNxZbl2>znUnYQrm%Sn(tCo2jZwmQ`LMrb?ukH= zW#Qr-VbO4O6@_58>*CDCFI}ECO1Nu^;qJEi*7+TQ?7Bs5J+S-BI)=9)JAuZ`BmFX_ap)&wT~l8cwve1yIM5z=9ov!B zK$3Z|vEkc$!sZ&YIi+f&?eS3W5783F;YHH}{3Wx8N41=X76GH~(J?1vkfpb)8#0BI z04P=11{cK{(NIK#En?WNJxueAq(jvdyq-%A?@)resYHnym9U3;eu^&VsHBp=Z1<>; zBAOCODlgirz7f_}h&%#;(2q>Z`18Dgi>P>(lHxfzGxWz}uOP0zSf@KX0mOG|US;tb@$za*Zz^VSlGW%@hjhgE}S@KhmahX_U^|lx=CUx6^cO zo0V@H4Q*x0w>QWsU8cr0qUnbcipj>d9@cpRKQ^fs8nuo@WRYa1ilQMdB~wlKUA9hO}V7Ytb!YSJT0EEFQj&fdcj zWzWguxb1F>D4#feDxz%dXpN{kP9Exvs9GOueIlalIN1vik0ag31Sh~MjL^kJ7dI|G zL$p)1)U+^4&!47?Ko<`#5nZf15oPb`Cn72@_DuyZy&t3Z>JzQKN7^9`-Olkr_y#D? zbRUUm##|21CEFN0djp zkJ=MuY>dw#P%rd#;wn<*B=;Z{=deQ+rOopT{I~~n z&XgcESxhmtg6)#HAH9;&bFl{{*z8`m58WqjYk35s?=rOQnL%Po7B*x~b%NsFm!+xm zrj^SIwJtBb&9YGOX3Ol38+)$r@%PM?g>owA9=)5h<@S#04k(JG75eRS4WXQ>Ia@HN z8V?QG%zY&Hwc?w_FO~e*uz6XB`aaCWUh#{n8yVL#{Cp^@GNi8xsH%SXp<0$z1fLOI z-aP~PjOe01BQveW*4>J?cI(bI40Tpy4gwgRsRu1frUPz|`PSXaiJJxKZ_xMpAp zK&~0_Rpv!_2VOPA?a_W~=lWMG0 zW3DH|N#yQ?YC_GW-O@-toouz4L`q8dh}CLZGK`bnPii1(wefMRCzU~Yonm~AN!?`n zMEZn=K|w#EnMh|8qo2?x(^umTO)1`MC9F^JUMIch(qC3iXeV?uqgM~XwFK+VxWmXQ z>0dEvh|B(ATEwmTaQX?uN^O^Ex`t;BZ02N>NF{0o-M(7Mh6m3Kc%qTmvtMwNPa656 z)m_{H<#z;9p^ojfGSwK%IS+gn$Rb>$iYOhNCz5f|(`O$O;00jE$~BVXaM_(>9N*X2 zG$iziJEoBW>7f(8d3|G}w8Z54Y+m6>3_^Gemq;cmrZQ3yi~uxPdo3#KmC zm*&6`W{m{pqmg39Rm(H z$)t$|B5M0!pM!%f%;gov1sh6vNWmnHB=(7_CZe`4j-^i;i|~8MT}vFR5*Mt7Gt1|i zLzy*Ghr(6cf0EulCtr|%=ltFJwyXR7*&+RgKzjRBJ1+P1nQR#vPTw=te&1+fRx#z5 zCc{OW!lhN=ikkOy>ddsMu4RK!-7b$B+5GMEX}1-N#=Wc598*5&F)wU`!je}oo&N5| ztxFs0gB$BZ8ynx3X>_KU6If^vAN6bQ8cV~4rE}_=nX~Hq8_Qoid-LqvSZHHycvHn| zj+>4^_3qH7J>k;I*T!#-2WnbErF-Ags5YBs)BQW|W|c2%P}_2rEUP%6E&A2_buxX2 z{Ffir%M5wsWQTsjfR{d#9?08!yZv@^Ag6Un+ZNEah4mRzN3bNJEFXK|4JMn-Xg8}~ z)#LJ(wzaUmO7?cy0bGAjrR&I6{=ib$VNm{wL5=hZ_DT|ju~*VTZjgZ&Fkk&yA)b}Q zK2n)^V@ry4#O|aV23hR?Yzt;=n{S6tRz}lOMoTcGC6uvmsw&Z>CsF@beLT1EoThBm zN7R{8>q~JtH39i0O_E@Qew4B{icuuxge@Yb9Sz5{qj)DKVlI-xAZ)pc;g}^Oo(I}r z5o{(5eI`^FFuL{R7pIP%Xo3Uegj&#$B#p*F)Szct$WrmC?gDHNh>nk#Pp?9VY#MYi{$0Hd*g9TnhIXLi@-K8S zC>FRNv2lml0Za=4rJ_lF3b+;{=LVW3t_1Mg2#c5=ma$C}=?PCuUr~JmXqu&dP8uc* zSKgd3B!%WH69%q~=`I-)8OdznlAe^z63&%#6}KuSd|nS8sVQPI^C47_DUAbG#Y_&248mN|#li*nA}8Wadv)G9~7d)#-}`UXl~+GgII#kK_q=w()Deqj9)m{is(kgxmy;x4-bC(1UX*eg zFk85f-Zf&jc1-4{m@WMB#zZb=Y3D>9@tg_#!i3Z|IF1}*aaT7yo*lw1^fjVm&N_We zHFaQH`vHfi54v#iRUL-gANH}m&>78G*u%YgfLX#wy)^HrH*>aekHxpK-RZ_&Gc-t8 zwP*_*Rw$2y>{-zFEDtKEs?y`+L5R8%UWxUNd0^tQ_%cqyk=lHO_RV2Gdhke_QTDNF zyQN{kr#@lEHtNSpSDnqLg+s02zHBl3l$QE|F>*Qg4w=ap66zrA_q2r{(G0+^7foZE z6sq!rLS$mAVxPh3b`5&4|2mGhz6`j1!tfQnjc8o_7-Y9IqUjQgKgimGj;pp&A5`Z4 zQI9WuFPu|`%(jbzW}lj#Sjgt%;+#<3)pi2H1H$76w5Wu4a@r;A|F|zj^dk;eANuHU za~_}Z%%}&>%jQuGllJC)*&T3KU->Ab3nn}OoIKiLT+yD0rhFP|v~Ec7ZDB6(91M@~ z`FOUW(L4l?0}ss->kaycZ72YSDvD^a)`ty-!+JlWcRHRPBlmbrl20$XoWW(@>BKk$ zxLLzQFg7xZjA}r{ISxPgF(^H-B;qN~(u_HsD>M;O$9CZFgEm}#Q+`Y)XUg7r7~&`8 zGf8bS6AAMFx8yhEu#yzQ>0kc)%1d%=?Ga6P;}8qUoi~*cg{2+~Ctetd=*YT>bXSfGuJddG!-Lut*B&4YFY;Sjrn1C0w=g(GyQ z8sTj&(Nd-I;`|q!`e9>|mZ^dpeFG_{kxM!Zujvkt!v%{q94v%uR9uP4t|=70#VU@@ zv+!-YlQ6~Yh@?l62COckMD;*#_?8NP#43RC5%eHp6)_B$H7^eKQA+qTWDwq_TxlXf zfe@TH+Xk|tuRc&?M8`PG0O=Q-|*BGOrcQ6wNk$sdTC> ztkF$9>+gSU=(mT$x%oE^T|eY+yMAn17tY&s+!&;(@UqULHuvE&Nj|oxPEZX8qBF%GBpQH z%^}mS+ZA_B`vafhrrYlwXuV^9=k(i-U`|WmjCZ=@p3yYx@%MjaJY2AQRuwknT}a-ux-90kl(P77tG!DPII`u>(0U1A-`vChu;y{vio-K?X+O-!N3#e zmNlwu(}&8mj7%(ajk(txGY-Ellv$1yy(TlDDW2Q)#{RGF4;OBl+ww-;>vh4xhM9qd zOW%F&ThHAQg1e4RE5rF4{r2n5rTi_y{4Mi}VE(pgeK@DszvKG8rJTwjtd*xjIn_%! zwZWX)P)^;nHoUoEq4lfo>)r2M2%J0rBlq_X2JB7%P;0!FIg@!mt0^8r3Ub~R@X6Ax7uw~!v&OlZh`ks-AzU$K0?R$2izaW@hHk}sE%3sPV z3ucvtvMQFcwgt1cEi~R~2xM&wWpyrP9Svq34P_l)%6c@I^=RO9Z!qi3wEEquEpJ#~ zw=Sp`t9H(I`a8dJ;&$t_PMSu3QC||SZCrTt_s#|jYXUt_1fD#1x941N-^oD#)6+*{ zv$82%V4lkj5 zZc8w?<#yxU+yep681*`D*6};Ji~7xxqQlciBSm|skG@+}5-K@%Cw;NxSg7dO+WYX1 z{ddyu?l=mp42=4)DfO)Uu;P80>}h-jPdV|RTVYd>(OE*kaQ+7WQUAWV@rB|ALm+p5 zIJa=cC=EGN-NFLs%z+>NklB%lePZ_HM?iuPKSWBVk5wLW>zFp!dNF1bjVw}xjP2!0^`yv&-%OOW;}kho*l*d+Mr^$= z?QD$8Vmll9Zr_%el?HZJoNVOiTh^U!v z8_5+_TszcO4~&W8UaRnZWQ*vSn~c{6Wk)IeBR$Dnm2D0WYtoV%IS1g9<7Gxpl-`4k zV%u*bwbCgsOEg6?&W|J6JSRX!f~JIgH;|b&NHgGWBYNfqN;=;W7LBJ@*$mhExT6kj>IxT>+!((; zKDQxMPzzIGR{F9`rAzaB3KD61P#v z=}(VQtolgHO%S1ay=|1F+aB5khd($4deD1uQwZtf7|GFx^-&!4|J;6JJ4iE;hOM60 zz$W0>mvm$?f`+k3hmEiTY+D&6(4{C73)38R+lc+p$U{pSBnI@UOJ@AqC1Yln)opUd zmsK#3>5`cRC2ZLTP+WZ-+f$lV+f%e@!am;EcGQ4EVu4rxP-21D$igB@{NSyceu9e} zR(j%0GOL^UKt3SBA#SCRAC@oXWG-z`*=?g1hr8}#V;$Ha4A}AgBav&pfl}~Mpre6j zJeWRfGt+xS*9&(U<}VgWe~eD2v9!o|(?##E-=WN3rwiLnif<&5 zG>9lE=!2S#TcB|!b)3u=nKaAQ2iAVcBN)MkY@HgEgR-n*6q zSNG07{iOqe^nL_5(VBwWUwdxmIse5uA!OXj94l|Ohja3#kAO$tRQ^0?!;O8{_XW&t?;Hx8dF-8i z0h@g>ha*)cYspX+G~hIL1#Pfn-~ENXHx5EK-#OnH%CR6vX&Ln1JZzgAbVbvh;k;rx z9`3&!%B!C4x|dTF&MI71Xfuj_z7e})Z|r$}&%8BMvHNbt{@bmgjR)S{R5922#*x>L z%)3LC2kut3-zf`i>WDoJSXx7sZFegV-LZu>9e%fb%e?xVx;J$TmQZ!;-RiD8kA}(* zVYh6HX;wMgb>sN;<1cjI&8k?|qbtkBvaFox^k2QdSEhdiJ5@Vnd7EDL1#-4xyDFpT z9(tw3_dkA!Q{!w0%Y;3-FX~8K!FTR3W_C8CONV_bTyC{)?y8Wzozq%_uOIAh({^Pl z{#32R&7Wo(yUgl8-Kd~+shrYg_Pj#h)u8;-dO5Buv^o+8OtAd{^u7P}w7Qi#T-?7z z(#4q8mt40~uVjm0`N{-+P`S1)Asf3o1dCOlxJMuJ*i*$l(GeOVS^flFQS#oC`ts1S zy11VCsmrFXQ#LNRl0!@X&nRnHr)+&B+Kje+MICuBR>HIx)>GZMf9=A z{>dXDvVC>NzHylEC@#=A*+j{1OEDax_MQUx7eTr26T<{0ct&JKUx*EQBIc-XUSb{Y zfrS1*W!Hgc3v_NNk{$1-)RI)8*pf#mpnMtqUVEQt8lzA~x@20v@VCHxW_w}+3Vs}L zL?Azh8g+uZBwK^vhva3#Z{d}PWHYnlh}N6vasboNCjwsBh^#-tpQjNQS`OqUEC)^o z&OW(lJhw_hB=`_W{`VncIYMk0i~NhEUP%n8$J?^eKX z{yW@a#op=# z1uA#mmWN7O?v@<9Z42c;@^0DYa8}L@&2>$nuycxv}t-{_v^a@+q(m& z9t~BTzFYBl;IYR;8=rVDTSqGu9j#PyQ>|1AJJV%vZD_aP>xb#O&V1z$^W?Zja;5W_ zeNjLEKG?#sWcw0LC0zyw*ykS(`V6P}0hs285N*V3!-~d&oJ^t(rsa@y(l;*I#BU%7&M zjvcXmzF)DpHlNMjAb(M_LGH`!VTS^+0m4pP_$=*1 zZrVN?a>J1s&ct+>5vM9e%RLS-B7O(%u(5%Ym;Jm2`zmVjSdD6kcaBCf;!_LP4aeW1 zpAk*e!9QZah5-5Nl6BU@47oxpI?D{dqLEk$R>jP`px(Dvyb`v864|SW)(bB!a=-B9 zva_QnMDM7csCN{gr$s~92*SFa^k1g;`Z8nZO4*@OY^J33uvj1;>rauAZLS6*t@#Cc zanP>PU%Y6pifGP=Zdn33DEZQ!X|Xg8O!~CXv}~h96?{-E)T=t_5=lGFopl~^(QcDx zte?UZsq7=R(FgCMR^%>4#z>lYSl;(2&`e~6F~=_K-~oB#*f!34EoO-V+~9$1E>n=8*D40$}MbS~jAq@;TE=N>!@Oq4FJ6*Kdu-SZwOenP1C;T@& z6m}7?G!=Y3j|e;{hn*t0sUKfo0@mj?|E=cylWU}+_Qt# zv$Gi|-fPk#D%)iTWM{ukfM5BbI8MD7LO|?C^hSK6hKL;XAa0v#1QGs3!&I6Gyog$i zccf+z_2nl&-HpDAF=Ut&f3OMN0s_jTr^)2rh`b;78KXAH;J#>oM2IzGP&kjt0_Q|B zIR_$O*~U+x7i~77D~3|why#v6U+szsVaS9t#&G>Za3X9tImgXot{8S%Rv3$joe@*{ zfL6lf8pg3-EE2s_ut%-z7IBCK_?vV#S#jZ_2WgqegFe>obJ|AwIoko@aXckA1`i$X zos!)a?v#CR?9LP4du~d`thDGfx%!bi7tY{ela`ugih=LBbS9NoUG^bH1b7jmN9`{c z0U<>YLZBLoUQsSam0DGd-6oc0@d8++9|RF%VLaIEwnV1})%d#i)uFLZTU|5^OJY0% z3eAK1uw~vj;JZXS=D$|WQY4;eN275*))_Ain%LK4ZYM%%eUBjfhq#QD;c~+K*Hd3L zw=<8^7R2KMi&j_2*RQ8P0vn8lOM zo6BzC3Xr`~l1KD6bdwEY9sI}P9difUOs6ydZ#dzo9Abr8f{4BQ)IzSS6shz=e+rj?TK+yA2`U>6}tw9`AA zb~WwpUkQH}EvWo}L}M#gt|@?&;<2x(b;bUXSzLQPgd%osCF*HIRJ% ziOQmkRn)<2DN5BkIG4!C5=w}1tsn{xq?k+MYV=7}p6&Rd{LXdCuk#8^2Cl zh^<>L2DgVv|MhE;px3?*h^PkoB$)wm47KEA^GT*KHP@#4ecaZ^ZSW=pio)A1w)P+@ z=tlsX3vgvZxK~;-M!kQqfEg1XMAt2tDIvP3XGq7+PXC&@B3J8_Ni*km_t>|)~#k=P7j;$6^r#1N-Yr~Q))LO@k!1*h>GqPk$a5F z67ko_h2KXKU)%AOpz<@4t1@;w32VfIX7BEC0+UQ!&$P5{T*QbEa5M_LCET#tFglytk04pzt%Pa;*s*XoT zvY5}16DZ~Z5xBq`d2FE2B*9&4lp+*yj)4KNNF1GpP@=;{V4gT`XsJo_8N~H4$sf2Q z8lwJ}Sm&KQ64CX<92hmHSs&m=5m6CKj;JUyc|?hB z2tNR*B5KwO?1BVV;Q|08IO#G%7nVciLOc(^of+?LQyKlJ)9rPRBdmvTVT`Q9OsI=4 zpR|O84T*fMU?ISghkVF{qqy}M2y)Q?QZ+t()DS?SrBHx!L~__V)Pu;WSbtrFQyXLn z0)?PTan5|jU^P+S(p8Iei}1%Z0CL_HXbYa~Auvs0g`|WH#kU3M3$uxP3}M+RRGSm$ z%L%Kaa;fg*5plkSw@}uXApyG@zHBwo<@?_pgf)bq6p@b$ZtAYM%ZDRrY<;eAQ;<;r zJ?~_q)2_LidFs|s{2s_8==3{w_j3cgvHqf!azrgIbQR-`(j;8OW7dFhm@?pWs*8`P z$7!jIX<(qtMt`eMK!}hDhn6)p0+qr-o^)mjoay37$*-5J2OK{}Mk$&sX?$=`T+vyK z@M&tDrcaop8q-+2jjUO8nOYZ!`g~iZIzSpLguOU1i7Q;3SR>V-&jrF098?%4jy=0} zH0;E97!WjGw~rUx<538+!H6K3piZa|g~T~V4wTgS819>UDds&!8}BoNP6-Jli9lB4 z$pe`Dr~|wEo9zQTQJ76>nhVFoJ(q<`Xl{(`0@To9EMc%%7YWSVfkVQA1CbGl9Mhu@ ztE*}vx-J$*-esrmuk1=hf0?E&8jM-bU@6}Cnh(QJ=P@&T1(<`wiS>@>0(dW!C(i{i zW3UtA8=?=US@fE=tUzRDSs=Z~Ju*6mxNCyQ37E_UWErQqY;+DxWJSJWhIH`< zG**&7;E}%b36D89gc1B=&2YB{-$cw}kgb`AwT#6@hBoOo!xGKl6b)G!QwC{yCIXO- zZbG~C!UAaV8IQoNoYs{v-i$%k;in#q5+>8C4rU!+7RCtcC}NzClL_0y`;2j%J8c8o z^I5uSScsqu=V8Y9*t`vhKJp^Uk<5q^pv706g8wiuuE?1cHG>kdkU6jh?gNZB$JqkW z1kGdf9=^UmqD1TjiJ2hynP74AZpA9VMk)lM?f^o;LBx?UH5MKOBcfSD5I_PPf=AlN;F|A(AdLD8 z5}2sfbkDA|ZDsd^(#BxBp?3OK*Vd})XUsz7i;I_7{$VOlFG zIjD{ude}>6B|IQ+lEdOr44{lOne4DFMnvV!ub*2XM>N>Luyg^Yvrdv{H^Mab^8gdE9D>q-*ar`Ovx4{Wp1lmn4ho?K z8X#!m_ym5-gl!C9g(Brk)tBRL_XSJ<+}=SUILLsdI##k`M+Twuv7HxE7cdKAC#83ljtTB7P3(+r^E2Tf7s;B z)feNu2+PmQ<27pmhl=UA0JaZ2UT!NU#W@<4D-N7Ya${3@3++!_A*1` zLKq-DF`H!dwvteyx!gKpyS$Z%2igkHLS=4cZMg?PG-d>FL|r(Y!`2?~DynrlE?``U zzr#*Ed6LzAL^?G^Cszn0gisU+MdaGb1O`W9nMUrE$VEDW1RcXWhT~3a zxRLN9QZcIIG9!A%NMwaHN)l-jPFsLlw7-ol8cj4~1Y15h2qDbUWBjU-&4XE@aS2VR zS-6JB#Q&RiHTFvyj3~!ReevyzVp&`T!W0YRSQ28m)O0j-f?4VoYs_OA>-p#4Ha0?F z0;2s&0%lZe5o;#S74J;6RS#xv6hAivG0Jvx#ox3zWpg>~#8#LsC<$z99-CDOPpJ4Br^p~DGN;nixD0C*^7YJ;%T3ib+Kb!R0})k zh4O^xgy&KipS5#e8(Z*=I4(1zl(-9s<9y)uQU?eg#7N*W z{WvV=_5AhJi%=_}6!Rc?XgM^gl6Z}3HB5FyRaM z9Xa0Fcf7T&^LS4rNBXTC>G`qFv%(3gw}LL8p$oZp!W0Ps%YJJ|sHKO+^w0=2K>%BF z3u54mXvt(ib{;0S=+Gtm!K#Q7TP6`LhBrWN{49_r7SY)JMHn)y6KETg0;rKS&km6f z%cloceflLHOaR5|v$f0eRkNRAw$^9Bxpl93XT z;>A@`rew4fD~TAf8cFnufypD<2%1a?N2!<_DGF&2r|4gtT zCuf`~W&xAQAcM2VY!fqrxPjz911|#~e!q$ivk3o%@*1MHY@9#5j6P$1N!1c#?i16X zC#*WjL;8|{xIOd>ftXwJ48UA4T@z+BSk&YW1P8;Dzo)9%mbwbF%QJJOP(jer5q2|A zDUfBIR>YnNhO2Jpui&pIb1ZL0!H+(e1Sgu=tgwE`6(M*J6mXa$Q%|?f}chV z(OH0vf%hc1@1(gM1Oj0j_2}nRR)-%Fq8Kc89+C8jta>n+=p*yw5o7HBHRXdd0Q~|W z&Tk&Za>z|A8vZ#ffH0ymG5cbl7*AL{!m1?&pTfG(*=g5>z9{i#z`MvfjujOf1J16} zs*Ra{V0{(W$n2D8nphVMY2v&w3wuNfU6W#8;|Hh^Wssdob@bx^Dgp_M=qm3eM7M$I z;^(bMdt?e1b3ufYJ$b?dh=g^7!*m0uC>w$!aRWFY9noO9Lk33S4CaQpLWpOjwfo>z z0`YS|NP`}PcQVi0u$RG$S}uQAru(UEBmV!5Ea$Ie#=nx~|5R4~A7#}+S@ln4W%#wO zin?%dWq5NFjt-d=Q<=-@vVyXJEa#p+Cy={2q~9{7eP5-{R|I6H_cPVjN;VXv&89NtMyTTd8)2FYUpE)1c;0k59r@HQGGOpM7yB1CGurI=qP_1cNL7|jp z_bqD8Khqc!%7122WecnHsP52}L(d<%FVCDZePQF3jk7xzPH;a~KNY0}=d@c0iAb{KyEr--{iB4YXADio)-!XSKSc)J-xQBDT zAup`Q|D3SCkX>`bhWxOhh#ndW-pkWxE2c7*i}Q?%uu3<*XIY8VeWY5J)kt9`)#=l_ zmo=2uQu@HMj?(Gkb3LUEl-{$PLFr7HT0ebs*+}UuN*`O!rnHIj>|D;FbS|Y^mh&i` zFXkzrbfHXbm_EB)MCoF&&JC0;N;UHA>c{Q zY+h9DOk_Y3bTj0e?&THvElb6*p4u}_B=FD+K7Q_KD z;G#KbFyG%)8LrquF8A|?LKRJ6bLAV_*R}J7A#-E6besS3YZEso<`0KTo0m!t1WOMD z4xS8^p2FF!Y|}548U$QWqDs(BLB(wU4d-99aK17TGE^cotuAx=>6ya!WM%43`Mr#d?6(Q-=`-%(*mR?R@aEo79%8(Qb2rYm z-8gdnNI0uxR&m2{-4HI?5-v7}H(T!11kOAj-2Fsw!`X0E^Fn`M&ynEHqk-mQfr{gS z$Day(#u0dII9Pn){MVu%vx;ddHmOc1^IPV_ByzFMcn>up$3Ly(t@PI5#hIx^sD}EW;#n zmGGvW6vtvbkXI8fD85&)!QVgEHebFVUo2=`*2yx=AIj92FoP_k1Q0nnbJE|5Fch6< zKplaO^FJ{>b>EPK7l&pJ1#;UL4ILk~%Lzur<S;BCMktQO;`7tT!Ug4XjdQKD&rKgjd=VTjG*?6BWSFiSX4CwtaPcPp z<(rmSO}J>If9U3>*|czBiU0AN8MEqeRvy86S)nL8B)@;?#GRpKnf#>uusnSD#5>%b z35dsD`R4nZYv&!o&AV=^0#*A1`=1Ipg}~UwyUvTjKIF&2hb{%a`s9b`>yY9IeH~St zqOV64XXxvQ;xT+FA5-Kl>$Q1>zfh)Uo0fIxAs9?XA%Jjf=2$q(vS9jd!M6%-=Y4xq zFzX#=ywNA7NqjbfwHDxUNi91P$2daKenB>5vtTC^PI|mWbP@b7pjF8jO{=*l&zZW z^X+rvf%;Q{r?|l2(A}qof{$7Qu1kSamjjn662^qQjo{I)V6b`y0WnbYu!5Cs3N&>D zt2+Z*y6%hwdiw&O84dP69q17dEx;qsUe3$R&IPzCw){g`HUP0X0T4$Q4aW!&$K@Zj zD;O0n`xJ5d+&G`RsJ6T(>y*nI!>TOEsN+|T&u&~)m4;PD+#S1eEUbMJinuzB6h?Xk zAr?;0`PPBi?xoz?U~cVVZrz<{mJXc{9y%X7)ECJ43=#C+_4hMM;77yXr7A*X1b@(AX-a%I6L<}nJ&W=&e{M@q`!_ekfLb! z{oVT)E`c}>?hcD7#lgH~jXJLo^u1-R5>-MsE-bn+c74o$HdL@> z+Hg-_5H2cyE$?O?J6&9~b*ZTSZc%-xsB!veIKS}5;Pt`jLtz{xd+e^UB3xYZ+SZ#} zuWQ2D8~m51UcusA+hjpV0?$s~5H4 zNrh-MJAd}+n-%`Sm$uIvLfOq=3%dO2RyrzO=C{pl4{ovqOm#qYX?kAInd|vlVJLs= z^r3sT^$R%*gQ403@5}TV`N6E3U{>X9o?q?v&e_9xVCm`E`QhABFz#d5j{!;>58OY{ ze*1AisZ)L+982yG?{B@`1y<57FG5uDq7tk!w$=S2TbGyr5g0-KGS%~*5>33PLGPd* z>vF|`gbe~vOP5Z|!`y`7ZCTV-hl{F~inaxdwuOpn@l)5j#+f72?X$Vz+=3g&zH)3% zIoA`)ty#*g3+C2^avQ)CB?|Ue`p5i(SZ7tv=LZVwe{5*LuV(2QKRTo!`dID;;g}%U z&VBaW=bi#0$jxEnhG~I*zzaHVSepUqJ>UHL{%QW&g_#TfN*v`|u-(gT3L7?js87pE zn>vi)X|raV?rI89fE28aciC@zeUyXBfEC(wQ3YDkYzY4M#;z@>jq40cNQ)#Sfe^bZ zqq_v+W+V`pVq^neFyJ^cwy}bRiE*)!1WqW4)5PgCZakSj;PfFNKNvL+mg3G>I1k3@ zLm+)Hn|3BJ(`j^cLDEAeWZKC?JJX`Z9%b~>{%0j*Xp^QhJLl*-+KbMA{`+eG6={}E zhg~sI+f5mt8*%p|~P8+X`%FnD3 zUUs%;YI15}>ed8)HA9L(?)VvZS_>}1{khN_(suakRy!Z}uJvxV zp(DdkO{=zLrR`(KBL}MO_K&Z(edb^to0r)hFbB`azixWwz0Z|QMWq34vVp+S^ z2_Q!G;~g}2F8VLtz62jDsQ2Fb)mtfr3X}^x#)%FvTZ+DI+5p;dW^v~BOA7-0fT!Fq zPmyXM`JjIg+@QP7foAA}Re^)c1OyIgU31D}OB-vRN=qsFi2$?@_=J^Y{y04xchY}y z&vA6X7VbjoLunmswP~GcS+ZK0&>czZ&D=^o5lG++B%C2%>ng2lN^1;3Q}EU0z729dcZqVCN|d6niTU~`nf{`fS;sKOZf1#3kL zV_b12r7*yJD;6t)f>m-$(X(gSj>Tn6_8cFt)XpflK82N|i9;z;yEKl-MyT^8B_mQ} zxcZ^tLqpnPUl~hw3?kSpgZ>vco=Z9}BG@K=jGU7|;a#4EV&DBKS;c#&-aVC4sNwQu z{<0KpxHGpn2VJlfN{^+hYtz-b&~u4uTe{i|#KdLI#K3*ZXWUv{x1V7-Zbf=~gl_vD zc3qq0c-8qoc(pAqYtLF?ju5Et&Y1jODFiRmhv>Q_ZI9CS7~P(v9Z}j5qmP2Ys8ZkQ zzTFKI$}>=kG{WjT{mx)#_*|%aO}W{aAYJf$unKBFfOY@g6)vphv51;U8=KR{gZo$1 z_-y$Sik<>J7t4@lB%wIBy@#6SIlaOYD{J z(S-j8YuVzmmh2E8sJsLJSS+I!ITbG?z)0-645+d*EgFIH&1O#z#Bo5>I|<5muo3noM)e;%K6?U-`&~wXT#USZ?D&XX8pYt%EaY8 zencMMk_~Kww)HKIA*wz}H6yAye17eX4b|s{O#|w6e>57I@FXW@5o|ws1x>t)tk=++ zZ^Lfi@OU&+@&mfKZ%fe+0ebL!U~Z*!<@%$FWb5&0>+$uosC5YGhc?=gZg`Y838rOamo{?}G8WIWMvi^XE=0v>U{$@%_ zsRNve$qw*g-FDo-RjR@|))M8Za(dzQTXRs{Mp{09WU1zE&IKs{h1GM9-JiHWo!qEgpIV!M0yr8YP)PHEW_cuJh-vKe&Z123&j{3| zHKq_4oVKq_4PpEG;Aba4J^2S`*!%G6gR39Dyml^XI|fl)q{$sIjzJ1wxsU?D!GGOD z>jUeDEs;70JBGBUQ}x#SH$J#=@9ovn$7P?C{Z1CC8$f5^n+rRHz^wkKM698oO5`~5 zui!iwW6>srZmhZ1nT^WL%MsTL@$OMB^>}`tx=+X)V8HL3_MeWCmLzG5lD0 zN0K}mC68i{6i2?!TTZA4ffJX72s2Y6swBZ6%geK!K;9ka9&#Ij-2089HYmAYaj)W$ zV|^?+Fp6MnAKgR!o-JALHl~>o~FbAz5@TyH*-Lwm!1{wk2lm+jK=nE+j`Lqa%~Ckt?xNQ&E#AMtZ-b zjQ;VEYGq(?9M1kFRqb~L%(vZ;QUdoL?pNX154@W_E{!21fy7}-Z3u}1wzS3eR3xP6 zZv>^BnaX<*xaj6`H%o5tH_oCFSHwLQXD+~4TrvJ%?!+?#bT+OS*+a`xOiQrnSv$}! zJ6nR)6n6(pif>uHQ}#<)@$MCpP9}Jlvlnq@n#+UdvNR-^xwLaGOW$OWCb ztOEx_I3mhruRt8d2i1yOxKs&*4_rW!%eI1-%en1vP3$;jpcYCWoOqNEE=7h14eSF} zH^YM_mu&`Pkl{g-%bw(8Ri5GDHn*$M3qb^2g8%{Nv~w=&<^%U-c+ljsPCigC!-FPQ zpkZF&=U$tI`ww-pO;D*kBIoZvd@fRv>Ep%LE6~g$arjmFDz=t zYZiRb+ty2md1K<1t=}_*x{!-+q1YIKlWNg1@$E8E=c| zZ?gV+4FBG4V7x~q|4{Y`G0e0wuoo?OLNO)Gv|aF2)z`!LYKU(sIfmbwhbkFgEAg*( z4zkC1{Z_`;O|bpe{6N10e&KW(!4B2r2ZmY)OPOgE!Jhh2IX(kBEQBje*Anb$iI^KW zZEIm>ECTk7Zm5Qt?jcw=PtFav<%2S2MoO>~)%k&mx&b{ibAVu{gko-wDF?xLgd!G5 z#dsEOv9Dj~q9Vd0@Cm|F%xlP3gA7PXsXgfMF)Kj1=C8Cc4DUhmx*2Wo*k#*V;X&upZ_1f|KI;l z{~n2i0Xe@e1=Wij0Dn@(KZJMBiw6M!E&&B7j0Pm{%}5O416n|5C6)mJfLH{tGoC@V zv1(cOx?ZKTLY;7hGAW3H3R@uF6-aN1M@+W$E&f)&DN|^SjVS2Hc%9@#4+B>0gMZTp z@hvC~DEK4zIY9h%c)siJU%2Hjy67O{SAgabpY5#o=h1#uSKul1_Iwpa33q^Z^An;o z(5N-(%~CT%X+dkzTcuW)j%aOqyVUN|tF+a6REoOuAt+7%Um2z49FW$c4yEDyvty;E4lr@vFqlXV18Xenv_&~q-lqFFRuMuQe(-_HC<^Ang zp3@NRy;V06U5TL6ki)8BW1dt4)6!yERVQGQeV8H!#wpxL1lPa|a)Xzh{?u&qlHW$B z)gzO+yaV;LYT$N)>>`gfPOdXx5r#_Pt!0>8ge~U|x!jgA+`0(c%P{J)i89=@2-o}+ zxDvV?Ds}fhjO{4H4_(!sG8|ll>&x&xm+hN}{h0pz9bT5{$7DIN4mVK&YvnB`m?Jx_ zsS}vaAjQz7m1QT0k!5P9ki+HD7#X$`!eqH+y4+6KTL77?vID8T!>Ixv`AffD!r)y$3Twd_3Tdkh*% z{Q9C)JXqoqH`&|aTj6;y;7a4OpDpz4oa@>7uxD2(zPrTr-C^&C?-KpzRjbsMBbSfN zPRxE?{Hi!w;x^xm-`;d<(>&Ny;X+J!34X8ycP+tzB{=+;4&dOO)P3o$G!Ms~3n6Bp z5(QBBuJC|e`xx@y4p&$JJN~Y80lw`yV3{yA5xQr$&&EsP4Ode)q-(@-_a89yPbEmy eFRd6U=5N6*LlE494MQ8)d!g7+7kjUZ(*FXvf-}hg literal 0 HcmV?d00001 From 13e6ee41ae00430b935733f324419074c170b250 Mon Sep 17 00:00:00 2001 From: skymike Date: Tue, 11 Nov 2025 01:16:59 +0100 Subject: [PATCH 42/43] Add Linux deployment instructions to README Included a section in the README.md for Linux deployment, providing a reference to a systemd unit and Nginx TLS proxy example located in deploy/linux/README.md. This addition aims to assist users in setting up the application on Linux systems. --- README.md | 4 ++ deploy/linux/README.md | 71 ++++++++++++++++++++++++++++++ deploy/linux/nginx/opentrader.conf | 53 ++++++++++++++++++++++ deploy/linux/opentrader.service | 35 +++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 deploy/linux/README.md create mode 100644 deploy/linux/nginx/opentrader.conf create mode 100644 deploy/linux/opentrader.service diff --git a/README.md b/README.md index c3860bb0..d53936a3 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,10 @@ opentrader trade grid Licensed under the [Apache 2.0](http://www.apache.org/licenses/LICENSE-2.0) License. See the [LICENSE](LICENSE) file for more information. +# Linux Deployment + +For a ready-to-use systemd unit and Nginx TLS proxy example (listening on port 9966), see `deploy/linux/README.md`. + # Disclaimer This software is for educational purposes only. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. Do not risk money that you are afraid to lose. There might be bugs in the code - this software DOES NOT come with ANY warranty. diff --git a/deploy/linux/README.md b/deploy/linux/README.md new file mode 100644 index 00000000..114bdca2 --- /dev/null +++ b/deploy/linux/README.md @@ -0,0 +1,71 @@ +OpenTrader Linux Deployment (systemd + Nginx) + +This guide sets up OpenTrader as a Linux service and exposes it securely via Nginx with TLS. The app listens on port 9966 and is reachable on your home network. + +Prerequisites +- Linux host with sudo +- Node.js 22.x on the host (required by OpenTrader). Recommended: nvm +- Nginx installed (for TLS/domain) + +1) Install Node 22 and OpenTrader +- Install nvm and Node 22: + - curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash + - source ~/.nvm/nvm.sh + - nvm install 22 && nvm use 22 +- Install OpenTrader CLI globally: + - npm install -g opentrader + +2) Create a dedicated user and working dir +- sudo useradd -r -m -d /opt/opentrader opentrader || true +- sudo mkdir -p /opt/opentrader +- sudo chown -R opentrader:opentrader /opt/opentrader + +3) Initialize configs and admin password +- Create configs in /opt/opentrader: + - sudo cp -n /opt/opentrader/config.sample.json5 /opt/opentrader/config.json5 || true + - sudo cp -n /opt/opentrader/exchanges.sample.json5 /opt/opentrader/exchanges.json5 || true + - If samples aren’t present in /opt/opentrader, copy them from this repo root: + - sudo cp config.sample.json5 /opt/opentrader/config.json5 + - sudo cp exchanges.sample.json5 /opt/opentrader/exchanges.json5 +- Set the admin password (runs as the opentrader user): + - sudo -u opentrader opentrader set-password + +4) Install the systemd unit +- Copy the unit file: + - sudo cp deploy/linux/opentrader.service /etc/systemd/system/opentrader.service +- Reload and start: + - sudo systemctl daemon-reload + - sudo systemctl enable --now opentrader +- Check status/logs: + - systemctl status opentrader + - journalctl -u opentrader -f + +By default, the service listens on 0.0.0.0:9966 so it’s available on your LAN. + +5) Nginx reverse proxy with TLS (optional, for domain) +- Replace example.com with your domain in deploy/linux/nginx/opentrader.conf and copy it: + - sudo cp deploy/linux/nginx/opentrader.conf /etc/nginx/sites-available/opentrader.conf + - sudo ln -s /etc/nginx/sites-available/opentrader.conf /etc/nginx/sites-enabled/opentrader.conf +- Test and reload Nginx: + - sudo nginx -t && sudo systemctl reload nginx +- Obtain Let’s Encrypt certificates (certbot example): + - sudo apt-get update && sudo apt-get install -y certbot python3-certbot-nginx + - sudo certbot --nginx -d example.com -d www.example.com + +6) Firewall +- For LAN access without Nginx, open 9966/tcp: + - sudo ufw allow 9966/tcp +- If using Nginx + TLS, allow 80,443: + - sudo ufw allow 80/tcp + - sudo ufw allow 443/tcp + +Service management +- Start: sudo systemctl start opentrader +- Stop: sudo systemctl stop opentrader +- Restart: sudo systemctl restart opentrader + +Notes +- Configure exchanges in /opt/opentrader/exchanges.json5 (created from exchanges.sample.json5) +- Configure strategy in /opt/opentrader/config.json5 (created from config.sample.json5) +- To change port/host, edit ExecStart in the unit to pass --host and --port or run `opentrader up --host 0.0.0.0 --port 9966` manually. + diff --git a/deploy/linux/nginx/opentrader.conf b/deploy/linux/nginx/opentrader.conf new file mode 100644 index 00000000..20fd083e --- /dev/null +++ b/deploy/linux/nginx/opentrader.conf @@ -0,0 +1,53 @@ +# Replace example.com with your domain. +# This config proxies HTTPS traffic to OpenTrader listening on localhost:9966. + +upstream opentrader_upstream { + server 127.0.0.1:9966; + keepalive 64; +} + +server { + listen 80; + listen [::]:80; + server_name example.com www.example.com; + + # Let’s Encrypt challenge + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name example.com www.example.com; + + # Managed by certbot after issuance + ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + # Proxy to OpenTrader UI/API + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://opentrader_upstream; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + + # Increase timeouts for long requests if needed + proxy_read_timeout 300s; + proxy_connect_timeout 60s; + proxy_send_timeout 300s; +} + diff --git a/deploy/linux/opentrader.service b/deploy/linux/opentrader.service new file mode 100644 index 00000000..8283f0fd --- /dev/null +++ b/deploy/linux/opentrader.service @@ -0,0 +1,35 @@ +[Unit] +Description=OpenTrader (self-hosted crypto trading bot) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=opentrader +Group=opentrader +WorkingDirectory=/opt/opentrader +Environment=NODE_ENV=production +# Optional: point to custom strategies directory +# Environment=CUSTOM_STRATEGIES_PATH=/opt/opentrader/strategies +ExecStart=/usr/bin/env opentrader up --host 0.0.0.0 --port 9966 -d +ExecStop=/usr/bin/env opentrader down +Restart=on-failure +RestartSec=5 + +# Hardening (optional) +NoNewPrivileges=true +ProtectSystem=full +ProtectHome=true +PrivateTmp=true +AmbientCapabilities= +CapabilityBoundingSet= +LockPersonality=true +ProtectControlGroups=true +ProtectKernelModules=true +ProtectKernelTunables=true +RestrictRealtime=true +RestrictSUIDSGID=true + +[Install] +WantedBy=multi-user.target + From acaf6d80820afa95ae1cb7030056c45195c01c96 Mon Sep 17 00:00:00 2001 From: skymike Date: Tue, 11 Nov 2025 01:46:44 +0100 Subject: [PATCH 43/43] full redesign and move to npm --- .github/workflows/node-build.yml | 37 ++++++++++ .nvmrc | 1 + _codex_tmp/.npmrc.bak | 7 ++ _codex_tmp/deploy.bak/linux/README.md | 73 +++++++++++++++++++ .../deploy.bak/linux/config.sample.json5 | 22 ++++++ .../deploy.bak/linux/exchanges.sample.json5 | 24 ++++++ .../deploy.bak/linux/nginx/opentraderx.conf | 60 +++++++++++++++ .../deploy.bak/linux/opentraderx.service | 35 +++++++++ app/frontend/index.html | 2 +- packages/bot/src/app.ts | 2 +- 10 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/node-build.yml create mode 100644 .nvmrc create mode 100644 _codex_tmp/.npmrc.bak create mode 100644 _codex_tmp/deploy.bak/linux/README.md create mode 100644 _codex_tmp/deploy.bak/linux/config.sample.json5 create mode 100644 _codex_tmp/deploy.bak/linux/exchanges.sample.json5 create mode 100644 _codex_tmp/deploy.bak/linux/nginx/opentraderx.conf create mode 100644 _codex_tmp/deploy.bak/linux/opentraderx.service diff --git a/.github/workflows/node-build.yml b/.github/workflows/node-build.yml new file mode 100644 index 00000000..1192c304 --- /dev/null +++ b/.github/workflows/node-build.yml @@ -0,0 +1,37 @@ +name: OpenTraderX Build (pnpm, frozen lockfile) + +on: + push: + branches: + - integrate-open-trader + - main + pull_request: + branches: + - '**' + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node 22 + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.12.1 + + - name: Install (frozen lockfile) + run: pnpm install --frozen-lockfile + + - name: Build + run: | + if [ -f package.json ]; then pnpm build; else echo "No package.json in repo root; skipping build"; fi diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000..2bd5a0a9 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +22 diff --git a/_codex_tmp/.npmrc.bak b/_codex_tmp/.npmrc.bak new file mode 100644 index 00000000..a251730e --- /dev/null +++ b/_codex_tmp/.npmrc.bak @@ -0,0 +1,7 @@ +lockfile=true +prefer-frozen-lockfile=true +shared-workspace-lockfile=true +strict-peer-dependencies=true +auto-install-peers=false +engine-strict=true + diff --git a/_codex_tmp/deploy.bak/linux/README.md b/_codex_tmp/deploy.bak/linux/README.md new file mode 100644 index 00000000..4a2db82e --- /dev/null +++ b/_codex_tmp/deploy.bak/linux/README.md @@ -0,0 +1,73 @@ +OpenTraderX Linux Deployment (systemd + Nginx) + +This guide sets up OpenTraderX as a Linux service and exposes it securely via Nginx with TLS. The app listens on port 9966 and is reachable on your home network. + +Prerequisites +- Linux host with sudo +- Node.js 22.x on the host (required by OpenTrader). Recommended: nvm +- Nginx installed (for TLS/domain) + +1) Install Node 22 and OpenTrader CLI +- Install nvm and Node 22: + - curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash + - source ~/.nvm/nvm.sh + - nvm install 22 && nvm use 22 +- Install OpenTrader CLI globally (OpenTraderX uses the opentrader CLI): + - npm install -g opentrader + +2) Create a dedicated user and working dir +- sudo useradd -r -m -d /opt/opentrader opentrader || true +- sudo mkdir -p /opt/opentrader +- sudo chown -R opentrader:opentrader /opt/opentrader + +3) Initialize configs and admin password +- Copy samples from this repo and edit: + - sudo cp deploy/linux/config.sample.json5 /opt/opentrader/config.json5 + - sudo cp deploy/linux/exchanges.sample.json5 /opt/opentrader/exchanges.json5 + - sudo nano /opt/opentrader/config.json5 + - sudo nano /opt/opentrader/exchanges.json5 +- Set the admin password (runs as the opentrader user): + - sudo -u opentrader opentrader set-password + +4) Install the systemd unit +- Copy the unit file: + - sudo cp deploy/linux/opentraderx.service /etc/systemd/system/opentraderx.service +- Reload and start: + - sudo systemctl daemon-reload + - sudo systemctl enable --now opentraderx +- Check status/logs: + - systemctl status opentraderx + - journalctl -u opentraderx -f + +By default, the service listens on 0.0.0.0:9966 so it’s available on your LAN. + +5) Nginx reverse proxy with TLS (optional, for domain) +- Replace example.com with your domain in deploy/linux/nginx/opentraderx.conf and copy it: + - sudo cp deploy/linux/nginx/opentraderx.conf /etc/nginx/sites-available/opentraderx.conf + - sudo ln -s /etc/nginx/sites-available/opentraderx.conf /etc/nginx/sites-enabled/opentraderx.conf +- Test and reload Nginx: + - sudo nginx -t && sudo systemctl reload nginx +- Obtain Let’s Encrypt certificates (certbot example): + - sudo apt-get update && sudo apt-get install -y certbot python3-certbot-nginx + - sudo certbot --nginx -d example.com -d www.example.com + +Optional UI branding via Nginx +- The provided Nginx config includes a sub_filter that changes page title text from "OpenTrader" to "OpenTraderX" for HTML responses only. +- If you prefer to disable this, remove the sub_filter lines under the location / block. + +6) Firewall +- For LAN access without Nginx, open 9966/tcp: + - sudo ufw allow 9966/tcp +- If using Nginx + TLS, allow 80,443: + - sudo ufw allow 80/tcp + - sudo ufw allow 443/tcp + +Service management +- Start: sudo systemctl start opentraderx +- Stop: sudo systemctl stop opentraderx +- Restart: sudo systemctl restart opentraderx + +Notes +- Configure exchanges in /opt/opentrader/exchanges.json5 (created from exchanges.sample.json5) +- Configure strategy in /opt/opentrader/config.json5 (created from config.sample.json5) +- To change port/host, edit ExecStart in the unit to pass --host and --port or run `opentrader up --host 0.0.0.0 --port 9966` manually. diff --git a/_codex_tmp/deploy.bak/linux/config.sample.json5 b/_codex_tmp/deploy.bak/linux/config.sample.json5 new file mode 100644 index 00000000..1e59553c --- /dev/null +++ b/_codex_tmp/deploy.bak/linux/config.sample.json5 @@ -0,0 +1,22 @@ +// Example strategy configuration for OpenTraderX +// Copy to /opt/opentrader/config.json5 and adjust values +{ + // Select one of: "grid", "dca", "rsi" + template: "grid", + + // Strategy-specific settings + settings: { + // GRID strategy + highPrice: 70000, // upper price of the grid + lowPrice: 60000, // lower price of the grid + gridLevels: 20, // number of grid levels + quantityPerGrid: 0.0001 // quantity in base currency per grid + }, + + // Trading pair + pair: "BTC/USDT", + + // Exchange account label defined in exchanges.json5 + exchange: "DEFAULT" +} + diff --git a/_codex_tmp/deploy.bak/linux/exchanges.sample.json5 b/_codex_tmp/deploy.bak/linux/exchanges.sample.json5 new file mode 100644 index 00000000..3befb8ef --- /dev/null +++ b/_codex_tmp/deploy.bak/linux/exchanges.sample.json5 @@ -0,0 +1,24 @@ +// Example exchanges configuration for OpenTraderX +// Copy to /opt/opentrader/exchanges.json5 and fill in your API credentials. +// You may define multiple labeled accounts; the "exchange" field in config.json5 +// should match one of the top-level keys here (e.g., "DEFAULT"). +{ + DEFAULT: { + // One of: OKX, BYBIT, BINANCE, KRAKEN, COINBASE, GATEIO, BITGET + code: "BYBIT", + + // Human-friendly name for your account + name: "Main Bybit", + + // API credentials (create read/trade keys on the exchange) + credentials: { + apiKey: "YOUR_API_KEY", + secret: "YOUR_API_SECRET", + + // Optional fields per-exchange; uncomment if required + // password: "YOUR_API_PASSPHRASE", + // uid: "YOUR_UID", + } + } +} + diff --git a/_codex_tmp/deploy.bak/linux/nginx/opentraderx.conf b/_codex_tmp/deploy.bak/linux/nginx/opentraderx.conf new file mode 100644 index 00000000..4368b4d2 --- /dev/null +++ b/_codex_tmp/deploy.bak/linux/nginx/opentraderx.conf @@ -0,0 +1,60 @@ +# Replace example.com with your domain. +# This config proxies HTTPS traffic to OpenTraderX listening on localhost:9966. + +upstream opentraderx_upstream { + server 127.0.0.1:9966; + keepalive 64; +} + +server { + listen 80; + listen [::]:80; + server_name example.com www.example.com; + + # Let’s Encrypt challenge + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name example.com www.example.com; + + # Managed by certbot after issuance + ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + # Proxy to OpenTraderX UI/API + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # Force plaintext from upstream so sub_filter can work + proxy_set_header Accept-Encoding ""; + proxy_pass http://opentraderx_upstream; + proxy_http_version 1.1; + proxy_set_header Connection ""; + + # Optional: lightweight UI branding tweak + # Caution: Only modifies text/html to avoid breaking JS/CSS bundles + sub_filter_types text/html; + sub_filter 'OpenTrader' 'OpenTraderX'; + sub_filter_once off; + } + + # Increase timeouts for long requests if needed + proxy_read_timeout 300s; + proxy_connect_timeout 60s; + proxy_send_timeout 300s; +} diff --git a/_codex_tmp/deploy.bak/linux/opentraderx.service b/_codex_tmp/deploy.bak/linux/opentraderx.service new file mode 100644 index 00000000..359bc334 --- /dev/null +++ b/_codex_tmp/deploy.bak/linux/opentraderx.service @@ -0,0 +1,35 @@ +[Unit] +Description=OpenTraderX (self-hosted crypto trading bot) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=opentrader +Group=opentrader +WorkingDirectory=/opt/opentrader +Environment=NODE_ENV=production +# Optional: point to custom strategies directory +# Environment=CUSTOM_STRATEGIES_PATH=/opt/opentrader/strategies +ExecStart=/usr/bin/env opentrader up --host 0.0.0.0 --port 9966 -d +ExecStop=/usr/bin/env opentrader down +Restart=on-failure +RestartSec=5 + +# Hardening (optional) +NoNewPrivileges=true +ProtectSystem=full +ProtectHome=true +PrivateTmp=true +AmbientCapabilities= +CapabilityBoundingSet= +LockPersonality=true +ProtectControlGroups=true +ProtectKernelModules=true +ProtectKernelTunables=true +RestrictRealtime=true +RestrictSUIDSGID=true + +[Install] +WantedBy=multi-user.target + diff --git a/app/frontend/index.html b/app/frontend/index.html index ef4ac11b..e09ccb06 100644 --- a/app/frontend/index.html +++ b/app/frontend/index.html @@ -4,7 +4,7 @@ - Opentrader + OpenTraderX diff --git a/packages/bot/src/app.ts b/packages/bot/src/app.ts index 8fc4799a..3e974ee5 100644 --- a/packages/bot/src/app.ts +++ b/packages/bot/src/app.ts @@ -48,7 +48,7 @@ export class App { await server.listen(); logger.info(`RPC Server listening on port ${params.server.port}`); - logger.info(`OpenTrader UI: http://${params.server.host}:${params.server.port}`); + logger.info(`OpenTraderX UI: http://${params.server.host}:${params.server.port}`); return new App(platform, server); }