diff --git a/FeatureEngineering/MarketStructure/trend_regime.py b/FeatureEngineering/MarketStructure/trend_regime.py index 50f540d..4545da4 100644 --- a/FeatureEngineering/MarketStructure/trend_regime.py +++ b/FeatureEngineering/MarketStructure/trend_regime.py @@ -85,7 +85,7 @@ def apply( if self.vol_required: high_vol = ( (struct_vol.get( - "bos_bull_struct_vol", pd.Series(False, index=idx)) == "high") + "bos_bull_struct_vol", pd.Series(False, index=idx)) == "high") | (struct_vol.get( "bos_bear_struct_vol", pd.Series(False, index=idx)) == "high") | (struct_vol.get( diff --git a/Strategies/Samplestrategy.py b/Strategies/Samplestrategy.py index 661bb7a..7d39fb5 100644 --- a/Strategies/Samplestrategy.py +++ b/Strategies/Samplestrategy.py @@ -1,13 +1,14 @@ import pandas as pd import talib.abstract as ta -from core.backtesting.reporting.core.context import ContextSpec -from core.backtesting.reporting.core.metrics import ExpectancyMetric, MaxDrawdownMetric +from core.reporting.core.context import ContextSpec +from core.reporting.core.metrics import ExpectancyMetric, MaxDrawdownMetric from FeatureEngineering.MarketStructure.engine import MarketStructureEngine from core.strategy.base import BaseStrategy from core.strategy.informatives import informative class Samplestrategy(BaseStrategy): + strategy_name = "Sample Strategy (Report)" def __init__( self, @@ -28,21 +29,9 @@ def populate_indicators_M30(self, df): # --- minimum techniczne df["atr"] = ta.ATR(df, 14) - # --- market structure HTF - df = MarketStructureEngine.apply( - df, - features=[ - "pivots", - "price_action", - "follow_through", - "structural_vol", - "trend_regime", - ], - ) - # --- bias flags (czytelne na GitHubie) - df["bias_long"] = df["trend_regime"] == "trend_up" - df["bias_short"] = df["trend_regime"] == "trend_down" + + return df @@ -53,17 +42,7 @@ def populate_indicators(self): # --- base indicators df["atr"] = ta.ATR(df, 14) - # --- market structure - df = MarketStructureEngine.apply( - df, - features=[ - "pivots", - "price_action", - "follow_through", - "structural_vol", - "trend_regime", - ], - ) + df['low_15'] = df['low'].rolling(15).min() df['high_15'] = df['high'].rolling(15).max() @@ -135,8 +114,6 @@ def populate_entry_trend(self): axis=1 ) - print(df["signal_entry"].notna().sum()) - self.df = df @@ -144,10 +121,10 @@ def populate_entry_trend(self): return df - def build_report_config(self): + def build_report_spec(self): return ( super() - .build_report_config() + .build_report_spec() .add_metric(ExpectancyMetric()) .add_metric(MaxDrawdownMetric()) .add_context( diff --git a/backtest_run.py b/backtest_run.py index 8627d87..f973fce 100644 --- a/backtest_run.py +++ b/backtest_run.py @@ -3,5 +3,25 @@ if __name__ == "__main__": + import cProfile + import pstats + from pathlib import Path + from datetime import datetime + + run_path = Path( + f"results/run_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ) + run_path.mkdir(parents=True, exist_ok=True) + + profile_path = run_path / "profile_full.prof" + + profiler = cProfile.Profile() + profiler.enable() + BacktestRunner(cfg).run() + profiler.disable() + profiler.dump_stats(profile_path) + + print(f"[PROFILE] saved to {profile_path}") + diff --git a/config/backtest.py b/config/backtest.py index f42f44f..24610a5 100644 --- a/config/backtest.py +++ b/config/backtest.py @@ -1,17 +1,23 @@ import logging +from enum import Enum +from pathlib import Path + +from config.logger_config import LoggerConfig logging.basicConfig(level=logging.INFO) # ================================================== -# DATA +# DATA SOURCE & TIME # ================================================== MARKET_DATA_PATH = "market_data" -BACKTEST_DATA_BACKEND = "dukascopy" # "dukascopy" +BACKTEST_DATA_BACKEND = "dukascopy" # "dukascopy", "csv", ... + +SERVER_TIMEZONE = "UTC" TIMERANGE = { - "start": "2025-12-15", - "end": "2025-12-31", + "start": "2025-01-01", + "end": "2025-12-29", } BACKTEST_MODE = "single" # "single" | "split" @@ -22,92 +28,113 @@ "FINAL": ("2025-12-24", "2025-12-31"), } -# Missing data handling metadata (report/UI) -# Examples: -# - "Forward-fill OHLC gaps" -# - "Drop candles with gaps" -# - "Leave gaps (no fill)" +# Metadata only (for reports / README) MISSING_DATA_HANDLING = "Forward-fill OHLC gaps" -SERVER_TIMEZONE = "UTC" - # ================================================== -# STRATEGY +# STRATEGY DEFINITION # ================================================== -# If None, report will fall back to STRATEGY_CLASS. -STRATEGY_NAME = "Sample " +STRATEGY_CLASS = "Samplestrategyreport" -# Optional short description (used in BacktestConfigSection) +STRATEGY_NAME = "Sample Strategy" STRATEGY_DESCRIPTION = "Sample strategy for dashboard showcase" -# Strategy class locator (string used by your loader) -STRATEGY_CLASS = "Samplestrategyreport" STARTUP_CANDLE_COUNT = 600 SYMBOLS = [ "XAUUSD", + "EURUSD" ] TIMEFRAME = "M1" - # ================================================== -# EXECUTION (SIMULATED) +# EXECUTION MODEL (SIMULATED) # ================================================== INITIAL_BALANCE = 10_000 -# Slippage assumed in "pips" for FX-like instruments in current convention. -SLIPPAGE = 0.1 - -# Optional: separate slippage for entry/exit (metadata now; logic later) -SLIPPAGE_ENTRY = None # if None -> use SLIPPAGE -SLIPPAGE_EXIT = None # if None -> use SLIPPAGE - MAX_RISK_PER_TRADE = 0.005 -# Execution delay metadata (not implemented yet) -# Could be "None", "1 bar", "200ms", etc. -EXECUTION_DELAY = "None" +# Slippage (metadata + future logic) +SLIPPAGE = 0.1 +SLIPPAGE_ENTRY = None # if None -> SLIPPAGE +SLIPPAGE_EXIT = None # if None -> SLIPPAGE + +# Execution / order semantics (metadata for now) +EXECUTION_DELAY = "None" # e.g. "1 bar", "200ms" -# Order types (metadata now; logic later) -# Use: "market" | "limit" -ENTRY_ORDER_TYPE_DEFAULT = "market" +ENTRY_ORDER_TYPE_DEFAULT = "market" # "market" | "limit" EXIT_ORDER_TYPE_DEFAULT = "limit" TP_ORDER_TYPE_DEFAULT = "limit" +EXIT_OVERRIDES_DESC = ( + "SL/BE/EOD treated as market exits; " + "TP exits treated as limit (strategy-dependent)." +) -# explanation for report (avoid wrong claims) -EXIT_OVERRIDES_DESC = "SL/BE/EOD treated as market exits; TP exits treated as limit (strategy-dependent)." +SPREAD_MODEL = "fixed_cost_overlay" # or "bid_ask_simulation" -# Spread model metadata: -# - "fixed_cost_overlay" (current reality: compute spread_usd_* from per-instrument fixed spreads) -# - "bid_ask_simulation" (future: actual bid/ask price simulation in fills) -SPREAD_MODEL = "fixed_cost_overlay" +# ================================================== +# CAPITAL & RISK MODEL (REPORTING / FUTURE) +# ================================================== + +POSITION_SIZING_MODEL = "Risk-based sizing (position_sizer)" +LEVERAGE = "1x" +MAX_CONCURRENT_POSITIONS = None # None = unlimited (diagnostic) +CAPITAL_FLOOR = None # e.g. 5000 # ================================================== -# CAPITAL MODEL (REPORT / FUTURE CONTROLS) +# REPORTING & ANALYTICS # ================================================== -# Position sizing model label for report -POSITION_SIZING_MODEL = "Risk-based sizing (position_sizer)" +# Enable / disable reporting entirely +ENABLE_REPORT = True -# (FUTURE IDEA)Leverage is metadata unless you model margin / leverage constraints -LEVERAGE = "1x" +# ---- STDOUT rendering control ---- +class StdoutMode(str, Enum): + OFF = "off" + CONSOLE = "console" + FILE = "file" + BOTH = "both" + +REPORT_STDOUT_MODE = StdoutMode.OFF -# (FUTURE IDEA) Concurrency -MAX_CONCURRENT_POSITIONS = None # None => Unlimited (diagnostic) +# Used only if FILE or BOTH +REPORT_STDOUT_FILE = "results/stdout_report.txt" -# (FUTURE IDEA) Kill-switch / capital floor -CAPITAL_FLOOR = None # e.g. 5000, or None +# ---- Dashboard / persistence ---- +GENERATE_DASHBOARD = True +PERSIST_REPORT = True + +# Fail if no trades (research safety) +REPORT_FAIL_ON_EMPTY = True # ================================================== -# OUTPUT / UI +# RUNTIME / DEBUG # ================================================== -SAVE_TRADES_CSV = False -PLOT_ONLY = False \ No newline at end of file +PLOT_ONLY = False # Skip backtest, just plots +SAVE_TRADES_CSV = False # Legacy / debug only + +from config.logger_config import LoggerConfig + +LOGGER_CONFIG = LoggerConfig( + stdout=True, + file=True, + timing=True, + profiling=False, + log_dir=Path("results/logs"), +) + +PROFILING = True + +USE_MULTIPROCESSING_STRATEGIES = False +USE_MULTIPROCESSING_BACKTESTS = True + +MAX_WORKERS_STRATEGIES = None # None = os.cpu_count() +MAX_WORKERS_BACKTESTS = None \ No newline at end of file diff --git a/config/logger_config.py b/config/logger_config.py new file mode 100644 index 0000000..21f142e --- /dev/null +++ b/config/logger_config.py @@ -0,0 +1,137 @@ +from dataclasses import dataclass +from pathlib import Path +import logging +from contextlib import contextmanager +from time import perf_counter +import cProfile +import pstats + + +LOG_PREFIX = { + "DATA": "📈 DATA PREPARER |", + "BT": "🧪 BACKTEST |", + "STRAT": "📐 STRATEGY |", + "REPORT": "📊 REPORT |", + "RUNNER": "🚀 RUN |", +} + +@dataclass +class LoggerConfig: + stdout: bool = True + file: bool = False + timing: bool = True + profiling: bool = False + log_dir: Path | None = None + + +class RunLogger: + def __init__(self, name: str, cfg: LoggerConfig, prefix: str = ""): + self.cfg = cfg + self.name = name + self.prefix = prefix + + self._t0 = perf_counter() + self._t_last = self._t0 + + self.logger = logging.getLogger(name) + self.logger.setLevel(logging.INFO) + self.logger.handlers.clear() + self.logger.propagate = False + self._timings: dict[str, float] = {} + + if cfg.stdout: + h = logging.StreamHandler() + h.setFormatter(logging.Formatter("%(message)s")) + self.logger.addHandler(h) + + if cfg.file: + if cfg.log_dir is None: + raise ValueError("LoggerConfig.file=True requires log_dir") + + cfg.log_dir.mkdir(parents=True, exist_ok=True) + path = cfg.log_dir / f"{name}.log" + + fh = logging.FileHandler(path, encoding="utf-8") + fh.setFormatter( + logging.Formatter("%(asctime)s | %(message)s") + ) + self.logger.addHandler(fh) + + # -------------------- + # BASIC LOG + # -------------------- + def log(self, msg: str): + if self.prefix: + self.logger.info(f"{self.prefix} {msg}") + else: + self.logger.info(msg) + + # -------------------- + # TIMED STEP + # -------------------- + def step(self, label: str): + if not self.cfg.timing: + return + + now = perf_counter() + delta = now - self._t_last + total = now - self._t0 + + msg = f"⏱️ {label:<30} +{delta:6.2f}s | total {total:6.2f}s" + self.log(msg) + + self._t_last = now + + def get_timings(self) -> dict[str, float]: + """ + Return collected timing sections. + """ + return dict(self._timings) + + # -------------------- + # CONTEXT TIMER + # -------------------- + @contextmanager + def time(self, label: str): + if not self.cfg.timing: + yield + return + + t0 = perf_counter() + yield + dt = perf_counter() - t0 + self.logger.info(f"⏱️ {label:<30} {dt:6.3f}s") + + @contextmanager + def section(self, name: str): + t0 = perf_counter() + yield + dt = perf_counter() - t0 + self._timings[name] = self._timings.get(name, 0.0) + dt + + +@contextmanager +def profiling(enabled: bool, path): + if not enabled: + yield + return + + pr = cProfile.Profile() + pr.enable() + yield + pr.disable() + stats = pstats.Stats(pr) + stats.sort_stats("cumtime") + stats.dump_stats(path) + + +class NullLogger: + @contextmanager + def section(self, name: str): + yield + + def log(self, msg: str): + pass + + def get_timings(self) -> dict[str, float]: + return {} \ No newline at end of file diff --git a/config/report_config.py b/config/report_config.py new file mode 100644 index 0000000..5ff6e58 --- /dev/null +++ b/config/report_config.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class StdoutMode(str, Enum): + OFF = "off" + CONSOLE = "console" + FILE = "file" + BOTH = "both" + + +@dataclass(frozen=True) +class ReportConfig: + """ + Runtime configuration for report generation. + + Controls: + - stdout rendering + - dashboard generation + - persistence + """ + + # ================================================== + # STDOUT + # ================================================== + + stdout_mode: StdoutMode = StdoutMode.CONSOLE + + # Used only if FILE or BOTH + stdout_file: Path | None = None + + # ================================================== + # DASHBOARD / FILE OUTPUT + # ================================================== + + generate_dashboard: bool = True + persist_report: bool = True + + # ================================================== + # SAFETY / RESEARCH + # ================================================== + + fail_on_empty: bool = True \ No newline at end of file diff --git a/core/backtesting/backtester.py b/core/backtesting/backtester.py deleted file mode 100644 index 57d4133..0000000 --- a/core/backtesting/backtester.py +++ /dev/null @@ -1,221 +0,0 @@ -import traceback -from typing import Optional, Any -import pandas as pd -from concurrent.futures import ProcessPoolExecutor, as_completed -import os - -from config.backtest import INITIAL_BALANCE, SLIPPAGE, MAX_RISK_PER_TRADE -from config.instrument_meta import INSTRUMENT_META, get_spread_abs - -from core.backtesting.execution_policy import ExecutionPolicy -from core.backtesting.simulate_exit_numba import simulate_exit_numba - -from core.domain.execution.exit_processor import ExitProcessor -from core.domain.cost.cost_engine import TradeCostEngine, InstrumentCtx -from core.backtesting.trade_factory import TradeFactory -from core.domain.risk.sizing import position_size -from core.strategy.plan_builder import PlanBuildContext - - -class Backtester: - def __init__(self, - strategy, - execution_policy: Optional[ExecutionPolicy] = None, - cost_engine: Optional[TradeCostEngine] = None - ): - self.strategy = strategy - self.execution_policy = execution_policy or ExecutionPolicy() - self.cost_engine = cost_engine or TradeCostEngine(self.execution_policy) - - def run_backtest(self, df: pd.DataFrame, symbol: Optional[str] = None) -> pd.DataFrame: - if symbol: - return self._backtest_single_symbol(df, symbol) - - all_trades = [] - for sym, group_df in df.groupby("symbol"): - trades = self._backtest_single_symbol(group_df, sym) - if not trades.empty: - all_trades.append(trades) - - return pd.concat(all_trades).sort_values(by="exit_time") if all_trades else pd.DataFrame() - - @staticmethod - def _instrument_ctx(symbol: str) -> InstrumentCtx: - meta = INSTRUMENT_META[symbol] - point_size = float(meta["point"]) - pip_value = float(meta["pip_value"]) - contract_size = float(meta.get("contract_size", 1.0)) - - spread_abs = get_spread_abs(symbol, point_size) - half_spread = 0.5 * spread_abs - - slippage_abs = float(SLIPPAGE) * point_size - - return InstrumentCtx( - symbol=symbol, - point_size=point_size, - pip_value=pip_value, - contract_size=contract_size, - spread_abs=spread_abs, - half_spread=half_spread, - slippage_abs=slippage_abs, - ) - - # ----------------------------- - # Backtest per symbol - # ----------------------------- - def _backtest_single_symbol(self, df: pd.DataFrame, symbol: str) -> pd.DataFrame: - trades = [] - - time_arr = df["time"].dt.tz_localize(None).values - high_arr = df["high"].values - low_arr = df["low"].values - close_arr = df["close"].values - - # instrument ctx do slippage itp. - ctx_inst = self._instrument_ctx(symbol) - - # 1) Build vector-friendly plans once (shared logic) - ctx = PlanBuildContext( - symbol=symbol, - strategy_name=type(self.strategy).__name__, - strategy_config=self.strategy.strategy_config, - ) - plans = self.strategy.build_trade_plans_backtest(df=df, ctx=ctx, allow_managed_in_backtest=False) - - # 2) Precompute arrays used in the hot loop - plan_valid = plans["plan_valid"].values - plan_dir = plans["plan_direction"].values - plan_tag = plans["plan_entry_tag"].values - plan_sl = plans["plan_sl"].values.astype(float) - plan_tp1 = plans["plan_tp1"].values.astype(float) - plan_tp2 = plans["plan_tp2"].values.astype(float) - - plan_sl_tag = plans["plan_sl_tag"].values.astype(str) - plan_tp1_tag = plans["plan_tp1_tag"].values.astype(str) - plan_tp2_tag = plans["plan_tp2_tag"].values.astype(str) - - n = len(df) - - for direction in ("long", "short"): - dir_flag = 1 if direction == "long" else -1 - last_exit_by_tag: dict[str, Any] = {} - - for entry_pos in range(n): - if not plan_valid[entry_pos]: - continue - if plan_dir[entry_pos] != direction: - continue - - entry_tag = str(plan_tag[entry_pos]) - entry_time = time_arr[entry_pos] - - last_exit = last_exit_by_tag.get(entry_tag) - if last_exit is not None and last_exit > entry_time: - continue - - sl = float(plan_sl[entry_pos]) - tp1 = float(plan_tp1[entry_pos]) - tp2 = float(plan_tp2[entry_pos]) - - level_tags = {"SL": plan_sl_tag[entry_pos], "TP1": plan_tp1_tag[entry_pos], "TP2": plan_tp2_tag[entry_pos]} - - entry_price = float(close_arr[entry_pos]) - - entry_price += ctx_inst.slippage_abs if direction == "long" else -ctx_inst.slippage_abs - - size = position_size( - entry_price=entry_price, - stop_price=sl, - max_risk=MAX_RISK_PER_TRADE, - account_size=INITIAL_BALANCE, # lub INITIAL_BALANCE - point_size=ctx_inst.point_size, - pip_value=ctx_inst.pip_value, - ) - - ( - exit_price, - exit_time, - exit_code, - tp1_exec, - tp1_price, - tp1_time, - ) = simulate_exit_numba( - dir_flag, - entry_pos, - entry_price, - sl, - tp1, - tp2, - high_arr, - low_arr, - close_arr, - time_arr, - ctx_inst.slippage_abs, - ) - - exit_result = ExitProcessor.process( - direction=direction, - entry_price=entry_price, - exit_price=exit_price, - exit_time=exit_time, - exit_code=exit_code, - tp1_executed=tp1_exec, - tp1_price=tp1_price, - tp1_time=tp1_time, - sl=sl, - tp1=tp1, - tp2=tp2, - position_size=size, - point_size=ctx_inst.point_size, - pip_value=ctx_inst.pip_value, - ) - - - - trade_dict = TradeFactory.create_trade( - symbol=symbol, - direction=direction, - entry_time=entry_time, - entry_price=entry_price, - entry_tag=entry_tag, - position_size=size, - sl=sl, - tp1=tp1, - tp2=tp2, - point_size=ctx_inst.point_size, - pip_value=ctx_inst.pip_value, - exit_result=exit_result, - level_tags=level_tags, - ) - - self.cost_engine.enrich(trade_dict, df=df, ctx=ctx_inst) - - trades.append(trade_dict) - last_exit_by_tag[entry_tag] = exit_time - - print(f"✅ Finished backtest for {symbol}, {len(trades)} trades.") - return pd.DataFrame(trades) - - # ----------------------------- - # Parallel run (legacy) - # ----------------------------- - def run(self) -> pd.DataFrame: - if getattr(self, "symbol", None) is not None: - return self._backtest_single_symbol(self.df, self.symbol) - - all_trades = [] - with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor: - futures = [] - for sym, group_df in self.df.groupby("symbol"): - futures.append(executor.submit(self._backtest_single_symbol, group_df.copy(), sym)) - - for future in as_completed(futures): - try: - trades = future.result() - all_trades.append(trades) - except Exception as e: - print(f"❌ Błąd w backteście: {e}") - traceback.print_exc() - - return pd.concat(all_trades).sort_values(by="exit_time") if all_trades else pd.DataFrame() diff --git a/core/backtesting/reporting/__init__.py b/core/backtesting/engine/__init__.py similarity index 100% rename from core/backtesting/reporting/__init__.py rename to core/backtesting/engine/__init__.py diff --git a/core/backtesting/engine/backtester.py b/core/backtesting/engine/backtester.py new file mode 100644 index 0000000..f83ee4c --- /dev/null +++ b/core/backtesting/engine/backtester.py @@ -0,0 +1,102 @@ +import pandas as pd +from typing import Optional + +from core.backtesting.engine.execution_loop import run_execution_loop +from core.domain.cost.cost_engine import TradeCostEngine +from core.backtesting.execution_policy import ExecutionPolicy +from core.domain.cost.instrument_ctx import build_instrument_ctx + + +class Backtester: + """ + Pure backtesting engine. + + Responsibilities: + - execute ONE strategy + - on ONE symbol + - over ONE dataframe slice + - return RAW trades (no equity, no analytics) + + Does NOT: + - aggregate symbols + - handle windows + - compute equity + - know about reports + """ + + def __init__( + self, + *, + execution_policy: Optional[ExecutionPolicy] = None, + cost_engine: Optional[TradeCostEngine] = None, + ): + self.execution_policy = execution_policy or ExecutionPolicy() + self.cost_engine = cost_engine or TradeCostEngine(self.execution_policy) + + # ================================================== + # MAIN API + # ================================================== + + def run( + self, + *, + signals_df: pd.DataFrame, + trade_plans: pd.DataFrame, + ) -> pd.DataFrame: + + if signals_df.empty or trade_plans.empty: + return pd.DataFrame() + + self._validate_signals(signals_df) + + symbol = signals_df["symbol"].iloc[0] + + trades = self._simulate_trades( + df=signals_df, + plans=trade_plans, + symbol=symbol, + ) + + if trades.empty: + return trades + + return trades + + # ================================================== + # INTERNAL + # ================================================== + + def _validate_signals(self, df: pd.DataFrame): + required = {"time", "symbol", "signal_entry"} + missing = required - set(df.columns) + if missing: + raise ValueError(f"Missing signal columns: {missing}") + + def _simulate_trades( + self, + *, + df: pd.DataFrame, + plans: pd.DataFrame, + symbol: str, + ) -> pd.DataFrame: + + instrument_ctx = build_instrument_ctx(symbol) + + trades = run_execution_loop( + df=df, + symbol=symbol, + plans=plans, + instrument_ctx=instrument_ctx, + ) + + if not trades: + return pd.DataFrame() + + for trade in trades: + self.cost_engine.apply( + trade_dict=trade, + df=df, + ctx=instrument_ctx, + ) + + return pd.DataFrame(trades) diff --git a/core/backtesting/engine/execution_loop.py b/core/backtesting/engine/execution_loop.py new file mode 100644 index 0000000..68ae9a3 --- /dev/null +++ b/core/backtesting/engine/execution_loop.py @@ -0,0 +1,140 @@ +import pandas as pd +from typing import Any + +from config.backtest import INITIAL_BALANCE, MAX_RISK_PER_TRADE +from core.backtesting.exit.simulate_exit_numba import simulate_exit_numba +from core.backtesting.trade_factory import TradeFactory +from core.domain.cost.instrument_ctx import InstrumentCtx +from core.domain.execution.exit_processor import ExitProcessor +from core.domain.risk.sizing import position_size + + +def run_execution_loop( + *, + df: pd.DataFrame, + symbol: str, + plans: pd.DataFrame, + instrument_ctx: InstrumentCtx, +) -> list[dict]: + + trades: list[dict] = [] + + time_arr = df["time"].dt.tz_localize(None).values + high_arr = df["high"].values + low_arr = df["low"].values + close_arr = df["close"].values + + plan_valid = plans["plan_valid"].values + plan_dir = plans["plan_direction"].values + plan_tag = plans["plan_entry_tag"].values + plan_sl = plans["plan_sl"].values.astype(float) + plan_tp1 = plans["plan_tp1"].values.astype(float) + plan_tp2 = plans["plan_tp2"].values.astype(float) + + plan_sl_tag = plans["plan_sl_tag"].values.astype(str) + plan_tp1_tag = plans["plan_tp1_tag"].values.astype(str) + plan_tp2_tag = plans["plan_tp2_tag"].values.astype(str) + + n = len(df) + + for direction in ("long", "short"): + dir_flag = 1 if direction == "long" else -1 + last_exit_by_tag: dict[str, Any] = {} + + for entry_pos in range(n): + if not plan_valid[entry_pos]: + continue + if plan_dir[entry_pos] != direction: + continue + + entry_tag = str(plan_tag[entry_pos]) + entry_time = time_arr[entry_pos] + + last_exit = last_exit_by_tag.get(entry_tag) + if last_exit is not None and last_exit > entry_time: + continue + + sl = plan_sl[entry_pos] + tp1 = plan_tp1[entry_pos] + tp2 = plan_tp2[entry_pos] + + level_tags = { + "SL": plan_sl_tag[entry_pos], + "TP1": plan_tp1_tag[entry_pos], + "TP2": plan_tp2_tag[entry_pos], + } + + entry_price = float(close_arr[entry_pos]) + entry_price += ( + instrument_ctx.slippage_abs + if direction == "long" + else -instrument_ctx.slippage_abs + ) + + size = position_size( + entry_price=entry_price, + stop_price=sl, + max_risk=MAX_RISK_PER_TRADE, + account_size=INITIAL_BALANCE, + point_size=instrument_ctx.point_size, + pip_value=instrument_ctx.pip_value, + ) + + ( + exit_price, + exit_time, + exit_code, + tp1_exec, + tp1_price, + tp1_time, + ) = simulate_exit_numba( + dir_flag, + entry_pos, + entry_price, + sl, + tp1, + tp2, + high_arr, + low_arr, + close_arr, + time_arr, + instrument_ctx.slippage_abs, + ) + + exit_result = ExitProcessor.process( + direction=direction, + entry_price=entry_price, + exit_price=exit_price, + exit_time=exit_time, + exit_code=exit_code, + tp1_executed=tp1_exec, + tp1_price=tp1_price, + tp1_time=tp1_time, + sl=sl, + tp1=tp1, + tp2=tp2, + position_size=size, + point_size=instrument_ctx.point_size, + pip_value=instrument_ctx.pip_value, + ) + + trade_dict = TradeFactory.create_trade( + symbol=symbol, + direction=direction, + entry_time=entry_time, + entry_price=entry_price, + entry_tag=entry_tag, + position_size=size, + sl=sl, + tp1=tp1, + tp2=tp2, + point_size=instrument_ctx.point_size, + pip_value=instrument_ctx.pip_value, + exit_result=exit_result, + level_tags=level_tags, + ) + + trades.append(trade_dict) + last_exit_by_tag[entry_tag] = exit_time + + return trades diff --git a/core/backtesting/engine/worker.py b/core/backtesting/engine/worker.py new file mode 100644 index 0000000..c692343 --- /dev/null +++ b/core/backtesting/engine/worker.py @@ -0,0 +1,47 @@ + +import pandas as pd + +from config.logger_config import RunLogger, LoggerConfig +from core.backtesting.engine.backtester import Backtester +from core.backtesting.strategy_runner import strategy_orchestration + + +def run_backtest_worker( + *, + signals_df: pd.DataFrame, + trade_plans: pd.DataFrame, +) -> pd.DataFrame: + """ + Run backtest for ONE strategy on ONE symbol. + Multiprocessing-safe. + """ + + backtester = Backtester() + + return backtester.run( + signals_df=signals_df, + trade_plans=trade_plans, + ) + + +def run_strategy_worker( + *, + symbol: str, + data_by_tf: dict[str, pd.DataFrame], + strategy_cls, + startup_candle_count: int, +): + logger = RunLogger( + name=f"StrategyWorker[{symbol}]", + cfg=LoggerConfig(stdout=False, file=False, timing=True), + prefix=f"📐 STRATEGY[{symbol}] |", + ) + + result = strategy_orchestration( + symbol=symbol, + data_by_tf=data_by_tf, + strategy_cls=strategy_cls, + startup_candle_count=startup_candle_count, + logger=logger, + ) + return result diff --git a/core/backtesting/execution_policy.py b/core/backtesting/execution_policy.py index ffcefe1..5c245a7 100644 --- a/core/backtesting/execution_policy.py +++ b/core/backtesting/execution_policy.py @@ -16,7 +16,12 @@ def __init__( self.exit_default_type = exit_default_type self.exit_signal_column = exit_signal_column - def classify_exit_type(self, exit_reason: str, has_exit_signal: bool = False, exit_signal_value: bool = False) -> str: + def classify_exit_type( + self, + exit_reason: str, + has_exit_signal: bool = False, + exit_signal_value: bool = False + ) -> str: if has_exit_signal and exit_signal_value: return EXEC_MARKET @@ -26,4 +31,4 @@ def classify_exit_type(self, exit_reason: str, has_exit_signal: bool = False, ex if r in ("TP2",): return EXEC_LIMIT - return self.exit_default_type \ No newline at end of file + return self.exit_default_type diff --git a/core/backtesting/reporting/config/__init__.py b/core/backtesting/exit/__init__.py similarity index 100% rename from core/backtesting/reporting/config/__init__.py rename to core/backtesting/exit/__init__.py diff --git a/core/backtesting/exit/exit_mapping.py b/core/backtesting/exit/exit_mapping.py new file mode 100644 index 0000000..bcc3eea --- /dev/null +++ b/core/backtesting/exit/exit_mapping.py @@ -0,0 +1,20 @@ +from core.domain.trade.trade_exit import TradeExitReason +from core.backtesting.exit.simulate_exit_numba import ( + EXIT_SL, + EXIT_TP1_BE, + EXIT_TP2, + EXIT_EOD, +) + + +def map_exit_code_to_reason(exit_code: int) -> TradeExitReason: + if exit_code == EXIT_SL: + return TradeExitReason.SL + if exit_code == EXIT_TP1_BE: + return TradeExitReason.BE + if exit_code == EXIT_TP2: + return TradeExitReason.TP2 + if exit_code == EXIT_EOD: + return TradeExitReason.TIMEOUT + + raise ValueError(f"Unknown exit_code: {exit_code}") diff --git a/core/backtesting/simulate_exit_numba.py b/core/backtesting/exit/simulate_exit_numba.py similarity index 93% rename from core/backtesting/simulate_exit_numba.py rename to core/backtesting/exit/simulate_exit_numba.py index 6f4c9ab..5edd109 100644 --- a/core/backtesting/simulate_exit_numba.py +++ b/core/backtesting/exit/simulate_exit_numba.py @@ -51,7 +51,6 @@ def simulate_exit_numba( sl = entry_price if direction == 1: # LONG - # TP1 (no slippage, partial logic) if (not tp1_executed) and high >= tp1_level: tp1_executed = True tp1_price = tp1_level @@ -66,20 +65,17 @@ def simulate_exit_numba( exit_price = sl - slippage_abs return exit_price, t, exit_code, tp1_executed, tp1_price, tp1_time - # TP2 HIT (NO slippage for now) if high >= tp2_level: exit_code = EXIT_TP2 exit_price = tp2_level return exit_price, t, exit_code, tp1_executed, tp1_price, tp1_time - else: # SHORT - # TP1 + else: if (not tp1_executed) and low <= tp1_level: tp1_executed = True tp1_price = tp1_level tp1_time = t - # SL HIT if high >= sl: if tp1_executed: exit_code = EXIT_TP1_BE @@ -88,7 +84,6 @@ def simulate_exit_numba( exit_price = sl + slippage_abs return exit_price, t, exit_code, tp1_executed, tp1_price, tp1_time - # TP2 HIT if low <= tp2_level: exit_code = EXIT_TP2 exit_price = tp2_level diff --git a/core/backtesting/plotting/plot.py b/core/backtesting/plotting/plot.py index 9ba6def..f0674f9 100644 --- a/core/backtesting/plotting/plot.py +++ b/core/backtesting/plotting/plot.py @@ -258,7 +258,6 @@ def connect(t0, p0, t1, p1): else: color, symbol, key = "gray", "x", "manual_exit" - # TP1 -> EXIT OR ENTRY -> EXIT if has_tp1: connect( t["tp1_time"], t["tp1_price"], @@ -290,7 +289,6 @@ def _add_zones(self): continue for zone in zones: - # OBSŁUGA OBU FORMATÓW if len(zone) == 2: zone_name, zdf = zone fillcolor = default_color @@ -372,7 +370,6 @@ def _add_bool_series(self): col=1, ) - def _layout(self): self.fig.update_layout( title=self.title, diff --git a/core/backtesting/raporter.py b/core/backtesting/raporter.py deleted file mode 100644 index d11501c..0000000 --- a/core/backtesting/raporter.py +++ /dev/null @@ -1,650 +0,0 @@ -import os -import contextlib -from io import StringIO - -import pandas as pd -from rich.console import Console -from rich.table import Table -from datetime import timedelta - -from config.backtest import BACKTEST_MODE - - -class BacktestReporter: - - def __init__( - self, - trades: pd.DataFrame, - signals: pd.DataFrame, - initial_balance: float - ): - self.console = Console() - self.trades = trades.copy() - self.signals = signals - self.initial_balance = initial_balance - - self._compute_equity_curve() - self._prepare_trades() - - # ------------------------------------------------------------------ - # PREPARE - # ------------------------------------------------------------------ - def _prepare_trades(self): - required_cols = [ - "symbol", "entry_time", "exit_time", - "entry_tag", "exit_tag", - "pnl_usd", "returns", - "duration" - ] - for c in required_cols: - if c not in self.trades.columns: - raise ValueError(f"Missing column in trades: {c}") - - self.trades["entry_tag"] = self.trades["entry_tag"].fillna("UNKNOWN") - self.trades["exit_tag"] = self.trades["exit_tag"].fillna("UNKNOWN") - - def compose_exit_tag(row): - reason = row["exit_tag"] - level = row["exit_level_tag"] - - if not isinstance(level, str) or level == "": - return reason - - return f"{reason}_{level}" - - self.trades["exit_tag_full"] = self.trades.apply( - compose_exit_tag, axis=1 - ) - - def _compute_equity_curve(self): - - self.trades = self.trades.sort_values(by="exit_time").reset_index(drop=True) - - self.trades["equity"] = self.initial_balance + self.trades["pnl_usd"].cumsum() - - self.trades["running_max"] = self.trades["equity"].cummax() - self.trades["drawdown"] = self.trades["running_max"] - self.trades["equity"] - self.max_balance = self.trades["equity"].max() - self.min_balance = self.trades["equity"].min() - self.max_drawdown = self.trades["drawdown"].max() - - def _aggregate_entry_tag(self, df: pd.DataFrame) -> dict: - trades = len(df) - if trades == 0: - return None - - equity = self.initial_balance + df["pnl_usd"].cumsum() - running_max = equity.cummax() - drawdown = (running_max - equity).max() - - wins = df[df["pnl_usd"] > 0] - losses = df[df["pnl_usd"] < 0] - - win_rate = len(wins) / trades if trades else 0 - avg_win = wins["pnl_usd"].mean() if not wins.empty else 0 - avg_loss = losses["pnl_usd"].mean() if not losses.empty else 0 - expectancy = win_rate * avg_win - (1 - win_rate) * abs(avg_loss) - - def parse_exit_tag(tag: str): - """ - Supports both legacy tags (SL_xxx, TP1_xxx_yyy) - and new domain tags (SL, TP2, BE, TIMEOUT). - """ - - if not isinstance(tag, str): - return "UNKNOWN", None, None - - parts = tag.split("_") - - if tag == "SL": - return "SL", None, "final" - - if tag == "TP2": - return "TP2", None, "final" - - if tag == "BE": - return "BE", None, "final" - - if tag == "TIMEOUT": - return "TIMEOUT", None, "final" - - if tag.startswith("SL") and len(parts) >= 2: - return "SL", parts[1], "final" - - if tag.startswith("TP1") and len(parts) >= 3: - return "TP1", parts[2], "partial" - - if tag.startswith("TP2") and len(parts) >= 3: - return "TP2", parts[2], "final" - - return "UNKNOWN", None, None - - df[["exit_event", "sl_source", "exit_stage"]] = df["exit_tag"].apply( - lambda t: pd.Series(parse_exit_tag(t)) - ) - - pct_sl = (df["exit_event"] == "SL").mean() * 100 - pct_be = (df["exit_event"] == "TP1").mean() * 100 - pct_tp2 = (df["exit_event"] == "TP2").mean() * 100 - - profit_pct = (equity.iloc[-1] / self.initial_balance - 1) * 100 - - return { - "trades": trades, - "profit_pct": profit_pct, - "pct_be": pct_be, - "pct_tp2": pct_tp2, - "pct_sl": pct_sl, - "exp": expectancy, - "drawdown": drawdown, - } - - # ------------------------------------------------------------------ - # CORE AGGREGATION LOGIC - # ------------------------------------------------------------------ - def _aggregate_trades(self, df: pd.DataFrame) -> dict: - if df.empty: - return { - "trades": 0, - "avg_profit_pct": 0, - "tot_profit_usd": 0, - "tot_profit_pct": 0, - "avg_duration": timedelta(0), - "win": 0, - "draw": 0, - "loss": 0, - "win_pct": 0, - "avg_winner": 0, - "avg_losser": 0, - "exp": 0, - } - - wins = df[df["pnl_usd"] > 0] - losses = df[df["pnl_usd"] < 0] - draws = df[df["pnl_usd"] == 0] - - effective_trades = len(wins) + len(losses) - win_rate = (len(wins) / effective_trades) if effective_trades > 0 else 0 - - avg_win = wins["pnl_usd"].mean() if not wins.empty else 0 - avg_loss = losses["pnl_usd"].mean() if not losses.empty else 0 - - expectancy = win_rate * avg_win - (1 - win_rate) * abs(avg_loss) - - return { - "trades": len(df), - "avg_profit_pct": df["returns"].mean() * 100, - "tot_profit_usd": df["pnl_usd"].sum(), - "tot_profit_pct": df["returns"].sum() * 100, - "avg_duration": df["duration"].mean(), - "win": len(wins), - "draw": len(draws), - "loss": len(losses), - "win_pct": win_rate * 100, - "avg_winner": avg_win, - "avg_losser": avg_loss, - "exp": expectancy, - } - - # ------------------------------------------------------------------ - # GENERIC GROUP TABLE - # ------------------------------------------------------------------ - def _print_group_table(self, title: str, group_col: str, df: pd.DataFrame): - self.console.rule(f"[bold yellow]{title}[/bold yellow]") - - total_df = [] - stats_list = [] - - # grupowanie i agregacja - for name, g in df.groupby(group_col): - stats = self._aggregate_trades(g) - stats["name"] = name - stats_list.append(stats) - total_df.append(g) - - if not stats_list: - print("⚠️ Brak danych do wyświetlenia.") - return - - # sortowanie po tot_profit_usd malejąco - stats_list = sorted(stats_list, key=lambda x: x["tot_profit_usd"], reverse=True) - - table = Table(show_header=True, header_style="bold magenta") - table.add_column(str(group_col)) - table.add_column("Trades", justify="right") - table.add_column("Tot Profit USD", justify="right") - table.add_column("Tot Profit %", justify="right") - table.add_column("Avg Duration", justify="center") - table.add_column("Win", justify="right") - table.add_column("Draw", justify="right") - table.add_column("Loss", justify="right") - table.add_column("Win %", justify="right") - table.add_column("Avg Winner", justify="right") - table.add_column("Avg Losser", justify="right") - table.add_column("Exp", justify="right") - - for stats in stats_list: - avg_duration_str = str(pd.to_timedelta(stats["avg_duration"], unit='s')).split('.')[0] - - table.add_row( - str(stats["name"]), - f"{stats['trades']}", - f"{stats['tot_profit_usd']:.3f}", - f"{stats['tot_profit_pct']:.2f}", - avg_duration_str, - f"{stats['win']}", - f"{stats['draw']}", - f"{stats['loss']}", - f"{stats['win_pct']:.1f}", - f"{stats['avg_winner']:.2f}", - f"{stats['avg_losser']:.2f}", - f"{stats['exp']:.2f}" - ) - - self.console.print(table) - - def _print_summary_metrics(self): - t = self.trades.sort_values("exit_time") - - start = t["entry_time"].min() - end = t["exit_time"].max() - - total_trades = len(t) - days = max((end - start).days, 1) - trades_per_day = total_trades / days - - final_balance = t["equity"].iloc[-1] - absolute_profit = final_balance - self.initial_balance - total_profit_pct = (final_balance / self.initial_balance - 1) * 100 - - print( - "CAGR DEBUG |", - "initial_balance =", self.initial_balance, - "final_balance =", final_balance, - "days =", days, - ) - - if final_balance <= 0: - cagr = -100.0 - else: - cagr = ((final_balance / self.initial_balance) ** (365 / days) - 1) * 100 - - - - daily_returns = t.groupby(t["exit_time"].dt.date)["pnl_usd"].sum() - max_daily_loss = daily_returns.min() - max_daily_loss_pct = max_daily_loss / self.initial_balance * 100 - - equity = t["equity"] - max_balance = equity.cummax() - drawdown = max_balance - equity - max_dd = drawdown.max() - max_dd_pct = (drawdown / max_balance).max() * 100 - - wins = t[t["pnl_usd"] > 0] - losses = t[t["pnl_usd"] < 0] - - profit_factor = ( - wins["pnl_usd"].sum() / abs(losses["pnl_usd"].sum()) - if not losses.empty else float("inf")) - - expectancy = self._aggregate_trades(t)["exp"] - - table = Table(title="SUMMARY METRICS", show_header=False) - table.add_column("Metric") - table.add_column("Value", justify="right") - - rows = [ - ("Backtesting from", str(start)), - ("Backtesting to", str(end)), - ("Total/Daily Avg Trades", f"{total_trades} / {trades_per_day:.1f}"), - ("Starting balance", f"{self.initial_balance:.2f} USD"), - ("Final balance", f"{final_balance:.2f} USD"), - ("Absolute profit", f"{absolute_profit:.2f} USD"), - ("Total profit %", f"{total_profit_pct:.2f}%"), - ("CAGR %", f"{cagr:.2f}%"), - ("Profit factor", f"{profit_factor:.2f}"), - ("Expectancy", f"{expectancy:.2f}"), - ("Max balance", f"{max_balance.max():.2f} USD"), - ("Min balance", f"{equity.min():.2f} USD"), - ("Absolute Drawdown", f"{max_dd:.2f} USD"), - ("Max % underwater", f"{max_dd_pct:.2f}%"), - ("Max daily loss $", f"{max_daily_loss:.2f} USD"), - ("Max daily loss %", f"{max_daily_loss_pct:.2f}%"), - ] - - for r in rows: - table.add_row(*r) - - self.console.print(table) - - # ------------------------------------------------------------------ - # PUBLIC REPORTS - # ------------------------------------------------------------------ - def print_entry_tag_stats(self): - self.console.rule("[bold yellow]ENTER TAG STATS[/bold yellow]") - - rows = [] - - for tag, g in self.trades.groupby("entry_tag"): - stats = self._aggregate_entry_tag(g) - if stats is None: - continue - - rows.append({ - "entry_tag": tag, - **stats - }) - - if not rows: - self.console.print("⚠️ No entry tag data.") - return - - # 🔑 SORTOWANIE PO TOTAL PROFIT % - rows = sorted(rows, key=lambda x: x["profit_pct"], reverse=True) - - table = Table(show_header=True, header_style="bold magenta") - table.add_column("Entry Tag") - table.add_column("Trades", justify="right") - table.add_column("TOT Profit %", justify="right") - table.add_column("%BE", justify="right") - table.add_column("%TP2", justify="right") - table.add_column("%SL", justify="right") - table.add_column("Exp $", justify="right") - table.add_column("Max DD $", justify="right") - - for r in rows: - table.add_row( - str(r["entry_tag"]), - f"{r['trades']}", - f"{r['profit_pct']:.2f}", - f"{r['pct_be']:.1f}", - f"{r['pct_tp2']:.1f}", - f"{r['pct_sl']:.1f}", - f"{r['exp']:.2f}", - f"{r['drawdown']:.2f}", - ) - - self.console.print(table) - - def print_exit_reason_stats(self): - self._print_group_table("EXIT REASON STATS", "exit_tag_full", self.trades) - - def print_tp1_entry_stats(self): - df = self.trades[self.trades['tp1_price'].notna()] - self._print_group_table( - "ENTER TAG STATS for trades that HIT TP1", - "entry_tag", - df - ) - - def print_tp1_exit_stats(self): - df = self.trades[self.trades['tp1_price'].notna()] - self._print_group_table( - "EXIT STATS for trades that HIT TP1", - "exit_tag", - df - ) - - def print_symbol_report(self): - self._print_group_table( - "BACKTESTING REPORT", - "symbol", - self.trades - ) - - def print_entry_tag_split_table(self, mode: str = "filtered"): - """ - mode: - - 'filtered' : tylko stabilne / decyzyjne tagi (default) - - 'all' : wszystkie tagi (diagnostyka) - """ - - if "window" not in self.trades.columns: - self.console.print( - "⚠️ Split report requires 'window' column in trades.") - return - - self.console.rule( - "[bold yellow]ENTRY TAG — STABILITY FILTERED[/bold yellow]" - if mode == "filtered" - else "[bold yellow]ENTRY TAG — FULL DIAGNOSTIC VIEW[/bold yellow]" - ) - - # --- aggregate --- - grouped = self.trades.groupby(["entry_tag", "window"]) - data = {} - - for (tag, window), g in grouped: - stats = self._aggregate_entry_tag(g) - if stats is None: - continue - data.setdefault(tag, {})[window] = stats - - rows = [] - - # --- scoring --- - def score_window(opt, val, fin): - score = 0.0 - - # trades (data reliability) - if opt and opt["trades"] >= 50: - score += 1 - if val and val["trades"] >= 30: - score += 1 - if fin and fin["trades"] >= 30: - score += 1 - - # EV persistence - if opt and opt["exp"] > 0: - score += 2 - if val and val["exp"] > 0: - score += 2 - if fin and fin["exp"] > 0: - score += 1 - - return score - - # --- collect rows --- - for tag, windows in data.items(): - opt = windows.get("OPT") - val = windows.get("VAL") - fin = windows.get("FINAL") - - if not opt: - continue - - score = score_window(opt, val, fin) - - if mode == "filtered": - if opt["trades"] < 50: - continue - if val and val["trades"] < 30: - continue - if opt["exp"] <= 0: - continue - if score <= 4: - continue - - rows.append((tag, score, opt, val, fin)) - - if not rows: - self.console.print("⚠️ No entry tags to display.") - return - - # --- sorting --- - if mode == "filtered": - rows.sort(key=lambda r: r[1], reverse=True) - else: - rows.sort( - key=lambda r: ( - -(r[4]["exp"] if r[4] else -1e9), # FINAL EV first - -r[1] # then stability score - ) - ) - - # --- table --- - table = Table(show_header=True, header_style="bold magenta") - - table.add_column("Entry Tag", style="bold") - table.add_column("Score", justify="right") - table.add_column("Trades OPT", justify="right") - table.add_column("Trades VAL", justify="right") - table.add_column("Trades FINAL", justify="right") - - metrics = [ - ("%TP2", "pct_tp2", True), - ("%SL", "pct_sl", True), - ("Exp $", "exp", False), - ] - - for name, _, _ in metrics: - table.add_column(f"{name} OPT", justify="right") - table.add_column(f"{name} VAL", justify="right") - table.add_column(f"{name} FINAL", justify="right") - - def fmt(w, key, pct=False): - if not w: - return "—" - v = w[key] - return f"{v:.1f}" if pct else f"{v:.2f}" - - # --- render rows --- - for tag, score, opt, val, fin in rows: - row = [ - tag, - f"{score:.1f}", - str(opt["trades"]), - str(val["trades"]) if val else "—", - str(fin["trades"]) if fin else "—", - ] - - for _, key, pct in metrics: - row.extend([ - fmt(opt, key, pct), - fmt(val, key, pct), - fmt(fin, key, pct), - ]) - - table.add_row(*row) - - self.console.print(table) - - def print_entry_tag_split_report(self): - """ - ENTRY TAG performance per backtest window (OPT / VAL / FINAL) - """ - - if "window" not in self.trades.columns: - self.console.print("⚠️ Split report requires 'window' column in trades.") - return - - self.console.rule("[bold yellow]ENTRY TAG SPLIT PERFORMANCE[/bold yellow]") - - rows = [] - - grouped = self.trades.groupby(["entry_tag", "window"]) - - # { entry_tag: { window: stats } } - data = {} - - for (tag, window), g in grouped: - stats = self._aggregate_entry_tag(g) - if stats is None: - continue - - data.setdefault(tag, {})[window] = stats - - for tag, windows in data.items(): - opt = windows.get("OPT") - val = windows.get("VAL") - fin = windows.get("FINAL") - - if not opt: - continue # bez OPT tag nie istnieje - - row = { - "entry_tag": tag, - "opt": opt["profit_pct"], - "val": val["profit_pct"] if val else None, - "final": fin["profit_pct"] if fin else None, - "delta_opt_val": ( - (val["profit_pct"] - opt["profit_pct"]) if val else None - ), - "delta_val_final": ( - (fin["profit_pct"] - val["profit_pct"]) if fin and val else None - ), - "trades": opt["trades"], - } - - rows.append(row) - - if not rows: - self.console.print("⚠️ No split entry tag data.") - return - - # sort: najlepsze OPT na górze - rows = sorted(rows, key=lambda x: x["opt"], reverse=True) - - table = Table(show_header=True, header_style="bold magenta") - table.add_column("Entry Tag") - table.add_column("OPT %", justify="right") - table.add_column("VAL %", justify="right") - table.add_column("FINAL %", justify="right") - table.add_column("Δ OPT→VAL", justify="right") - table.add_column("Δ VAL→FINAL", justify="right") - table.add_column("Trades", justify="right") - - for r in rows: - table.add_row( - r["entry_tag"], - f"{r['opt']:.2f}", - f"{r['val']:.2f}" if r["val"] is not None else "—", - f"{r['final']:.2f}" if r["final"] is not None else "—", - f"{r['delta_opt_val']:.2f}" if r["delta_opt_val"] is not None else "—", - f"{r['delta_val_final']:.2f}" if r["delta_val_final"] is not None else "—", - f"{r['trades']}", - ) - - self.console.print(table) - - # ------------------------------------------------------------------ - # RUN ALL REPORTS - # ------------------------------------------------------------------ - def run(self): - - if BACKTEST_MODE == "single": - self.console.rule("[bold cyan]SUMMARY METRICS[/bold cyan]") - - self._print_summary_metrics() - - #self.console.rule("[bold cyan]DETAILED REPORTS[/bold cyan]") - - self.print_entry_tag_stats() - self.print_exit_reason_stats() - - # self.print_tp1_entry_stats() - # self.print_tp1_exit_stats() - self.print_symbol_report() - elif BACKTEST_MODE == "split": - self.console.rule("[bold cyan]SUMMARY METRICS[/bold cyan]") - - self._print_summary_metrics() - - if "window" in self.trades.columns: - self.print_entry_tag_split_table(mode="all") - - self.print_exit_reason_stats() - self.print_symbol_report() - - def save(self, filename: str): - os.makedirs(os.path.dirname(filename), exist_ok=True) - - buffer = StringIO() - with contextlib.redirect_stdout(buffer): - self.run() - - with open(filename, "w", encoding="utf-8") as f: - f.write(buffer.getvalue()) - - print(f"✅ Raport zapisany do pliku: {filename}") diff --git a/core/backtesting/reporting/config/report_config.py b/core/backtesting/reporting/config/report_config.py deleted file mode 100644 index c9be752..0000000 --- a/core/backtesting/reporting/config/report_config.py +++ /dev/null @@ -1,19 +0,0 @@ -from dataclasses import dataclass, field -from typing import List - -from core.backtesting.reporting.core.base import BaseMetric -from core.backtesting.reporting.core.context import ContextSpec - - -@dataclass -class ReportConfig: - metrics: List[BaseMetric] = field(default_factory=list) - contexts: List[ContextSpec] = field(default_factory=list) - - def add_metric(self, metric: BaseMetric): - self.metrics.append(metric) - return self - - def add_context(self, context: ContextSpec): - self.contexts.append(context) - return self diff --git a/core/backtesting/reporting/core/context.py b/core/backtesting/reporting/core/context.py deleted file mode 100644 index 0d075bb..0000000 --- a/core/backtesting/reporting/core/context.py +++ /dev/null @@ -1,23 +0,0 @@ -from dataclasses import dataclass -from typing import Optional, Set, Any - -import pandas as pd - - -@dataclass(frozen=True) -class ContextSpec: - name: str - column: str - source: str - allowed_values: Optional[Set] = None - - -@dataclass -class ReportContext: - trades: pd.DataFrame - equity: pd.Series | None - drawdown: pd.Series | None - df_plot: pd.DataFrame - initial_balance: float - config: Any - strategy: Any diff --git a/core/backtesting/reporting/core/persistence.py b/core/backtesting/reporting/core/persistence.py deleted file mode 100644 index ba48a24..0000000 --- a/core/backtesting/reporting/core/persistence.py +++ /dev/null @@ -1,55 +0,0 @@ -from pathlib import Path -from datetime import datetime -import json -import pandas as pd - - -class ReportPersistence: - """ - Persist report outputs for dashboards / post-analysis. - """ - - def __init__(self, base_dir: Path = Path("results/reports")): - self.base_dir = base_dir - - def persist( - self, - *, - trades: pd.DataFrame, - equity: pd.Series, - report_data: dict, - meta: dict | None = None, - ) -> Path: - - ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S") - run_dir = self.base_dir / f"run_{ts}" - run_dir.mkdir(parents=True, exist_ok=True) - - # ---------------------------- - # trades snapshot - # ---------------------------- - trades.to_parquet(run_dir / "trades.parquet", index=False) - - # ---------------------------- - # equity snapshot - # ---------------------------- - equity.to_frame(name="equity").to_parquet( - run_dir / "equity.parquet" - ) - - # ---------------------------- - # report (JSON) - # ---------------------------- - with open(run_dir / "report.json", "w") as f: - json.dump(report_data, f, indent=2, default=str) - - # ---------------------------- - # meta - # ---------------------------- - meta_payload = meta or {} - meta_payload["timestamp_utc"] = ts - - with open(run_dir / "meta.json", "w") as f: - json.dump(meta_payload, f, indent=2) - - return run_dir diff --git a/core/backtesting/reporting/core/preparer.py b/core/backtesting/reporting/core/preparer.py deleted file mode 100644 index 96c5049..0000000 --- a/core/backtesting/reporting/core/preparer.py +++ /dev/null @@ -1,24 +0,0 @@ -import pandas as pd - - -class RiskDataPreparer: - """ - Prepares trades DataFrame for risk metrics. - Adds equity curve and drawdown-related columns. - """ - - def __init__(self, initial_balance: float): - self.initial_balance = initial_balance - - def prepare(self, trades: pd.DataFrame) -> pd.DataFrame: - if trades.empty: - return trades - - df = trades.sort_values("exit_time").copy() - - df["equity"] = self.initial_balance + df["pnl_net_usd"].cumsum() - - df["equity_peak"] = df["equity"].cummax() - df["drawdown"] = df["equity_peak"] - df["equity"] - - return df diff --git a/core/backtesting/reporting/runner.py b/core/backtesting/reporting/runner.py deleted file mode 100644 index afaf736..0000000 --- a/core/backtesting/reporting/runner.py +++ /dev/null @@ -1,100 +0,0 @@ -from config.backtest import INITIAL_BALANCE -from core.backtesting.reporting.core.context import ReportContext -from core.backtesting.reporting.core.equity import EquityPreparer -from core.backtesting.reporting.core.formating import materialize -from core.backtesting.reporting.core.persistence import ReportPersistence -from core.backtesting.reporting.core.sections.backtest_config import BacktestConfigSection -from core.backtesting.reporting.core.sections.capital_exposure import CapitalExposureSection -from core.backtesting.reporting.core.sections.conditional_entry_tag import ConditionalEntryTagPerformanceSection -from core.backtesting.reporting.core.sections.conditional_expectancy import ConditionalExpectancySection -from core.backtesting.reporting.core.sections.kpi import CorePerformanceSection -from core.backtesting.reporting.core.sections.drawdown_structure import DrawdownStructureSection -from core.backtesting.reporting.core.sections.entry_tag_performance import EntryTagPerformanceSection -from core.backtesting.reporting.core.sections.exit_logic_diagnostics import ExitLogicDiagnosticsSection -from core.backtesting.reporting.core.sections.tail_risk import TailRiskSection -from core.backtesting.reporting.core.sections.trade_distribution import TradeDistributionSection -from core.backtesting.reporting.renders.dashboard.dashboard_renderer import DashboardRenderer -from core.backtesting.reporting.renders.stdout import StdoutRenderer -from core.backtesting.reporting.reports.risk import RiskReport - - -class ReportRunner: - """ - Orchestrates report execution. - Prepares ReportContext and delegates computation to RiskReport. - """ - - def __init__(self, strategy, trades_df, config, renderer=None): - self.strategy = strategy - self.trades_df = trades_df - self.config = config - self.renderer = renderer or StdoutRenderer() - - def run(self): - # ================================================== - # PREPARE EQUITY & DRAWDOWN - # ================================================== - - equity_preparer = EquityPreparer( - initial_balance=self.config.INITIAL_BALANCE - ) - - trades_with_equity = equity_preparer.prepare(self.trades_df) - - equity = trades_with_equity["equity"] - drawdown = trades_with_equity["drawdown"] - - # ================================================== - # BUILD REPORT CONTEXT - # ================================================== - - ctx = ReportContext( - trades=trades_with_equity, - equity=equity, - drawdown=drawdown, - df_plot=self.strategy.df_plot, - initial_balance=INITIAL_BALANCE, - config=self.config, - strategy=self.strategy, - ) - - - # ================================================== - # BUILD REPORT (SECTIONS) - # ================================================== - - report = RiskReport( - sections=[ - BacktestConfigSection(), - CorePerformanceSection(), - TradeDistributionSection(), - TailRiskSection(), - ConditionalExpectancySection(), - EntryTagPerformanceSection(), - ConditionalEntryTagPerformanceSection(), - ExitLogicDiagnosticsSection(), - DrawdownStructureSection(), - CapitalExposureSection(), - - ] - ) - - data = report.compute(ctx) - data = materialize(data) - - self.renderer.render(data) - - # ========================== - # PERSIST FOR DASHBOARD - # ========================== - - ReportPersistence().persist( - trades=ctx.trades, - equity=ctx.equity, - report_data=data, - meta={}, - ) - - DashboardRenderer().render(data, ctx) - - print("\n✅ Dashboard built successfully\n") diff --git a/core/backtesting/reporting/core/__init__.py b/core/backtesting/results_logic/__init__.py similarity index 100% rename from core/backtesting/reporting/core/__init__.py rename to core/backtesting/results_logic/__init__.py diff --git a/core/backtesting/results_logic/metadata.py b/core/backtesting/results_logic/metadata.py new file mode 100644 index 0000000..ab4329d --- /dev/null +++ b/core/backtesting/results_logic/metadata.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass, asdict +from datetime import datetime +from typing import Any + + +@dataclass +class BacktestMetadata: + run_id: str + created_at: str + + backtest_mode: str + windows: dict[str, tuple[str, str]] | None + + strategies: list[str] + strategy_names: dict[str, str] + + symbols: list[str] + timeframe: str + + initial_balance: float + slippage: float + max_risk_per_trade: float + + notes: str | None = None + + @staticmethod + def now(run_id: str, **kwargs) -> "BacktestMetadata": + return BacktestMetadata( + run_id=run_id, + created_at=datetime.utcnow().isoformat(), + **kwargs, + ) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) diff --git a/core/backtesting/results_logic/result.py b/core/backtesting/results_logic/result.py new file mode 100644 index 0000000..4a49221 --- /dev/null +++ b/core/backtesting/results_logic/result.py @@ -0,0 +1,80 @@ +from pathlib import Path +import json +import pandas as pd +from core.backtesting.results_logic.metadata import BacktestMetadata + + +class BacktestResult: + """ + Immutable snapshot of backtest output. + """ + + def __init__( + self, + *, + metadata: BacktestMetadata, + trades: pd.DataFrame, + analytics: pd.DataFrame | None = None, + ): + self.metadata = metadata + self.trades = trades + self.analytics = analytics + + # ================================================== + # SAVE / LOAD + # ================================================== + + def save(self, path: str | Path): + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + + with open(path / "metadata.json", "w", encoding="utf-8") as f: + json.dump(self.metadata.to_dict(), f, indent=2) + + self.trades.to_parquet(path / "trades.parquet", index=False) + + if self.analytics is not None: + self.analytics.to_parquet(path / "analytics.parquet", index=False) + + self._write_readme(path) + + @staticmethod + def load(path: str | Path) -> "BacktestResult": + path = Path(path) + + with open(path / "metadata.json", encoding="utf-8") as f: + meta = BacktestMetadata(**json.load(f)) + + trades = pd.read_parquet(path / "trades.parquet") + + analytics = ( + pd.read_parquet(path / "analytics.parquet") + if (path / "analytics.parquet").exists() + else None + ) + + return BacktestResult( + metadata=meta, + trades=trades, + analytics=analytics, + ) + + # ================================================== + # README + # ================================================== + + def _write_readme(self, path: Path): + m = self.metadata + txt = f"""# Backtest Result + +Run ID: {m.run_id} +Created: {m.created_at} + +Strategies: +{chr(10).join(f"- {sid}: {m.strategy_names[sid]}" for sid in m.strategies)} + +Symbols: {", ".join(m.symbols)} +Timeframe: {m.timeframe} +Backtest mode: {m.backtest_mode} +""" + (path / "README.md").write_text(txt, encoding="utf-8") diff --git a/core/backtesting/results_logic/store.py b/core/backtesting/results_logic/store.py new file mode 100644 index 0000000..ecdb68e --- /dev/null +++ b/core/backtesting/results_logic/store.py @@ -0,0 +1,33 @@ +from pathlib import Path +from core.backtesting.results_logic.result import BacktestResult + + +class ResultStore: + """ + File-based registry for BacktestResult. + Single source of truth for run paths. + """ + + def __init__(self, base_path: str | Path = "results/backtests"): + self.base_path = Path(base_path) + self.base_path.mkdir(parents=True, exist_ok=True) + + def run_path(self, run_id: str) -> Path: + return self.base_path / run_id + + def save(self, result: BacktestResult) -> Path: + path = self.run_path(result.metadata.run_id) + result.save(path) + return path + + def load(self, run_id: str) -> BacktestResult: + path = self.run_path(run_id) + if not path.exists(): + raise FileNotFoundError(run_id) + return BacktestResult.load(path) + + def list_runs(self) -> list[str]: + return sorted( + p.name for p in self.base_path.iterdir() + if p.is_dir() + ) diff --git a/core/backtesting/runner.py b/core/backtesting/runner.py index 7bba406..782d368 100644 --- a/core/backtesting/runner.py +++ b/core/backtesting/runner.py @@ -1,303 +1,369 @@ import os -from time import perf_counter from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime +from pathlib import Path +from uuid import uuid4 import pandas as pd -from config.backtest import INITIAL_BALANCE -from core.backtesting.reporting.core.contex_enricher import TradeContextEnricher -from core.backtesting.reporting.core.preparer import RiskDataPreparer -from core.backtesting.reporting.runner import ReportRunner -from core.data_provider import CsvMarketDataCache +from config.logger_config import RunLogger, profiling +from config.report_config import ReportConfig, StdoutMode from core.backtesting.backend_factory import create_backtest_backend -from core.data_provider.providers.default_provider import DefaultOhlcvDataProvider - -from core.backtesting.backtester import Backtester -from core.backtesting.plotting.plot import TradePlotter - -from core.backtesting.strategy_runner import run_strategy_single +from core.backtesting.engine.backtester import Backtester +from core.backtesting.engine.worker import run_backtest_worker, run_strategy_worker +from core.backtesting.results_logic.metadata import BacktestMetadata +from core.backtesting.results_logic.result import BacktestResult +from core.backtesting.results_logic.store import ResultStore +from core.backtesting.strategy_runner import strategy_orchestration +from core.data_provider import DefaultOhlcvDataProvider, CsvMarketDataCache from core.live_trading.strategy_loader import load_strategy_class +from core.reporting.runner import ReportRunner +from core.reporting.summary_runner import SummaryReportRunner class BacktestRunner: """ - Orchestrates: - - data loading - - strategy execution (possibly multi-symbol) - - backtesting - - reporting - - plotting + Application-layer orchestrator. + + Responsibilities: + - load market data + - execute strategies + - run backtests + - build BacktestResult + - trigger reporting """ def __init__(self, cfg): - self.config = cfg + self.cfg = cfg + + self.provider = None + + self.strategy = None + self.strategies = [] - # 🔑 STRATEGY CONTRACT - self.strategy = None # reference strategy (for reporting config) - self.strategies = [] # all strategy instances (per symbol) + self.signals_df: pd.DataFrame | None = None + self.trades_df: pd.DataFrame | None = None - self.signals_df = None - self.trades_df = None + self.log_run = RunLogger( + "Run", + self.cfg.LOGGER_CONFIG, + prefix="🚀 RUN |" + ) + self.log_data = RunLogger( + "Data", + self.cfg.LOGGER_CONFIG, + prefix="📈 DATA |" + ) + self.log_strategy = RunLogger( + "Strategy", + self.cfg.LOGGER_CONFIG, + prefix="📐 STRATEGY |" + ) + self.log_backtest = RunLogger( + "Backtest", + self.cfg.LOGGER_CONFIG, + prefix="📊 BACKTEST |" + ) + self.log_report = RunLogger( + "Report", + self.cfg.LOGGER_CONFIG, + prefix="📄 REPORT |" + ) # ================================================== - # 1️⃣ LOAD DATA ONCE + # 1️⃣ DATA LOADING # ================================================== - def load_data(self): + def load_data(self) -> dict[str, dict[str, pd.DataFrame]]: + self.log_data.log("start") - t_start = perf_counter() - print("⏱️ load_data | start") + strategy_cls = load_strategy_class(self.cfg.STRATEGY_CLASS) + informative_tfs = strategy_cls.get_required_informatives() + base_tf = self.cfg.TIMEFRAME + all_tfs = [base_tf] + informative_tfs - backend = create_backtest_backend(self.config.BACKTEST_DATA_BACKEND) + backend = create_backtest_backend(self.cfg.BACKTEST_DATA_BACKEND) - start = pd.Timestamp(self.config.TIMERANGE["start"], tz="UTC") - end = pd.Timestamp(self.config.TIMERANGE["end"], tz="UTC") + start = pd.Timestamp(self.cfg.TIMERANGE["start"], tz="UTC") + end = pd.Timestamp(self.cfg.TIMERANGE["end"], tz="UTC") self.provider = DefaultOhlcvDataProvider( backend=backend, - cache=CsvMarketDataCache(self.config.MARKET_DATA_PATH), + cache=CsvMarketDataCache(self.cfg.MARKET_DATA_PATH), backtest_start=start, backtest_end=end, + logger=self.log_data, ) all_data = {} - for symbol in self.config.SYMBOLS: - t_sym = perf_counter() - - df = self.provider.get_ohlcv( - symbol=symbol, - timeframe=self.config.TIMEFRAME, - start=start, - end=end, - ) - - print( - f"⏱️ load_data | get_ohlcv {symbol:<10} " - f"{perf_counter() - t_sym:8.3f}s ({len(df)} rows)" - ) - - all_data[symbol] = df + with self.log_data.time("load_all"): + for symbol in self.cfg.SYMBOLS: + per_symbol = {} + for tf in all_tfs: + per_symbol[tf] = self.provider.get_ohlcv( + symbol=symbol, + timeframe=tf, + start=start, + end=end, + ) + all_data[symbol] = per_symbol - print(f"⏱️ load_data | TOTAL {perf_counter() - t_start:8.3f}s") + self.log_data.log( + f"summary | symbols={len(all_data)} timeframes={len(all_tfs)}" + ) return all_data - # ================================================== - # 2️⃣ RUN STRATEGIES (PARALLEL) + # 2️⃣ STRATEGY EXECUTION # ================================================== - def run_strategies_parallel(self, all_data: dict): + def run_strategies(self, all_data): + if self.cfg.USE_MULTIPROCESSING_STRATEGIES: + return self.run_strategies_parallel(all_data) + else: + return self.run_strategies_single(all_data) + + def run_strategies_single(self, all_data) -> pd.DataFrame: + self.log_strategy.log(f"start | symbols={len(all_data)}") + + strategy_cls = load_strategy_class(self.cfg.STRATEGY_CLASS) + self.strategy_runs = [] + + with self.log_strategy.time("execution"): + for symbol, data_by_tf in all_data.items(): + with self.log_strategy.time(f"{symbol}"): + result = strategy_orchestration( + symbol=symbol, + data_by_tf=data_by_tf, + strategy_cls=strategy_cls, + startup_candle_count=self.cfg.STARTUP_CANDLE_COUNT, + logger=self.log_strategy, + ) - t_start = perf_counter() - print(f"📈 STRATEGIES | start ({len(all_data)} symbols)") + if hasattr(result, "timing"): + t = result.timing + self.log_strategy.log( + f"{symbol:<8} | " + f"exec={t.get('execute_strategy', 0):.2f}s " + f"ind={t.get('execute.indicators', 0):.2f}s " + f"entry={t.get('execute.entry', 0):.2f}s " + f"exit={t.get('execute.exit', 0):.2f}s" + ) + + self.strategy_runs.append(result) + + with self.log_strategy.time("aggregate"): + self.signals_df = ( + pd.concat([r.df_signals for r in self.strategy_runs]) + .sort_values(["time", "symbol"]) + .reset_index(drop=True) + ) - all_signals = [] - self.strategies = [] - self.strategy = None + self.log_strategy.log("finished") + return self.signals_df - # ================================================= - # 🔥 SINGLE SYMBOL - # ================================================= - if len(all_data) == 1: - symbol, df = next(iter(all_data.items())) - - df_signals, strategy = run_strategy_single( - symbol, - df, - self.provider, - load_strategy_class(self.config.STRATEGY_CLASS), - self.config.STARTUP_CANDLE_COUNT, - ) + def run_strategies_parallel(self, all_data) -> pd.DataFrame: + self.log_strategy.log(f"start parallel | symbols={len(all_data)}") - all_signals.append(df_signals) - self.strategies.append(strategy) - self.strategy = strategy + strategy_cls = load_strategy_class(self.cfg.STRATEGY_CLASS) + self.strategy_runs = [] - # ================================================= - # 🚀 MULTI SYMBOL - # ================================================= - else: - with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor: + with self.log_strategy.time("parallel_execution"): + with ProcessPoolExecutor() as executor: futures = [ executor.submit( - run_strategy_single, - symbol, - df, - self.provider, - load_strategy_class(self.config.STRATEGY_CLASS), - self.config.STARTUP_CANDLE_COUNT, + run_strategy_worker, + symbol=symbol, + data_by_tf=data_by_tf, + strategy_cls=strategy_cls, + startup_candle_count=self.cfg.STARTUP_CANDLE_COUNT, ) - for symbol, df in all_data.items() + for symbol, data_by_tf in all_data.items() ] + for f in as_completed(futures): + self.strategy_runs.append(f.result()) - for future in as_completed(futures): - df_signals, strategy = future.result() - - all_signals.append(df_signals) - self.strategies.append(strategy) - - # 🔑 set reference strategy ONCE - if self.strategy is None: - self.strategy = strategy - - if not all_signals: - raise RuntimeError("No signals generated by strategies") - - self.signals_df = ( - pd.concat(all_signals) - .sort_values(by=["time", "symbol"]) - .reset_index(drop=True) - ) + with self.log_strategy.time("aggregate"): + self.signals_df = ( + pd.concat([r.df_signals for r in self.strategy_runs]) + .sort_values(["time", "symbol"]) + .reset_index(drop=True) + ) - print(f"📈 STRATEGIES | TOTAL {perf_counter() - t_start:8.3f}s") + self.log_strategy.log("finished parallel") return self.signals_df # ================================================== - # 3️⃣ BACKTEST WINDOWS + # 3️⃣ BACKTEST # ================================================== - def _run_backtest_window(self, start, end, label): + def run_backtests(self) -> pd.DataFrame: + if self.cfg.USE_MULTIPROCESSING_BACKTESTS: + return self.run_backtests_parallel() + else: + return self.run_backtests_single() + + def run_backtests_single(self) -> pd.DataFrame: + self.log_backtest.log("start") + + self.trades_by_run = [] - df_slice = self.signals_df[ - (self.signals_df["time"] >= start) & - (self.signals_df["time"] <= end) - ].copy() + with self.log_backtest.time("execution"): + for run in self.strategy_runs: + backtester = Backtester() + trades = backtester.run( + signals_df=run.df_signals, + trade_plans=run.trade_plans, + ) + self.trades_by_run.append(trades) + self.log_backtest.log( + f"{run.symbol:<8} | trades={len(trades)}" + ) + + self.trades_df = ( + pd.concat(self.trades_by_run) + .sort_values("exit_time") + .reset_index(drop=True) + ) - if df_slice.empty: - raise RuntimeError(f"No signals in window: {label}") + self.log_backtest.log( + f"summary | total_trades={len(self.trades_df)}" + ) + return self.trades_df - backtester = Backtester(strategy=self.strategy) - trades = backtester.run_backtest(df_slice) - trades["window"] = label - return trades + def run_backtests_parallel(self) -> pd.DataFrame: + if not self.strategy_runs: + raise RuntimeError("No strategy runs to backtest") - # ================================================== - # 4️⃣ RUN BACKTEST(S) - # ================================================== + self.trades_by_run = [] + max_workers = self.cfg.MAX_WORKERS_BACKTESTS + workers = max_workers or os.cpu_count() - def run_backtests(self): + self.log_backtest.log( + f"start parallel | runs={len(self.strategy_runs)} workers={workers}" + ) - if self.config.BACKTEST_MODE == "single": - start = pd.Timestamp(self.config.TIMERANGE["start"], tz="UTC") - end = pd.Timestamp(self.config.TIMERANGE["end"], tz="UTC") + with self.log_backtest.time("parallel_execution"): + with ProcessPoolExecutor(max_workers=max_workers) as executor: + future_to_run = { + executor.submit( + run_backtest_worker, + signals_df=run.df_signals, + trade_plans=run.trade_plans, + ): run + for run in self.strategy_runs + } - self.trades_df = self._run_backtest_window(start, end, "FULL") + for future in as_completed(future_to_run): + run = future_to_run[future] + trades = future.result() - elif self.config.BACKTEST_MODE == "split": - all_trades = [] + self.trades_by_run.append(trades) - for name, (start, end) in self.config.BACKTEST_WINDOWS.items(): - trades = self._run_backtest_window( - pd.Timestamp(start, tz="UTC"), - pd.Timestamp(end, tz="UTC"), - name - ) - all_trades.append(trades) + self.log_backtest.log( + f"{run.symbol:<8} | trades={len(trades)}" + ) + # ----------------------------- + # AGGREGATE + # ----------------------------- + with self.log_backtest.time("aggregate"): self.trades_df = ( - pd.concat(all_trades) - .sort_values(by=["exit_time", "symbol"]) + pd.concat(self.trades_by_run) + .sort_values("exit_time") .reset_index(drop=True) ) - else: - raise ValueError(f"Unknown BACKTEST_MODE: {self.config.BACKTEST_MODE}") - - if self.trades_df.empty: - raise RuntimeError("No trades after backtest") + self.log_backtest.log( + f"summary | total_trades={len(self.trades_df)}" + ) return self.trades_df # ================================================== - # 5️⃣ REPORTING (RISK / STRATEGY) + # 4️⃣ RESULT BUILDING # ================================================== - def run_report(self): - - # 1️⃣ PREPARE RISK STATE - preparer = RiskDataPreparer( - initial_balance=INITIAL_BALANCE + def _build_result(self) -> BacktestResult: + if self.trades_df is None: + raise RuntimeError("No trades to build result") + + run_id = f"bt_{uuid4().hex[:8]}" + + metadata = BacktestMetadata.now( + run_id=run_id, + backtest_mode=self.cfg.BACKTEST_MODE, + windows=( + self.cfg.BACKTEST_WINDOWS + if self.cfg.BACKTEST_MODE == "split" + else None + ), + strategies=[s.get_strategy_id() for s in self.strategies], + strategy_names={ + s.get_strategy_id(): s.get_strategy_name() + for s in self.strategies + }, + symbols=self.cfg.SYMBOLS, + timeframe=self.cfg.TIMEFRAME, + initial_balance=self.cfg.INITIAL_BALANCE, + slippage=self.cfg.SLIPPAGE, + max_risk_per_trade=self.cfg.MAX_RISK_PER_TRADE, ) - prepared_df = preparer.prepare(self.trades_df) - # 2️⃣ ENRICH CONTEXTS (CANDLE → TRADE) - enricher = TradeContextEnricher(self.strategy.df_plot) - prepared_df = enricher.enrich( - prepared_df, - self.strategy.report_config.contexts + return BacktestResult( + metadata=metadata, + trades=self.trades_df, ) - - - # 3️⃣ RUN REPORT (PURE) - ReportRunner( - strategy=self.strategy, - trades_df=prepared_df, - config=self.config - ).run() - # ================================================== - # 6️⃣ PLOTTING - # ================================================== - - def plot_results(self): - - plots_folder = "results/plots" - os.makedirs(plots_folder, exist_ok=True) - - for strategy in self.strategies: - symbol = strategy.symbol - - trades_symbol = None - if self.trades_df is not None: - trades_symbol = self.trades_df[ - self.trades_df["symbol"] == symbol - ] - if trades_symbol.empty: - trades_symbol = None - - plotter = TradePlotter( - df=strategy.df_plot, - trades=trades_symbol, - bullish_zones=strategy.get_bullish_zones(), - bearish_zones=strategy.get_bearish_zones(), - extra_series=strategy.get_extra_values_to_plot(), - bool_series=strategy.bool_series(), - title=f"{symbol} chart", - ) - - plotter.plot() - plotter.save(f"{plots_folder}/{symbol}.png") - - # ================================================== - # 7️⃣ MAIN ENTRYPOINT + # 5️⃣ MAIN ENTRYPOINT # ================================================== def run(self): - - t_start = perf_counter() - print("🚀 BacktestRunner | start") - - # LOAD DATA - all_data = self.load_data() - - # STRATEGIES - self.run_strategies_parallel(all_data) - - # PLOT ONLY - if self.config.PLOT_ONLY: - self.plot_results() - print(f"📊 Plot-only finished TOTAL {perf_counter() - t_start:.3f}s") - return - - # BACKTEST - self.run_backtests() - - if self.config.BACKTEST_MODE == "backtest": - print(f"🧪 Backtest finished TOTAL {perf_counter() - t_start:.3f}s") - return - - # REPORT - self.run_report() - - print(f"🏁 Full run finished TOTAL {perf_counter() - t_start:.3f}s") + self.run_path = Path( + f"results/run_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ) + self.run_path.mkdir(parents=True, exist_ok=True) + + self.log_run.log(f"run_path={self.run_path}") + self.log_run.log("start") + + with self.log_run.time("data"): + all_data = self.load_data() + + with self.log_run.time("strategy"): + self.run_strategies(all_data) + + with profiling(self.cfg.PROFILING, self.run_path / "profile.prof"): + with self.log_run.time("backtest"): + self.run_backtests() + + with self.log_run.time("persist"): + result = self._build_result() + run_path = ResultStore().save(result) + + for run, trades in zip(self.strategy_runs, self.trades_by_run): + with self.log_report.time(f"{run.symbol}"): + ReportRunner( + trades=trades, + df_context=run.df_context, + report_spec=run.report_spec, + metadata=result.metadata, + config=self.cfg, + report_config=ReportConfig( + stdout_mode=StdoutMode.OFF, + generate_dashboard=True, + persist_report=True, + ), + run_path=run_path / "per_symbol" / run.symbol, + ).run() + + with self.log_report.time("summary"): + SummaryReportRunner( + strategy_runs=self.strategy_runs, + trades_by_run=self.trades_by_run, + config=self.cfg, + run_path=run_path, + ).run() + + self.log_run.log("finished") diff --git a/core/backtesting/strategy_runner.py b/core/backtesting/strategy_runner.py index 6ed7ddd..d2235ec 100644 --- a/core/backtesting/strategy_runner.py +++ b/core/backtesting/strategy_runner.py @@ -1,86 +1,153 @@ +from dataclasses import dataclass +from typing import Any + import pandas as pd -from core.strategy.orchestration.strategy_execution import execute_strategy +from config.logger_config import RunLogger, NullLogger +from core.strategy.orchestration.informatives import apply_informatives +from core.strategy.plan_builder import PlanBuildContext +from core.utils.timeframe import tf_to_minutes + + +@dataclass(frozen=True) +class StrategyRunResult: + """ + Immutable result of running ONE strategy on ONE symbol. + """ + + symbol: str + strategy_id: str + strategy_name: str + + df_signals: pd.DataFrame + df_context: pd.DataFrame + trade_plans: pd.DataFrame + report_spec: Any + timing: dict[str, float] -def run_strategy_single( + +def strategy_orchestration( + *, symbol: str, - df: pd.DataFrame, - provider, + data_by_tf: dict[str, pd.DataFrame], strategy_cls, startup_candle_count: int, + logger: RunLogger | None = None, ): """ Run single strategy instance for one symbol. - Must be top-level for multiprocessing. + + Returns: + StrategyRunResult + + Multiprocessing-safe. """ - strategy = strategy_cls( - df=df.copy(), - symbol=symbol, - startup_candle_count=startup_candle_count, - ) - strategy.validate() + logger = logger or NullLogger() - df_plot = execute_strategy( - strategy=strategy, - df=df, - provider=provider, - symbol=symbol, - startup_candle_count=startup_candle_count, - ) + # ================================================== + # 1️⃣ BASE TIMEFRAME + # ================================================== + + with logger.section("base_tf"): + base_tf = min(data_by_tf.keys(), key=tf_to_minutes) + df_base = data_by_tf[base_tf].copy() + + # ================================================== + # 2️⃣ STRATEGY INIT + # ================================================== + + with logger.section("strategy_init"): + strategy = strategy_cls( + df=df_base, + symbol=symbol, + startup_candle_count=startup_candle_count, + ) + strategy.validate() + + # ================================================== + # 3️⃣ EXECUTION PIPELINE + # ================================================== + + with logger.section("execute_strategy"): + df_context = apply_informatives( + df=df_base, + strategy=strategy, + data_by_tf=data_by_tf, + ) + + strategy.df = df_context - report_config = strategy.build_report_config() - strategy.report_config = report_config + with logger.section("execute.indicators"): + strategy.populate_indicators() - # ------------------------------------------------- - # FINALIZE DATAFRAMES (EXPLICIT CONTRACT) - # ------------------------------------------------- + with logger.section("execute.entry"): + strategy.populate_entry_trend() - df_plot = df_plot.copy() + with logger.section("signal_stats"): + entry_count = strategy.df["signal_entry"].notna().sum() - # --- LEGACY BACKTEST CONTRACT --- + logger.log( + f"entry signals = {entry_count} ") - REQUIRED_COLUMNS = [ - "time", - "open", - "high", - "low", - "close", - ] + with logger.section("execute.exit"): + strategy.populate_exit_trend() + df_context = strategy.df + # ================================================== + # 4️⃣ BUILD df_signals (EXECUTION CONTRACT) + # ================================================== - SIGNAL_COLUMNS = [ - "signal_entry", - "signal_exit", - "levels", - ] + REQUIRED_COLUMNS = ["time", "open", "high", "low", "close"] + SIGNAL_COLUMNS = ["signal_entry", "signal_exit", "levels"] - # build clean df for backtester - df_signals = df_plot[ - REQUIRED_COLUMNS + [ - c for c in SIGNAL_COLUMNS if c in df_plot.columns - ] - ].copy() + missing = [c for c in REQUIRED_COLUMNS if c not in df_context.columns] + if missing: + raise RuntimeError( + f"Strategy context missing required columns: {missing}" + ) + + df_signals = df_context[ + REQUIRED_COLUMNS + + [c for c in SIGNAL_COLUMNS if c in df_context.columns] + ].copy() # --- HARD GUARANTEES --- - if "signal_entry" not in df_signals.columns: + if "signal_entry" not in df_signals: df_signals["signal_entry"] = None - - if "signal_exit" not in df_signals.columns: + if "signal_exit" not in df_signals: df_signals["signal_exit"] = None - - if "levels" not in df_signals.columns: + if "levels" not in df_signals: df_signals["levels"] = None - if "symbol" not in df_signals.columns: - df_signals["symbol"] = symbol + df_signals["symbol"] = symbol - # attach explicitly for downstream consumers - strategy.df_plot = df_plot - strategy.df_signals = df_signals + # ================================================== + # BUILD TRADE PLANS (STRATEGY RESPONSIBILITY) + # ================================================== - # report configuration (strategy declaration → runtime state) - strategy.report_config = strategy.build_report_config() + with logger.section("build_context_plans"): + ctx = PlanBuildContext( + symbol=symbol, + strategy_name=strategy.get_strategy_name(), + strategy_config=strategy.strategy_config, + ) - return df_signals, strategy \ No newline at end of file + with logger.section("build_trade_plans"): + trade_plans = strategy.build_trade_plans_backtest( + df=df_signals, + ctx=ctx, + allow_managed_in_backtest=False, + ) + + return StrategyRunResult( + symbol=symbol, + strategy_id=strategy.get_strategy_id(), + strategy_name=strategy.get_strategy_name(), + df_signals=df_signals, + df_context=df_context, + trade_plans=trade_plans, + report_spec=strategy.build_report_spec(), + timing=logger.get_timings(), + ) diff --git a/core/backtesting/reporting/core/sections/__init__.py b/core/backtesting/tests/__init__.py similarity index 100% rename from core/backtesting/reporting/core/sections/__init__.py rename to core/backtesting/tests/__init__.py diff --git a/core/backtesting/tests/conftest.py b/core/backtesting/tests/conftest.py new file mode 100644 index 0000000..ba8b7b0 --- /dev/null +++ b/core/backtesting/tests/conftest.py @@ -0,0 +1,47 @@ +import pandas as pd +import pytest + +from core.domain.cost.instrument_ctx import build_instrument_ctx + + +@pytest.fixture +def instrument_ctx(): + return build_instrument_ctx("EURUSD") + + +@pytest.fixture +def base_df(): + return pd.DataFrame({ + "time": pd.date_range("2022-01-01", periods=6, freq="1H"), + "open": [1, 1, 1, 1, 1, 1], + "high": [1, 1.1, 1.1, 1.1, 1.1, 1.1], + "low": [1, 0.9, 0.9, 0.9, 0.9, 0.9], + "close": [1, 1.05, 1.05, 1.05, 1.05, 1.05], + "signal_entry": [None, True, None, None, None, None], + "symbol": ["EURUSD"] * 6, + }) + + +@pytest.fixture +def long_plan(): + return pd.DataFrame({ + "plan_valid": [False, True, False, False, False, False], + "plan_direction": ["long"] * 6, + "plan_entry_tag": ["L1"] * 6, + "plan_sl": [0.9] * 6, + "plan_tp1": [1.1] * 6, + "plan_tp2": [1.2] * 6, + "plan_sl_tag": ["SL"] * 6, + "plan_tp1_tag": ["TP1"] * 6, + "plan_tp2_tag": ["TP2"] * 6, + }) + + +@pytest.fixture +def short_plan(long_plan): + df = long_plan.copy() + df["plan_direction"] = "short" + df["plan_sl"] = 1.2 + df["plan_tp1"] = 1.0 + df["plan_tp2"] = 0.9 + return df diff --git a/core/backtesting/tests/strategy_runner.py b/core/backtesting/tests/strategy_runner.py new file mode 100644 index 0000000..7b8e291 --- /dev/null +++ b/core/backtesting/tests/strategy_runner.py @@ -0,0 +1,17 @@ +import pytest +from core.backtesting.strategy_runner import StrategyRunResult + + +def test_strategy_run_result_is_immutable(): + with pytest.raises(TypeError): + StrategyRunResult( + symbol="EURUSD", + strategy_id="s", + strategy_name="x", + df_signals=None, + df_context=None, + trade_plans=None, + report_spec=None, + timing={}, + extra_field=123, + ) diff --git a/core/backtesting/tests/test_backend_factory.py b/core/backtesting/tests/test_backend_factory.py new file mode 100644 index 0000000..b8ac85d --- /dev/null +++ b/core/backtesting/tests/test_backend_factory.py @@ -0,0 +1,13 @@ +import pytest +from core.backtesting.backend_factory import create_backtest_backend +from core.data_provider.backends.dukascopy_backend import DukascopyBackend + + +def test_create_dukascopy_backend(): + backend = create_backtest_backend("dukascopy") + assert isinstance(backend, DukascopyBackend) + + +def test_create_backend_invalid_name(): + with pytest.raises(ValueError): + create_backtest_backend("invalid_backend") \ No newline at end of file diff --git a/core/backtesting/tests/test_backtest_result_roundtrip.py b/core/backtesting/tests/test_backtest_result_roundtrip.py new file mode 100644 index 0000000..a327299 --- /dev/null +++ b/core/backtesting/tests/test_backtest_result_roundtrip.py @@ -0,0 +1,32 @@ +import pandas as pd +from core.backtesting.results_logic.result import BacktestResult +from core.backtesting.results_logic.metadata import BacktestMetadata +from core.backtesting.results_logic.store import ResultStore + + +def test_backtest_result_save_load(tmp_path): + meta = BacktestMetadata( + run_id="test", + created_at="now", + backtest_mode="single", + windows=None, + strategies=["s1"], + strategy_names={"s1": "TestStrategy"}, + symbols=["EURUSD"], + timeframe="H1", + initial_balance=1000, + slippage=0.0, + max_risk_per_trade=0.01, + ) + + trades = pd.DataFrame([{"a": 1}]) + + result = BacktestResult(metadata=meta, trades=trades) + + store = ResultStore(base_path=tmp_path) + path = store.save(result) + + loaded = store.load("test") + + assert loaded.metadata.run_id == "test" + assert loaded.trades.equals(trades) \ No newline at end of file diff --git a/core/backtesting/tests/test_backtester_api.py b/core/backtesting/tests/test_backtester_api.py new file mode 100644 index 0000000..121f09a --- /dev/null +++ b/core/backtesting/tests/test_backtester_api.py @@ -0,0 +1,34 @@ +import pandas as pd +import pytest + +from core.backtesting.engine.backtester import Backtester + + +def test_backtester_empty_inputs(): + bt = Backtester() + out = bt.run(signals_df=pd.DataFrame(), trade_plans=pd.DataFrame()) + assert out.empty + + +def test_backtester_requires_signal_columns(): + bt = Backtester() + + signals = pd.DataFrame({ + "close": [1, 2, 3], + "symbol": ["EURUSD"] * 3, + }) + + plans = pd.DataFrame({ + "plan_valid": [True, True, True], + "plan_direction": ["long"] * 3, + "plan_entry_tag": ["A"] * 3, + "plan_sl": [0.9] * 3, + "plan_tp1": [1.1] * 3, + "plan_tp2": [1.2] * 3, + "plan_sl_tag": ["SL"] * 3, + "plan_tp1_tag": ["TP1"] * 3, + "plan_tp2_tag": ["TP2"] * 3, + }) + + with pytest.raises(ValueError): + bt.run(signals_df=signals, trade_plans=plans) \ No newline at end of file diff --git a/core/backtesting/tests/test_execution_exit_long.py b/core/backtesting/tests/test_execution_exit_long.py new file mode 100644 index 0000000..5703d03 --- /dev/null +++ b/core/backtesting/tests/test_execution_exit_long.py @@ -0,0 +1,45 @@ +from core.backtesting.engine.execution_loop import run_execution_loop + + +def test_long_hits_sl(base_df, long_plan, instrument_ctx): + plans = long_plan.copy() + plans["plan_tp1"] = 10.0 + plans["plan_sl"] = 0.9 + + df = base_df.copy() + df["high"] = 1.0 + df.loc[2, "low"] = 0.85 + + trades = run_execution_loop( + df=df, symbol="EURUSD", plans=plans, instrument_ctx=instrument_ctx + ) + + assert trades[0]["exit_level_tag"] == "SL" + + +def test_long_tp1_then_be(base_df, long_plan, instrument_ctx): + df = base_df.copy() + df.loc[2, "high"] = 1.1 + df.loc[3, "low"] = df.loc[1, "close"] + + trades = run_execution_loop( + df=df, symbol="EURUSD", plans=long_plan, instrument_ctx=instrument_ctx + ) + + assert trades[0]["exit_level_tag"] == "TP1" + + +def test_long_hits_tp2_without_tp1(base_df, long_plan, instrument_ctx): + plans = long_plan.copy() + plans["plan_tp1"] = 10.0 + plans["plan_sl"] = 0.8 + + df = base_df.copy() + df["low"] = 0.95 + df.loc[2, "high"] = 1.21 + + trades = run_execution_loop( + df=df, symbol="EURUSD", plans=plans, instrument_ctx=instrument_ctx + ) + + assert trades[0]["exit_level_tag"] == "TP2" \ No newline at end of file diff --git a/core/backtesting/tests/test_execution_exit_short.py b/core/backtesting/tests/test_execution_exit_short.py new file mode 100644 index 0000000..89799c0 --- /dev/null +++ b/core/backtesting/tests/test_execution_exit_short.py @@ -0,0 +1,16 @@ +from core.backtesting.engine.execution_loop import run_execution_loop + + +def test_short_hits_sl(base_df, short_plan, instrument_ctx): + plans = short_plan.copy() + plans["plan_tp1"] = -10.0 + + df = base_df.copy() + df["low"] = 1.05 + df.loc[2, "high"] = 1.25 + + trades = run_execution_loop( + df=df, symbol="EURUSD", plans=plans, instrument_ctx=instrument_ctx + ) + + assert trades[0]["exit_level_tag"] == "SL" \ No newline at end of file diff --git a/core/backtesting/tests/test_execution_policy.py b/core/backtesting/tests/test_execution_policy.py new file mode 100644 index 0000000..c291f1f --- /dev/null +++ b/core/backtesting/tests/test_execution_policy.py @@ -0,0 +1,28 @@ +from core.backtesting.execution_policy import ExecutionPolicy, EXEC_MARKET, EXEC_LIMIT + + +def test_exit_signal_forces_market_exit(): + policy = ExecutionPolicy() + + out = policy.classify_exit_type( + exit_reason="TP2", + has_exit_signal=True, + exit_signal_value=True, + ) + + assert out == EXEC_MARKET + + +def test_sl_is_market_exit(): + policy = ExecutionPolicy() + assert policy.classify_exit_type("SL") == EXEC_MARKET + + +def test_tp2_is_limit_exit(): + policy = ExecutionPolicy() + assert policy.classify_exit_type("TP2") == EXEC_LIMIT + + +def test_unknown_reason_fallback(): + policy = ExecutionPolicy(exit_default_type="custom") + assert policy.classify_exit_type("UNKNOWN") == "custom" \ No newline at end of file diff --git a/core/backtesting/tests/test_execution_properties.py b/core/backtesting/tests/test_execution_properties.py new file mode 100644 index 0000000..4101a68 --- /dev/null +++ b/core/backtesting/tests/test_execution_properties.py @@ -0,0 +1,12 @@ +from core.backtesting.engine.execution_loop import run_execution_loop + + +def test_execution_is_deterministic(base_df, long_plan, instrument_ctx): + t1 = run_execution_loop( + df=base_df, symbol="EURUSD", plans=long_plan, instrument_ctx=instrument_ctx + ) + t2 = run_execution_loop( + df=base_df, symbol="EURUSD", plans=long_plan, instrument_ctx=instrument_ctx + ) + + assert t1 == t2 \ No newline at end of file diff --git a/core/backtesting/tests/test_exit_mapping.py b/core/backtesting/tests/test_exit_mapping.py new file mode 100644 index 0000000..d8c5c30 --- /dev/null +++ b/core/backtesting/tests/test_exit_mapping.py @@ -0,0 +1,21 @@ +import pytest +from core.backtesting.exit.exit_mapping import map_exit_code_to_reason +from core.domain.trade.trade_exit import TradeExitReason +from core.backtesting.exit.simulate_exit_numba import ( + EXIT_SL, + EXIT_TP1_BE, + EXIT_TP2, + EXIT_EOD, +) + + +def test_exit_code_mapping(): + assert map_exit_code_to_reason(EXIT_SL) is TradeExitReason.SL + assert map_exit_code_to_reason(EXIT_TP1_BE) is TradeExitReason.BE + assert map_exit_code_to_reason(EXIT_TP2) is TradeExitReason.TP2 + assert map_exit_code_to_reason(EXIT_EOD) is TradeExitReason.TIMEOUT + + +def test_unknown_exit_code_raises(): + with pytest.raises(ValueError): + map_exit_code_to_reason(999) \ No newline at end of file diff --git a/core/backtesting/tests/test_last_exit_by_tag.py b/core/backtesting/tests/test_last_exit_by_tag.py new file mode 100644 index 0000000..c240104 --- /dev/null +++ b/core/backtesting/tests/test_last_exit_by_tag.py @@ -0,0 +1,24 @@ +from core.backtesting.engine.execution_loop import run_execution_loop + + +def test_no_overlapping_trades_by_tag(base_df, long_plan, instrument_ctx): + plans = long_plan.copy() + plans["plan_valid"] = True + plans["plan_entry_tag"] = ["A"] * len(plans) + + plans["plan_tp1"] = 10.0 + plans["plan_sl"] = 0.5 + + df = base_df.copy() + df["high"] = 1.0 + df["low"] = 1.0 + + trades = run_execution_loop( + df=df, + symbol="EURUSD", + plans=plans, + instrument_ctx=instrument_ctx, + ) + + for t1, t2 in zip(trades, trades[1:]): + assert t1["exit_time"] <= t2["entry_time"] \ No newline at end of file diff --git a/core/backtesting/tests/test_plan_validity.py b/core/backtesting/tests/test_plan_validity.py new file mode 100644 index 0000000..6b745bc --- /dev/null +++ b/core/backtesting/tests/test_plan_validity.py @@ -0,0 +1,15 @@ +from core.backtesting.engine.execution_loop import run_execution_loop + + +def test_plan_valid_false_produces_no_trades(base_df, long_plan, instrument_ctx): + plans = long_plan.copy() + plans["plan_valid"] = False + + trades = run_execution_loop( + df=base_df, + symbol="EURUSD", + plans=plans, + instrument_ctx=instrument_ctx, + ) + + assert trades == [] diff --git a/core/backtesting/tests/test_worker.py b/core/backtesting/tests/test_worker.py new file mode 100644 index 0000000..d8bec84 --- /dev/null +++ b/core/backtesting/tests/test_worker.py @@ -0,0 +1,11 @@ +import pandas as pd +from core.backtesting.engine.worker import run_backtest_worker + + +def test_run_backtest_worker_smoke(base_df, long_plan): + out = run_backtest_worker( + signals_df=base_df, + trade_plans=long_plan, + ) + + assert out is not None \ No newline at end of file diff --git a/core/backtesting/tests/test_worker_and_parallel.py b/core/backtesting/tests/test_worker_and_parallel.py new file mode 100644 index 0000000..9ac7501 --- /dev/null +++ b/core/backtesting/tests/test_worker_and_parallel.py @@ -0,0 +1,10 @@ +from core.backtesting.engine.worker import run_backtest_worker + + +def test_run_backtest_worker_smoke(base_df, long_plan): + trades = run_backtest_worker( + signals_df=base_df, + trade_plans=long_plan, + ) + + assert trades is not None diff --git a/core/data_provider/clients/dukascopy_client.py b/core/data_provider/clients/dukascopy_client.py index 672127a..1f56c82 100644 --- a/core/data_provider/clients/dukascopy_client.py +++ b/core/data_provider/clients/dukascopy_client.py @@ -74,10 +74,6 @@ def _run_dukascopy_node( ) -> Path: workdir = Path(tempfile.mkdtemp()) - print( - f"📥 Dukascopy | fetching {symbol} {timeframe} " - f"{start.strftime('%Y-%m-%d')} → {end.strftime('%Y-%m-%d')}" - ) cmd = [ self.npx_cmd, @@ -94,10 +90,6 @@ def _run_dukascopy_node( cwd=workdir, stdout=None, stderr=None, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", ) if proc.returncode != 0: @@ -128,11 +120,6 @@ def _run_dukascopy_node( f"STDERR:\n{proc.stderr}" ) - print( - f"✅ Dukascopy | OK {symbol} {timeframe} " - f"size={csv_path.stat().st_size / 1024:.1f} KB " - f"file={csv_path.name}" - ) return csv_path diff --git a/core/data_provider/providers/default_provider.py b/core/data_provider/providers/default_provider.py index acf1755..135bbe7 100644 --- a/core/data_provider/providers/default_provider.py +++ b/core/data_provider/providers/default_provider.py @@ -2,6 +2,7 @@ import pandas as pd +from config.logger_config import RunLogger from core.data_provider.ohlcv_schema import sort_and_deduplicate, ensure_utc_time from core.utils.timeframe import timeframe_to_pandas_freq @@ -23,11 +24,13 @@ def __init__( cache, backtest_start: pd.Timestamp, backtest_end: pd.Timestamp, + logger ): self.backend = backend self.cache = cache self.backtest_start = self._to_utc(backtest_start) self.backtest_end = self._to_utc(backtest_end) + self.logger = logger # ------------------------------------------------- # Helpers @@ -70,103 +73,113 @@ def shift_time_by_candles( # ------------------------------------------------- def get_ohlcv( - self, - *, - symbol: str, - timeframe: str, - start: pd.Timestamp, - end: pd.Timestamp, + self, + *, + symbol: str, + timeframe: str, + start: pd.Timestamp, + end: pd.Timestamp, ) -> pd.DataFrame: start = self._to_utc(start) end = self._to_utc(end) - coverage = self.cache.coverage( - symbol=symbol, - timeframe=timeframe, - ) - - - pieces: list[pd.DataFrame] = [] + with self.logger.section(f"{symbol} {timeframe}"): - # ================================================= - # 1️⃣ NO CACHE AT ALL - # ================================================= - if coverage is None: - df = self.backend.fetch_ohlcv( + coverage = self.cache.coverage( symbol=symbol, timeframe=timeframe, - start=start, - end=end, ) - df = self._validate(df) - self.cache.save(symbol=symbol, timeframe=timeframe, df=df) - return df - cov_start, cov_end = coverage + pieces: list[pd.DataFrame] = [] - # ================================================= - # 2️⃣ MISSING BEFORE (FIXED) - # ================================================= + # ================================================= + # 1️⃣ NO CACHE + # ================================================= + if coverage is None: + self.logger.log("cache MISS → fetch full range") - freq = timeframe_to_pandas_freq(timeframe) - first_required_bar = start.floor(freq) - - if first_required_bar < cov_start: - df_pre = self.backend.fetch_ohlcv( - symbol=symbol, - timeframe=timeframe, - start=first_required_bar, - end=cov_start, - ) - df_pre = self._validate(df_pre) - if not df_pre.empty: - self.cache.append( + df = self.backend.fetch_ohlcv( symbol=symbol, timeframe=timeframe, - df=df_pre, + start=start, + end=end, ) - pieces.append(df_pre) + df = self._validate(df) - # ================================================= - # 3️⃣ CACHED MIDDLE - # ================================================= - df_mid = self.cache.load_range( - symbol=symbol, - timeframe=timeframe, - start=max(start, cov_start), - end=min(end, cov_end), - ) - pieces.append(df_mid) + self.cache.save(symbol=symbol, timeframe=timeframe, df=df) + self.logger.log(f"fetched {len(df)} bars") - # ================================================= - # 4️⃣ MISSING AFTER (FIXED) - # ================================================= + return df - freq = timeframe_to_pandas_freq(timeframe) - last_required_bar = end.floor(freq) + cov_start, cov_end = coverage + self.logger.log(f"cache coverage {cov_start} → {cov_end}") + + freq = timeframe_to_pandas_freq(timeframe) + first_required_bar = start.floor(freq) + last_required_bar = end.floor(freq) - if last_required_bar > cov_end: - df_post = self.backend.fetch_ohlcv( + # ================================================= + # 2️⃣ BEFORE + # ================================================= + if first_required_bar < cov_start: + self.logger.log("cache MISS before") + + df_pre = self.backend.fetch_ohlcv( + symbol=symbol, + timeframe=timeframe, + start=first_required_bar, + end=cov_start, + ) + df_pre = self._validate(df_pre) + + if not df_pre.empty: + self.cache.append(symbol=symbol, timeframe=timeframe, df=df_pre) + pieces.append(df_pre) + self.logger.log(f"fetched {len(df_pre)} bars (pre)") + else: + self.logger.log("cache HIT before") + + # ================================================= + # 3️⃣ MIDDLE + # ================================================= + df_mid = self.cache.load_range( symbol=symbol, timeframe=timeframe, - start=cov_end, - end=last_required_bar, + start=max(start, cov_start), + end=min(end, cov_end), ) - df_post = self._validate(df_post) - if not df_post.empty: - self.cache.append( + pieces.append(df_mid) + self.logger.log(f"cache HIT middle ({len(df_mid)} bars)") + + # ================================================= + # 4️⃣ AFTER + # ================================================= + if last_required_bar > cov_end: + self.logger.log("cache MISS after") + + df_post = self.backend.fetch_ohlcv( symbol=symbol, timeframe=timeframe, - df=df_post, + start=cov_end, + end=last_required_bar, ) - pieces.append(df_post) - - # ================================================= - # 5️⃣ FINAL MERGE - # ================================================= - df = pd.concat(pieces, ignore_index=True) - return self._validate(df) + df_post = self._validate(df_post) + + if not df_post.empty: + self.cache.append(symbol=symbol, timeframe=timeframe, df=df_post) + pieces.append(df_post) + self.logger.log(f"fetched {len(df_post)} bars (post)") + else: + self.logger.log("cache HIT after") + + # ================================================= + # 5️⃣ MERGE + # ================================================= + df = pd.concat(pieces, ignore_index=True) + self.logger.log(f"final {len(df)} bars") + + return self._validate(df) # ------------------------------------------------- # Informative data diff --git a/core/data_provider/tests/test_default_provider.py b/core/data_provider/tests/test_default_provider.py index 3451f4d..3cd9d05 100644 --- a/core/data_provider/tests/test_default_provider.py +++ b/core/data_provider/tests/test_default_provider.py @@ -1,5 +1,7 @@ import pandas as pd +from config.logger_config import NullLogger + from core.data_provider import CsvMarketDataCache from core.data_provider.providers.default_provider import DefaultOhlcvDataProvider @@ -22,6 +24,7 @@ def test_no_cache_fetches_and_saves(tmp_path, utc): cache=cache, backtest_start=start, backtest_end=end, + logger=NullLogger(), ) out = p.get_ohlcv(symbol="EURUSD", timeframe="M1", start=start, end=end) @@ -55,7 +58,8 @@ def test_missing_before_fetches_pre_and_appends(tmp_path, utc): from core.data_provider.providers.default_provider import DefaultOhlcvDataProvider p = DefaultOhlcvDataProvider( backend=backend, cache=cache, - backtest_start=start, backtest_end=end + backtest_start=start, backtest_end=end, + logger=NullLogger(), ) out = p.get_ohlcv(symbol="EURUSD", timeframe="M1", start=start, end=end) @@ -88,7 +92,8 @@ def test_missing_after_fetches_post_and_appends(tmp_path, utc): from core.data_provider.providers.default_provider import DefaultOhlcvDataProvider p = DefaultOhlcvDataProvider( backend=backend, cache=cache, - backtest_start=start, backtest_end=end + backtest_start=start, backtest_end=end, + logger=NullLogger(), ) out = p.get_ohlcv(symbol="EURUSD", timeframe="M1", start=start, end=end) diff --git a/core/domain/cost/cost_engine.py b/core/domain/cost/cost_engine.py index c82036b..b8ac8d2 100644 --- a/core/domain/cost/cost_engine.py +++ b/core/domain/cost/cost_engine.py @@ -21,7 +21,7 @@ def __init__(self, execution_policy): self.execution_policy = execution_policy - def enrich(self, trade_dict: dict, *, df, ctx: InstrumentCtx) -> None: + def apply(self, trade_dict: dict, *, df, ctx: InstrumentCtx) -> None: attach_execution_types( trade_dict, df=df, diff --git a/core/domain/cost/instrument_ctx.py b/core/domain/cost/instrument_ctx.py index 07c71a9..e13f104 100644 --- a/core/domain/cost/instrument_ctx.py +++ b/core/domain/cost/instrument_ctx.py @@ -1,5 +1,8 @@ from dataclasses import dataclass +from config.backtest import SLIPPAGE +from config.instrument_meta import INSTRUMENT_META, get_spread_abs + @dataclass(frozen=True) class InstrumentCtx: @@ -9,4 +12,26 @@ class InstrumentCtx: contract_size: float spread_abs: float half_spread: float - slippage_abs: float \ No newline at end of file + slippage_abs: float + + +def build_instrument_ctx(symbol: str) -> InstrumentCtx: + meta = INSTRUMENT_META[symbol] + + point_size = float(meta["point"]) + pip_value = float(meta["pip_value"]) + contract_size = float(meta.get("contract_size", 1.0)) + + spread_abs = get_spread_abs(symbol, point_size) + half_spread = 0.5 * spread_abs + slippage_abs = float(SLIPPAGE) * point_size + + return InstrumentCtx( + symbol=symbol, + point_size=point_size, + pip_value=pip_value, + contract_size=contract_size, + spread_abs=spread_abs, + half_spread=half_spread, + slippage_abs=slippage_abs, + ) \ No newline at end of file diff --git a/core/domain/tests/test_exit_mapping.py b/core/domain/tests/test_exit_mapping.py index a9c8c96..872821c 100644 --- a/core/domain/tests/test_exit_mapping.py +++ b/core/domain/tests/test_exit_mapping.py @@ -2,7 +2,7 @@ from core.domain.execution.execution_mapping import map_exit_code_to_reason from core.domain.trade.trade_exit import TradeExitReason -from core.backtesting.simulate_exit_numba import ( +from core.backtesting.exit.simulate_exit_numba import ( EXIT_SL, EXIT_TP1_BE, EXIT_TP2, diff --git a/core/domain/tests/test_exit_processor.py b/core/domain/tests/test_exit_processor.py index 89a30ad..e30a0fe 100644 --- a/core/domain/tests/test_exit_processor.py +++ b/core/domain/tests/test_exit_processor.py @@ -4,7 +4,7 @@ from core.domain.execution.exit_processor import ExitProcessor from core.domain.trade.trade_exit import TradeExitReason -from core.backtesting.simulate_exit_numba import EXIT_SL, EXIT_TP2 +from core.backtesting.exit.simulate_exit_numba import EXIT_SL, EXIT_TP2 def test_exit_processor_sl(): diff --git a/core/domain/tests/test_trade_cost_engine.py b/core/domain/tests/test_trade_cost_engine.py index 0437679..aa1b057 100644 --- a/core/domain/tests/test_trade_cost_engine.py +++ b/core/domain/tests/test_trade_cost_engine.py @@ -28,7 +28,7 @@ def test_trade_cost_engine_does_not_mutate_prices(): slippage_abs=0.0, ) - engine.enrich(trade, df=None, ctx=ctx) + engine.apply(trade, df=None, ctx=ctx) assert trade["entry_price"] == 1.0000 assert trade["exit_price"] == 1.0100 diff --git a/core/domain/tests/test_trade_cost_engine_financing.py b/core/domain/tests/test_trade_cost_engine_financing.py index 71b3cc8..174f9ce 100644 --- a/core/domain/tests/test_trade_cost_engine_financing.py +++ b/core/domain/tests/test_trade_cost_engine_financing.py @@ -33,6 +33,6 @@ def test_financing_is_zero_when_disabled(monkeypatch): slippage_abs=0.0, ) - engine.enrich(trade, df=None, ctx=ctx) + engine.apply(trade, df=None, ctx=ctx) assert trade["financing_usd_total"] == 0.0 \ No newline at end of file diff --git a/core/live_trading/run_trading.py b/core/live_trading/run_trading.py index 8bd33d2..958bdca 100644 --- a/core/live_trading/run_trading.py +++ b/core/live_trading/run_trading.py @@ -93,7 +93,6 @@ def _build_provider(self) -> MT5Client: provider = MT5Client(bars_per_tf=bars_per_tf) - print(f"📡 Informative TFs: {bars_per_tf}") return provider # ================================================== diff --git a/core/backtesting/reporting/TODO_notes b/core/reporting/TODO_notes similarity index 100% rename from core/backtesting/reporting/TODO_notes rename to core/reporting/TODO_notes diff --git a/core/backtesting/reporting/renders/__init__.py b/core/reporting/__init__.py similarity index 100% rename from core/backtesting/reporting/renders/__init__.py rename to core/reporting/__init__.py diff --git a/core/backtesting/reporting/renders/dashboard/__init__.py b/core/reporting/config/__init__.py similarity index 100% rename from core/backtesting/reporting/renders/dashboard/__init__.py rename to core/reporting/config/__init__.py diff --git a/core/reporting/config/report_spec.py b/core/reporting/config/report_spec.py new file mode 100644 index 0000000..29bd64b --- /dev/null +++ b/core/reporting/config/report_spec.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass, field +from typing import List + +from core.reporting.core.base import BaseMetric +from core.reporting.core.context import ContextSpec + + +@dataclass +class StrategyReportSpec: + """ + Declarative report specification provided by a strategy. + + Defines: + - which metrics should be computed + - which contextual features should be attached to trades + """ + + metrics: List[BaseMetric] = field(default_factory=list) + contexts: List[ContextSpec] = field(default_factory=list) + + def add_metric(self, metric: BaseMetric) -> "StrategyReportSpec": + self.metrics.append(metric) + return self + + def add_context(self, context: ContextSpec) -> "StrategyReportSpec": + self.contexts.append(context) + return self diff --git a/core/backtesting/reporting/reports/__init__.py b/core/reporting/core/__init__.py similarity index 100% rename from core/backtesting/reporting/reports/__init__.py rename to core/reporting/core/__init__.py diff --git a/core/backtesting/reporting/core/aggregration.py b/core/reporting/core/aggregration.py similarity index 94% rename from core/backtesting/reporting/core/aggregration.py rename to core/reporting/core/aggregration.py index b8279f7..1b57ef1 100644 --- a/core/backtesting/reporting/core/aggregration.py +++ b/core/reporting/core/aggregration.py @@ -1,8 +1,8 @@ import numpy as np import pandas as pd -from core.backtesting.reporting.core.base import BaseAggregator -from core.backtesting.reporting.core.context import ContextSpec +from core.reporting.core.base import BaseAggregator +from core.reporting.core.context import ContextSpec class ContextualAggregator(BaseAggregator): diff --git a/core/backtesting/reporting/core/base.py b/core/reporting/core/base.py similarity index 100% rename from core/backtesting/reporting/core/base.py rename to core/reporting/core/base.py diff --git a/core/backtesting/reporting/core/contex_enricher.py b/core/reporting/core/contex_enricher.py similarity index 100% rename from core/backtesting/reporting/core/contex_enricher.py rename to core/reporting/core/contex_enricher.py diff --git a/core/reporting/core/context.py b/core/reporting/core/context.py new file mode 100644 index 0000000..dae69cd --- /dev/null +++ b/core/reporting/core/context.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from typing import Optional, Set, Any + +import pandas as pd + + +@dataclass(frozen=True) +class ContextSpec: + name: str + column: str + source: str + allowed_values: Optional[Set] = None + + +@dataclass +class ReportContext: + # --- core data --- + trades: pd.DataFrame + equity: pd.Series | None + drawdown: pd.Series | None + + df_plot: pd.DataFrame | None + + initial_balance: float + config: Any + + metadata: Any | None = None + + strategy: Any | None = None + + def __post_init__(self): + """ + Adapter layer: + - old code expects ctx.strategy + - new code should use ctx.metadata + """ + if self.strategy is None and self.metadata is not None: + self.strategy = self.metadata \ No newline at end of file diff --git a/core/backtesting/reporting/core/equity.py b/core/reporting/core/equity.py similarity index 100% rename from core/backtesting/reporting/core/equity.py rename to core/reporting/core/equity.py diff --git a/core/backtesting/reporting/core/formating.py b/core/reporting/core/formating.py similarity index 100% rename from core/backtesting/reporting/core/formating.py rename to core/reporting/core/formating.py diff --git a/core/backtesting/reporting/core/metrics.py b/core/reporting/core/metrics.py similarity index 91% rename from core/backtesting/reporting/core/metrics.py rename to core/reporting/core/metrics.py index 888109d..65a7554 100644 --- a/core/backtesting/reporting/core/metrics.py +++ b/core/reporting/core/metrics.py @@ -1,6 +1,6 @@ import pandas as pd -from core.backtesting.reporting.core.base import BaseMetric +from core.reporting.core.base import BaseMetric class ExpectancyMetric(BaseMetric): diff --git a/core/reporting/core/persistence.py b/core/reporting/core/persistence.py new file mode 100644 index 0000000..0739557 --- /dev/null +++ b/core/reporting/core/persistence.py @@ -0,0 +1,59 @@ +from pathlib import Path +import json +import pandas as pd + + +class ReportPersistence: + """ + Persist report outputs INSIDE a backtest run directory. + + Contract: + - base_path == results/backtests/{run_id}/report + - does NOT create run_id + - does NOT timestamp directories + - does NOT duplicate raw trades + """ + + def __init__(self, base_path: Path): + self.base_path = Path(base_path) + self.base_path.mkdir(parents=True, exist_ok=True) + + def persist( + self, + *, + trades: pd.DataFrame, + equity: pd.Series, + report_data: dict, + meta: dict | None = None, + ) -> Path: + """ + Persists analytics + report artifacts. + + Files written: + - report.json + - equity.parquet + - meta.json + """ + + # ---------------------------- + # equity snapshot + # ---------------------------- + equity.to_frame(name="equity").to_parquet( + self.base_path / "equity.parquet", + index=False, + ) + + # ---------------------------- + # report (JSON) + # ---------------------------- + with open(self.base_path / "report.json", "w", encoding="utf-8") as f: + json.dump(report_data, f, indent=2, default=str) + + # ---------------------------- + # meta + # ---------------------------- + meta_payload = meta or {} + with open(self.base_path / "meta.json", "w", encoding="utf-8") as f: + json.dump(meta_payload, f, indent=2) + + return self.base_path diff --git a/core/reporting/core/preparer.py b/core/reporting/core/preparer.py new file mode 100644 index 0000000..681b94d --- /dev/null +++ b/core/reporting/core/preparer.py @@ -0,0 +1,33 @@ +import pandas as pd + + +class RiskDataPreparer: + def __init__(self, initial_balance: float, timezone: str = "UTC"): + self.initial_balance = initial_balance + self.timezone = timezone + + def prepare(self, trades: pd.DataFrame) -> pd.DataFrame: + trades = trades.copy() + + for col in ("entry_time", "exit_time"): + if col not in trades.columns: + continue + + if not pd.api.types.is_datetime64_any_dtype(trades[col]): + trades[col] = pd.to_datetime(trades[col]) + + if trades[col].dt.tz is None: + trades[col] = trades[col].dt.tz_localize(self.timezone) + + else: + trades[col] = trades[col].dt.tz_convert(self.timezone) + + trades = trades.sort_values("exit_time").reset_index(drop=True) + + + trades["equity"] = self.initial_balance + trades["pnl_usd"].cumsum() + + running_max = trades["equity"].cummax() + trades["drawdown"] = trades["equity"] - running_max + + return trades diff --git a/core/backtesting/reporting/core/section.py b/core/reporting/core/section.py similarity index 71% rename from core/backtesting/reporting/core/section.py rename to core/reporting/core/section.py index 697f3e5..8b75c22 100644 --- a/core/backtesting/reporting/core/section.py +++ b/core/reporting/core/section.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.context import ReportContext class ReportSection(ABC): diff --git a/core/backtesting/reporting/core/sections/TODO_ideas b/core/reporting/core/sections/TODO_ideas similarity index 100% rename from core/backtesting/reporting/core/sections/TODO_ideas rename to core/reporting/core/sections/TODO_ideas diff --git a/core/reporting/core/sections/__init__.py b/core/reporting/core/sections/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/backtesting/reporting/core/sections/backtest_config.py b/core/reporting/core/sections/backtest_config.py similarity index 99% rename from core/backtesting/reporting/core/sections/backtest_config.py rename to core/reporting/core/sections/backtest_config.py index 5dbd527..bbdf3a2 100644 --- a/core/backtesting/reporting/core/sections/backtest_config.py +++ b/core/reporting/core/sections/backtest_config.py @@ -3,7 +3,7 @@ import inspect from typing import Any, Dict, Optional, List -from core.backtesting.reporting.core.section import ReportSection +from core.reporting.core.section import ReportSection def extract_informative_timeframes(strategy: Any) -> Optional[List[str]]: diff --git a/core/backtesting/reporting/core/sections/capital_exposure.py b/core/reporting/core/sections/capital_exposure.py similarity index 95% rename from core/backtesting/reporting/core/sections/capital_exposure.py rename to core/reporting/core/sections/capital_exposure.py index 3cc88cf..f2363a5 100644 --- a/core/backtesting/reporting/core/sections/capital_exposure.py +++ b/core/reporting/core/sections/capital_exposure.py @@ -1,8 +1,8 @@ import pandas as pd from typing import Dict, Any -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class CapitalExposureSection(ReportSection): @@ -111,7 +111,7 @@ def _overtrading_diagnostics(self, trades, trades_per_day): ) grouped = ( - daily.groupby("bucket") + daily.groupby("bucket", observed=True) .agg( days=("day", "count"), avg_trades=("trades", "mean"), diff --git a/core/backtesting/reporting/core/sections/conditional_entry_tag.py b/core/reporting/core/sections/conditional_entry_tag.py similarity index 94% rename from core/backtesting/reporting/core/sections/conditional_entry_tag.py rename to core/reporting/core/sections/conditional_entry_tag.py index baea69f..464e9f2 100644 --- a/core/backtesting/reporting/core/sections/conditional_entry_tag.py +++ b/core/reporting/core/sections/conditional_entry_tag.py @@ -3,8 +3,8 @@ import pandas as pd -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class ConditionalEntryTagPerformanceSection(ReportSection): @@ -25,7 +25,6 @@ def compute(self, ctx: ReportContext) -> Dict[str, Any]: if "entry_tag" not in trades.columns: return {"error": "entry_tag missing"} - trades["entry_time"] = trades["entry_time"].astype("datetime64[ns, UTC]") trades["hour"] = trades["entry_time"].dt.hour trades["weekday"] = trades["entry_time"].dt.day_name() @@ -97,7 +96,7 @@ def _detect_context_columns(self, trades): "returns", "entry_tag", "exit_tag", "exit_level_tag", "duration", "window", "equity", "equity_peak", "drawdown", - "hour", "weekday", + "hour", "weekday", "tp1_price","tp1_pnl", "exec_type_entry", "exec_type_tp1", @@ -120,6 +119,10 @@ def _detect_context_columns(self, trades): "costs_usd_total", "pnl_net_usd", + "strategy_id", + "financing_usd_weekend", + "financing_usd_overnight", + "financing_usd_total" } issues = [] diff --git a/core/backtesting/reporting/core/sections/conditional_expectancy.py b/core/reporting/core/sections/conditional_expectancy.py similarity index 95% rename from core/backtesting/reporting/core/sections/conditional_expectancy.py rename to core/reporting/core/sections/conditional_expectancy.py index 900ee70..099240f 100644 --- a/core/backtesting/reporting/core/sections/conditional_expectancy.py +++ b/core/reporting/core/sections/conditional_expectancy.py @@ -3,8 +3,8 @@ import pandas as pd -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class ConditionalExpectancySection(ReportSection): @@ -23,8 +23,6 @@ def compute(self, ctx: ReportContext) -> Dict[str, Any]: if trades.empty: return {"error": "No trades available"} - trades["entry_time"] = trades["entry_time"].astype("datetime64[ns, UTC]") - results: Dict[str, Any] = {} issues = [] @@ -122,7 +120,7 @@ def _detect_context_columns(self, trades): "returns", "entry_tag", "exit_tag", "exit_level_tag", "duration", "window", "equity", "equity_peak", "drawdown", - "hour", "weekday", + "hour", "weekday", "tp1_price", "tp1_pnl", "exec_type_entry", "exec_type_tp1", @@ -145,6 +143,10 @@ def _detect_context_columns(self, trades): "costs_usd_total", "pnl_net_usd", + "strategy_id", + "financing_usd_weekend", + "financing_usd_overnight", + "financing_usd_total" } cols = [] diff --git a/core/backtesting/reporting/core/sections/drawdown_structure.py b/core/reporting/core/sections/drawdown_structure.py similarity index 97% rename from core/backtesting/reporting/core/sections/drawdown_structure.py rename to core/reporting/core/sections/drawdown_structure.py index e3292f3..11358b4 100644 --- a/core/backtesting/reporting/core/sections/drawdown_structure.py +++ b/core/reporting/core/sections/drawdown_structure.py @@ -1,8 +1,8 @@ import numpy as np from typing import Dict, Any -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class DrawdownStructureSection(ReportSection): diff --git a/core/backtesting/reporting/core/sections/entry_tag_performance.py b/core/reporting/core/sections/entry_tag_performance.py similarity index 96% rename from core/backtesting/reporting/core/sections/entry_tag_performance.py rename to core/reporting/core/sections/entry_tag_performance.py index 138ea5a..156ca2a 100644 --- a/core/backtesting/reporting/core/sections/entry_tag_performance.py +++ b/core/reporting/core/sections/entry_tag_performance.py @@ -1,8 +1,8 @@ import numpy as np from typing import Dict, Any -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class EntryTagPerformanceSection(ReportSection): diff --git a/core/backtesting/reporting/core/sections/exit_logic_diagnostics.py b/core/reporting/core/sections/exit_logic_diagnostics.py similarity index 96% rename from core/backtesting/reporting/core/sections/exit_logic_diagnostics.py rename to core/reporting/core/sections/exit_logic_diagnostics.py index 94d9f2b..b313035 100644 --- a/core/backtesting/reporting/core/sections/exit_logic_diagnostics.py +++ b/core/reporting/core/sections/exit_logic_diagnostics.py @@ -1,8 +1,8 @@ import numpy as np from typing import Dict, Any -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class ExitLogicDiagnosticsSection(ReportSection): diff --git a/core/backtesting/reporting/core/sections/kpi.py b/core/reporting/core/sections/kpi.py similarity index 98% rename from core/backtesting/reporting/core/sections/kpi.py rename to core/reporting/core/sections/kpi.py index dd1ac1f..0f35a9a 100644 --- a/core/backtesting/reporting/core/sections/kpi.py +++ b/core/reporting/core/sections/kpi.py @@ -4,8 +4,8 @@ import numpy as np import pandas as pd -from core.backtesting.reporting.core.context import ReportContext -from core.backtesting.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection class CorePerformanceSection(ReportSection): diff --git a/core/backtesting/reporting/core/sections/tail_risk.py b/core/reporting/core/sections/tail_risk.py similarity index 93% rename from core/backtesting/reporting/core/sections/tail_risk.py rename to core/reporting/core/sections/tail_risk.py index 0ee370e..43b8a36 100644 --- a/core/backtesting/reporting/core/sections/tail_risk.py +++ b/core/reporting/core/sections/tail_risk.py @@ -1,8 +1,8 @@ import numpy as np from typing import Dict, Any -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class TailRiskSection(ReportSection): diff --git a/core/backtesting/reporting/core/sections/trade_distribution.py b/core/reporting/core/sections/trade_distribution.py similarity index 93% rename from core/backtesting/reporting/core/sections/trade_distribution.py rename to core/reporting/core/sections/trade_distribution.py index 1adaba3..6721b0f 100644 --- a/core/backtesting/reporting/core/sections/trade_distribution.py +++ b/core/reporting/core/sections/trade_distribution.py @@ -1,5 +1,5 @@ -from core.backtesting.reporting.core.section import ReportSection -from core.backtesting.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection +from core.reporting.core.context import ReportContext class TradeDistributionSection(ReportSection): diff --git a/core/reporting/renders/__init__.py b/core/reporting/renders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/backtesting/reporting/renders/base.py b/core/reporting/renders/base.py similarity index 100% rename from core/backtesting/reporting/renders/base.py rename to core/reporting/renders/base.py diff --git a/core/reporting/renders/dashboard/__init__.py b/core/reporting/renders/dashboard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/backtesting/reporting/renders/dashboard/dashboard_renderer.py b/core/reporting/renders/dashboard/dashboard_renderer.py similarity index 81% rename from core/backtesting/reporting/renders/dashboard/dashboard_renderer.py rename to core/reporting/renders/dashboard/dashboard_renderer.py index 36193be..5f0e6ba 100644 --- a/core/backtesting/reporting/renders/dashboard/dashboard_renderer.py +++ b/core/reporting/renders/dashboard/dashboard_renderer.py @@ -1,21 +1,22 @@ from pathlib import Path -from jinja2 import Environment, FileSystemLoader import json import shutil +from jinja2 import Environment, FileSystemLoader class DashboardRenderer: """ - Renders RiskReport output into a single-page HTML dashboard. - NO computations. Layout handled in HTML/CSS. + Renders RiskReport into HTML dashboard. + NO computations. """ - def __init__(self): + def __init__(self, run_path: Path): base = Path(__file__).parent self.template_dir = base / "templates" self.static_dir = base / "static" - self.output_dir = Path("dashboard_output") - self.output_dir.mkdir(exist_ok=True) + + self.output_dir = run_path / "report" + self.output_dir.mkdir(parents=True, exist_ok=True) self.env = Environment( loader=FileSystemLoader(self.template_dir), @@ -34,9 +35,9 @@ def render(self, report_data: dict, ctx) -> Path: "time": ctx.trades["exit_time"].astype(str).tolist(), "equity": ctx.trades["equity"].tolist(), "drawdown": ctx.trades["drawdown"].tolist(), - } + }, }, - default=str + default=str, ), ) @@ -47,9 +48,8 @@ def render(self, report_data: dict, ctx) -> Path: return out def _copy_static(self): - target = self.output_dir / "static" if target.exists(): shutil.rmtree(target) - shutil.copytree(self.static_dir, target) + diff --git a/core/backtesting/reporting/renders/dashboard/static/dashboard.css b/core/reporting/renders/dashboard/static/dashboard.css similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/dashboard.css rename to core/reporting/renders/dashboard/static/dashboard.css diff --git a/core/backtesting/reporting/renders/dashboard/static/dashboard.css.map b/core/reporting/renders/dashboard/static/dashboard.css.map similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/dashboard.css.map rename to core/reporting/renders/dashboard/static/dashboard.css.map diff --git a/core/backtesting/reporting/renders/dashboard/static/dashboard.js b/core/reporting/renders/dashboard/static/dashboard.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/dashboard.js rename to core/reporting/renders/dashboard/static/dashboard.js diff --git a/core/backtesting/reporting/renders/dashboard/static/dashboard.scss b/core/reporting/renders/dashboard/static/dashboard.scss similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/dashboard.scss rename to core/reporting/renders/dashboard/static/dashboard.scss diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/backtest_config.js b/core/reporting/renders/dashboard/static/sections/backtest_config.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/backtest_config.js rename to core/reporting/renders/dashboard/static/sections/backtest_config.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/capital_exposure.js b/core/reporting/renders/dashboard/static/sections/capital_exposure.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/capital_exposure.js rename to core/reporting/renders/dashboard/static/sections/capital_exposure.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/conditional_entry_tag.js b/core/reporting/renders/dashboard/static/sections/conditional_entry_tag.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/conditional_entry_tag.js rename to core/reporting/renders/dashboard/static/sections/conditional_entry_tag.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/conditional_expectancy.js b/core/reporting/renders/dashboard/static/sections/conditional_expectancy.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/conditional_expectancy.js rename to core/reporting/renders/dashboard/static/sections/conditional_expectancy.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/drawdown_structure.js b/core/reporting/renders/dashboard/static/sections/drawdown_structure.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/drawdown_structure.js rename to core/reporting/renders/dashboard/static/sections/drawdown_structure.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/entry_exit_diagnostics.js b/core/reporting/renders/dashboard/static/sections/entry_exit_diagnostics.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/entry_exit_diagnostics.js rename to core/reporting/renders/dashboard/static/sections/entry_exit_diagnostics.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/equity_drawdown.js b/core/reporting/renders/dashboard/static/sections/equity_drawdown.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/equity_drawdown.js rename to core/reporting/renders/dashboard/static/sections/equity_drawdown.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/kpi.js b/core/reporting/renders/dashboard/static/sections/kpi.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/kpi.js rename to core/reporting/renders/dashboard/static/sections/kpi.js diff --git a/core/backtesting/reporting/renders/dashboard/static/sections/trade_distribution.js b/core/reporting/renders/dashboard/static/sections/trade_distribution.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/sections/trade_distribution.js rename to core/reporting/renders/dashboard/static/sections/trade_distribution.js diff --git a/core/backtesting/reporting/renders/dashboard/static/utils/format.js b/core/reporting/renders/dashboard/static/utils/format.js similarity index 100% rename from core/backtesting/reporting/renders/dashboard/static/utils/format.js rename to core/reporting/renders/dashboard/static/utils/format.js diff --git a/core/backtesting/reporting/renders/dashboard/templates/dashboard.html b/core/reporting/renders/dashboard/templates/dashboard.html similarity index 100% rename from core/backtesting/reporting/renders/dashboard/templates/dashboard.html rename to core/reporting/renders/dashboard/templates/dashboard.html diff --git a/core/backtesting/reporting/renders/plot_render.py b/core/reporting/renders/plot_render.py similarity index 100% rename from core/backtesting/reporting/renders/plot_render.py rename to core/reporting/renders/plot_render.py diff --git a/core/backtesting/reporting/renders/stdout.py b/core/reporting/renders/stdout.py similarity index 100% rename from core/backtesting/reporting/renders/stdout.py rename to core/reporting/renders/stdout.py diff --git a/core/reporting/reports/__init__.py b/core/reporting/reports/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/backtesting/reporting/reports/risk.py b/core/reporting/reports/risk.py similarity index 78% rename from core/backtesting/reporting/reports/risk.py rename to core/reporting/reports/risk.py index 13615e4..99c7e16 100644 --- a/core/backtesting/reporting/reports/risk.py +++ b/core/reporting/reports/risk.py @@ -1,7 +1,7 @@ -from core.backtesting.reporting.core.aggregration import ContextualAggregator -from core.backtesting.reporting.core.base import BaseReport -from core.backtesting.reporting.core.context import ReportContext -from core.backtesting.reporting.core.section import ReportSection +from core.reporting.core.aggregration import ContextualAggregator +from core.reporting.core.base import BaseReport +from core.reporting.core.context import ReportContext +from core.reporting.core.section import ReportSection class RiskMonitoringReport(BaseReport): diff --git a/core/reporting/runner.py b/core/reporting/runner.py new file mode 100644 index 0000000..995ccf0 --- /dev/null +++ b/core/reporting/runner.py @@ -0,0 +1,140 @@ +import pandas as pd + +from config.backtest import INITIAL_BALANCE, StdoutMode +from core.reporting.core.contex_enricher import TradeContextEnricher +from core.reporting.core.context import ReportContext +from core.reporting.core.equity import EquityPreparer +from core.reporting.core.formating import materialize +from core.reporting.core.persistence import ReportPersistence +from core.reporting.core.sections.backtest_config import BacktestConfigSection +from core.reporting.core.sections.capital_exposure import CapitalExposureSection +from core.reporting.core.sections.conditional_entry_tag import ConditionalEntryTagPerformanceSection +from core.reporting.core.sections.conditional_expectancy import ConditionalExpectancySection +from core.reporting.core.sections.drawdown_structure import DrawdownStructureSection +from core.reporting.core.sections.entry_tag_performance import EntryTagPerformanceSection +from core.reporting.core.sections.exit_logic_diagnostics import ExitLogicDiagnosticsSection +from core.reporting.core.sections.kpi import CorePerformanceSection +from core.reporting.core.sections.tail_risk import TailRiskSection +from core.reporting.core.sections.trade_distribution import TradeDistributionSection +from core.reporting.renders.dashboard.dashboard_renderer import DashboardRenderer +from core.reporting.renders.stdout import StdoutRenderer +from core.reporting.reports.risk import RiskReport + + +class ReportRunner: + """ + Executes analytics + report rendering + for ONE StrategyRunResult. + """ + + def __init__( + self, + *, + trades: pd.DataFrame, + df_context: pd.DataFrame, + report_spec, + metadata, + config, + report_config, + run_path, + ): + self.trades = trades + self.df_context = df_context + self.report_spec = report_spec + self.metadata = metadata + self.config = config + self.report_config = report_config + self.run_path = run_path + + self.renderer = None + + if report_config.stdout_mode == StdoutMode.CONSOLE: + self.renderer = StdoutRenderer() + elif report_config.stdout_mode == StdoutMode.OFF: + self.renderer = None + else: + raise NotImplementedError( + f"StdoutMode {report_config.stdout_mode} not supported" + ) + + # ================================================== + # MAIN + # ================================================== + + def run(self): + # ----------------------------- + # ANALYTICS + # ----------------------------- + + preparer = EquityPreparer( + initial_balance=self.config.INITIAL_BALANCE + ) + + trades = preparer.prepare(self.trades) + + enricher = TradeContextEnricher(self.df_context) + trades = enricher.enrich( + trades, + self.report_spec.contexts + ) + + ctx = ReportContext( + trades=trades, + equity=trades["equity"], + drawdown=trades["drawdown"], + df_plot=self.df_context, + initial_balance=self.config.INITIAL_BALANCE, + config=self.config, + metadata=self.metadata, + ) + + # ----------------------------- + # REPORT COMPUTATION + # ----------------------------- + + report = RiskReport( + sections=[ + BacktestConfigSection(), + CorePerformanceSection(), + TradeDistributionSection(), + TailRiskSection(), + ConditionalExpectancySection(), + EntryTagPerformanceSection(), + ConditionalEntryTagPerformanceSection(), + ExitLogicDiagnosticsSection(), + DrawdownStructureSection(), + CapitalExposureSection(), + ] + ) + + data = materialize(report.compute(ctx)) + + # ----------------------------- + # STDOUT + # ----------------------------- + + if self.renderer is not None: + self.renderer.render(data) + + # ----------------------------- + # PERSISTENCE + # ----------------------------- + + if self.report_config.persist_report: + ReportPersistence( + base_path=self.run_path / "report" / self.metadata.run_id + ).persist( + trades=ctx.trades, + equity=ctx.equity, + report_data=data, + meta=self.metadata.to_dict(), + ) + + # ----------------------------- + # DASHBOARD + # ----------------------------- + + if self.report_config.generate_dashboard: + DashboardRenderer( + run_path=self.run_path / "dashboard" / self.metadata.run_id + ).render(data, ctx) \ No newline at end of file diff --git a/core/reporting/summary_runner.py b/core/reporting/summary_runner.py new file mode 100644 index 0000000..249bbc0 --- /dev/null +++ b/core/reporting/summary_runner.py @@ -0,0 +1,117 @@ +import json + +import pandas as pd + +from core.reporting.core.contex_enricher import TradeContextEnricher +from core.reporting.core.context import ReportContext +from core.reporting.core.equity import EquityPreparer +from core.reporting.core.formating import materialize +from core.reporting.core.sections.backtest_config import BacktestConfigSection +from core.reporting.core.sections.capital_exposure import CapitalExposureSection +from core.reporting.core.sections.conditional_entry_tag import ConditionalEntryTagPerformanceSection +from core.reporting.core.sections.conditional_expectancy import ConditionalExpectancySection +from core.reporting.core.sections.drawdown_structure import DrawdownStructureSection +from core.reporting.core.sections.entry_tag_performance import EntryTagPerformanceSection +from core.reporting.core.sections.exit_logic_diagnostics import ExitLogicDiagnosticsSection +from core.reporting.core.sections.kpi import CorePerformanceSection +from core.reporting.core.sections.tail_risk import TailRiskSection +from core.reporting.core.sections.trade_distribution import TradeDistributionSection +from core.reporting.renders.dashboard.dashboard_renderer import DashboardRenderer +from core.reporting.renders.stdout import StdoutRenderer +from core.reporting.reports.risk import RiskReport + + +class SummaryReportRunner: + """ + Portfolio / multi-run summary report. + SINGLE stdout. SINGLE dashboard. + """ + + def __init__( + self, + *, + strategy_runs, + trades_by_run, + config, + run_path, + ): + self.strategy_runs = strategy_runs + self.trades_by_run = trades_by_run + self.config = config + self.run_path = run_path + + self.stdout_renderer = StdoutRenderer() + + def run(self): + # ================================================== + # AGGREGATE TRADES + # ================================================== + + trades_all = [] + for run, trades in zip(self.strategy_runs, self.trades_by_run): + df = trades.copy() + df["symbol"] = run.symbol + df["strategy_id"] = run.strategy_id + trades_all.append(df) + + trades_all = pd.concat(trades_all).reset_index(drop=True) + + # ================================================== + # PORTFOLIO EQUITY + # ================================================== + + preparer = EquityPreparer( + initial_balance=self.config.INITIAL_BALANCE + ) + + trades_all = preparer.prepare(trades_all) + + # ================================================== + # REPORT CONTEXT (PORTFOLIO) + # ================================================== + + ctx = ReportContext( + trades=trades_all, + equity=trades_all["equity"], + drawdown=trades_all["drawdown"], + df_plot=None, + initial_balance=self.config.INITIAL_BALANCE, + config=self.config, + metadata={ + "scope": "portfolio", + "symbols": sorted(trades_all["symbol"].unique().tolist()), + "strategies": sorted(trades_all["strategy_id"].unique().tolist()), + }, + ) + + # ================================================== + # REPORT + # ================================================== + + report = RiskReport( + sections=[ + CorePerformanceSection(), + DrawdownStructureSection(), + CapitalExposureSection(), + ] + ) + + data = materialize(report.compute(ctx)) + + # ================================================== + # STDOUT (SINGLE) + # ================================================== + + self.stdout_renderer.render(data) + + # ================================================== + # PERSIST + DASHBOARD + # ================================================== + + out = self.run_path / "summary" + out.mkdir(parents=True, exist_ok=True) + + with open(out / "report.json", "w") as f: + json.dump(data, f, indent=2, default=str) + + DashboardRenderer(run_path=out).render(data, ctx) \ No newline at end of file diff --git a/core/strategy/base.py b/core/strategy/base.py index 45c97d2..522fc65 100644 --- a/core/strategy/base.py +++ b/core/strategy/base.py @@ -1,12 +1,14 @@ from __future__ import annotations +import hashlib +import json from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional import pandas as pd -from core.backtesting.reporting.config.report_config import ReportConfig -from core.backtesting.reporting.core.metrics import ExpectancyMetric, MaxDrawdownMetric +from core.reporting.config.report_spec import StrategyReportSpec +from core.reporting.core.metrics import ExpectancyMetric, MaxDrawdownMetric from core.strategy.plan_builder import PlanBuildContext, build_trade_plan_from_row, build_plans_frame from core.strategy.trade_plan import TradePlan, TradeAction @@ -33,16 +35,32 @@ def __init__( symbol: str, strategy_config: Optional[Dict[str, Any]] = None, startup_candle_count: int = 0, + strategy_name: str = None ): self.df = df self.symbol = symbol self.strategy_config = strategy_config or {} self.startup_candle_count = startup_candle_count + self.strategy_name = strategy_name # ================================================== # Informatives (DECLARATION ONLY) # ================================================== + def get_strategy_name(self) -> str: + return self.strategy_name or type(self).__name__ + + def get_strategy_id(self) -> str: + """ + Stable strategy identifier based on config. + """ + raw = { + "class": type(self).__name__, + "config": self.strategy_config, + } + payload = json.dumps(raw, sort_keys=True) + return hashlib.md5(payload.encode()).hexdigest()[:8] + @classmethod def get_required_informatives(cls) -> List[str]: tfs = set() @@ -113,9 +131,9 @@ def validate(self) -> None: if "time" not in self.df.columns: raise ValueError("Strategy DF must contain 'time' column") - def build_report_config(self): + def build_report_spec(self): return ( - ReportConfig() + StrategyReportSpec() .add_metric(ExpectancyMetric()) .add_metric(MaxDrawdownMetric()) ) diff --git a/core/strategy/orchestration/informatives.py b/core/strategy/orchestration/informatives.py index 459381e..8706001 100644 --- a/core/strategy/orchestration/informatives.py +++ b/core/strategy/orchestration/informatives.py @@ -10,28 +10,23 @@ def apply_informatives( *, df: pd.DataFrame, strategy, - provider, - symbol: str, - startup_candle_count: int, + data_by_tf: dict[str, pd.DataFrame], ) -> pd.DataFrame: """ Apply strategy informatives (HTF data) to main dataframe. - Shared orchestration used by BOTH backtest and live runners. - Strategy only declares informatives; this function executes them. + NO IO. + Uses preloaded data_by_tf. """ - if provider is None: - return df - if "time" not in df.columns: raise ValueError("Main dataframe must contain 'time' column") # ================================================== - # 1️⃣ Collect informative methods from strategy + # 1️⃣ Collect informative methods # ================================================== - informatives: Dict[str, list] = defaultdict(list) + informatives: dict[str, list] = defaultdict(list) for attr in dir(strategy): fn = getattr(strategy, attr) @@ -42,57 +37,46 @@ def apply_informatives( if not informatives: return df - informative_results: Dict[str, pd.DataFrame] = {} + out = df.copy() - for tf in informatives: - df_tf = provider.get_informative_df( - symbol=symbol, - timeframe=tf, - startup_candle_count=startup_candle_count, - ) + # ================================================== + # 2️⃣ Apply informatives per TF + # ================================================== - if "time" not in df_tf.columns: + for tf, methods in informatives.items(): + if tf not in data_by_tf: raise RuntimeError( - f"Informative DF for TF={tf} has no 'time'. " - f"Columns={list(df_tf.columns)}" + f"Strategy requires informative TF={tf} " + f"but it was not preloaded" ) - informative_results[tf] = df_tf + df_tf = data_by_tf[tf].copy() - for tf, methods in informatives.items(): - df_tf = informative_results[tf] + if "time" not in df_tf.columns: + raise RuntimeError( + f"Informative DF for TF={tf} has no 'time' column" + ) + + # apply strategy methods for method in methods: df_tf = method(df_tf) - informative_results[tf] = df_tf - - out = df.copy() - for tf, df_tf in informative_results.items(): + # merge-asof suffix = f"_{tf}" - cols_to_drop = [c for c in out.columns if c.endswith(suffix)] - if cols_to_drop: - out = out.drop(columns=cols_to_drop) - df_tf_prefixed = df_tf.rename( - columns={c: f"{c}_{tf}" for c in df_tf.columns if c != "time"} - ).copy() - - df_tf_prefixed[f"time_{tf}"] = df_tf["time"] + columns={ + c: f"{c}{suffix}" + for c in df_tf.columns + if c != "time" + } + ) out = pd.merge_asof( out.sort_values("time"), - df_tf_prefixed.sort_values(f"time_{tf}"), - left_on="time", - right_on=f"time_{tf}", + df_tf_prefixed.sort_values("time"), + on="time", direction="backward", ) - if "time_x" in out.columns: - out = out.rename(columns={"time_x": "time"}) - if "time_y" in out.columns: - out = out.drop(columns=["time_y"]) - - out = out.drop(columns=[f"time_{tf}"]) - - return out + return out \ No newline at end of file diff --git a/core/strategy/orchestration/strategy_execution.py b/core/strategy/orchestration/strategy_execution.py index 8a44c00..835d39f 100644 --- a/core/strategy/orchestration/strategy_execution.py +++ b/core/strategy/orchestration/strategy_execution.py @@ -6,9 +6,7 @@ def execute_strategy( *, strategy, df: pd.DataFrame, - provider, - symbol: str, - startup_candle_count: int, + data_by_tf: dict[str, pd.DataFrame], ) -> pd.DataFrame: """ Shared strategy execution pipeline. @@ -18,9 +16,7 @@ def execute_strategy( df = apply_informatives( df=df, strategy=strategy, - provider=provider, - symbol=symbol, - startup_candle_count=startup_candle_count, + data_by_tf=data_by_tf, ) strategy.df = df diff --git a/core/strategy/tests/test_apply_informatives.py b/core/strategy/tests/test_apply_informatives.py index 5b894dc..bfb7841 100644 --- a/core/strategy/tests/test_apply_informatives.py +++ b/core/strategy/tests/test_apply_informatives.py @@ -26,9 +26,7 @@ def test_apply_informatives_merges_htf(): out = apply_informatives( df=df, strategy=strat, - provider=DummyProvider(), - symbol="XAUUSD", - startup_candle_count=10, + data_by_tf={"M30":df}, ) assert any(c.endswith("_M30") for c in out.columns) diff --git a/core/strategy/tests/test_execute_strategy.py b/core/strategy/tests/test_execute_strategy.py index 80f74c8..e0ea116 100644 --- a/core/strategy/tests/test_execute_strategy.py +++ b/core/strategy/tests/test_execute_strategy.py @@ -25,9 +25,8 @@ def test_execute_strategy_pipeline(): out = execute_strategy( strategy=strat, df=df, - provider=DummyProvider(), - symbol="XAUUSD", - startup_candle_count=10, + data_by_tf={"M30": df}, + ) assert "signal_entry" in out.columns diff --git a/core/strategy/tests/test_strategy_contract.py b/core/strategy/tests/test_strategy_contract.py index b948db3..7502635 100644 --- a/core/strategy/tests/test_strategy_contract.py +++ b/core/strategy/tests/test_strategy_contract.py @@ -27,11 +27,10 @@ def test_strategy_domain_execution_with_informatives(): df_with_htf = apply_informatives( df=df, strategy=strategy, - provider=DummyProvider(), - symbol="XAUUSD", - startup_candle_count=10, + data_by_tf={"M30": df}, ) + strategy.df = df_with_htf strategy.validate() diff --git a/core/strategy/tests/test_strategy_signals.py b/core/strategy/tests/test_strategy_signals.py index c6fa742..31995c9 100644 --- a/core/strategy/tests/test_strategy_signals.py +++ b/core/strategy/tests/test_strategy_signals.py @@ -30,9 +30,7 @@ def test_strategy_does_not_generate_entries_when_conditions_not_met(): df_with_htf = apply_informatives( df=df, strategy=strategy, - provider=DummyProvider(), - symbol="XAUUSD", - startup_candle_count=10, + data_by_tf = {"M30": df}, ) strategy.df = df_with_htf diff --git a/core/utils/timeframe.py b/core/utils/timeframe.py index 0886d50..502a231 100644 --- a/core/utils/timeframe.py +++ b/core/utils/timeframe.py @@ -30,3 +30,21 @@ def timeframe_to_pandas_freq(timeframe: str) -> str: return f"{int(tf[1:])}D" raise ValueError(f"Unsupported timeframe: {timeframe}") + + +def tf_to_minutes(tf: str) -> int: + tf = tf.strip().upper() + + if tf.startswith("M"): + return int(tf[1:]) + + if tf.startswith("H"): + return int(tf[1:]) * 60 + + if tf.startswith("D"): + return int(tf[1:]) * 1440 + + if tf.startswith("W"): + return int(tf[1:]) * 10080 + + raise ValueError(f"Unsupported timeframe: {tf}") \ No newline at end of file diff --git a/dashboard_output/dashboard.html b/dashboard_output/dashboard.html index 5042831..e8881d7 100644 --- a/dashboard_output/dashboard.html +++ b/dashboard_output/dashboard.html @@ -13,7 +13,7 @@
diff --git a/live_state/active_trades.json b/live_state/active_trades.json index b982086..9e26dfe 100644 --- a/live_state/active_trades.json +++ b/live_state/active_trades.json @@ -1,18 +1 @@ -{ - "383653031": { - "trade_id": 383653031, - "symbol": "EURUSD", - "direction": "short", - "entry_price": 1.18151, - "volume": 0.0, - "sl": 1.182, - "tp1": 1.1810200000000002, - "tp2": 1.1805300000000003, - "tp1_executed": false, - "entry_time": "2026-02-04T16:58:00+00:00", - "entry_tag": "short", - "strategy": "Samplestrategy", - "strategy_config": {}, - "ticket": 383653031 - } -} \ No newline at end of file +{} \ No newline at end of file diff --git a/live_state/closed_trades.json b/live_state/closed_trades.json index a3b0875..c16f261 100644 --- a/live_state/closed_trades.json +++ b/live_state/closed_trades.json @@ -78,5 +78,25 @@ "exit_time": "2026-02-04 14:24:00+00:00", "exit_reason": "BROKER_CLOSED", "exit_level_tag": "TP2_live" + }, + "383653031": { + "trade_id": 383653031, + "symbol": "EURUSD", + "direction": "short", + "entry_price": 1.18151, + "volume": 0.0, + "sl": 1.182, + "tp1": 1.1810200000000002, + "tp2": 1.1805300000000003, + "tp1_executed": false, + "entry_time": "2026-02-04T16:58:00+00:00", + "entry_tag": "short", + "strategy": "Samplestrategy", + "strategy_config": {}, + "ticket": 383653031, + "exit_price": 1.1805300000000003, + "exit_time": "2026-02-05 19:35:00+00:00", + "exit_reason": "BROKER_CLOSED", + "exit_level_tag": "TP2_live" } } \ No newline at end of file