From 9a3b835c951177b099a929e1f00437a5ab8a3724 Mon Sep 17 00:00:00 2001 From: Bu5hm4nn Date: Sun, 29 Mar 2026 11:12:11 +0200 Subject: [PATCH] feat(DATA-DB-004): add Databento settings UI and independent scenario config - Updated backtests page with Data Source card - Data source selector (databento/yfinance/synthetic) - Dataset dropdown (XNAS.BASIC, GLBX.MDP3) - Resolution dropdown (ohlcv-1d, ohlcv-1h) - Cost estimate display (placeholder for now) - Added Scenario Configuration card - Underlying symbol selector (GLD/GC/XAU) - Start/end date inputs - Start price input (0 = auto-derive) - Underlying units, loan amount, margin call LTV - BacktestPageService updates: - get_historical_prices() with data_source parameter - get_cost_estimate() for Databento cost estimation - get_cache_stats() for cache status display - Support for injected custom provider identity - DataSourceInfo for provider metadata - BacktestSettingsRepository integration: - Load/save settings per workspace - Default values from BacktestSettings.create_default() - Test update: TLT validation message changed to reflect new multi-symbol support (GLD, GC, XAU) --- .../backtest_settings.json | 23 + .../backtest_settings.json | 23 + .../backtest_settings.json | 23 + app/pages/backtests.py | 458 +++++++++++++++--- app/services/backtesting/ui_service.py | 200 +++++++- tests/test_backtest_ui.py | 2 +- 6 files changed, 666 insertions(+), 63 deletions(-) create mode 100644 .workspaces/83207143-b45c-4288-a1d2-c1363a54a020/backtest_settings.json create mode 100644 .workspaces/b02f124a-9573-4259-8291-13516c8e4cb9/backtest_settings.json create mode 100644 .workspaces/ea5841e7-36e5-4faf-8f90-77c3a8fd6311/backtest_settings.json diff --git a/.workspaces/83207143-b45c-4288-a1d2-c1363a54a020/backtest_settings.json b/.workspaces/83207143-b45c-4288-a1d2-c1363a54a020/backtest_settings.json new file mode 100644 index 0000000..2e69dd3 --- /dev/null +++ b/.workspaces/83207143-b45c-4288-a1d2-c1363a54a020/backtest_settings.json @@ -0,0 +1,23 @@ +{ + "settings_id": "3e5143f6-29da-4416-8fca-edeaaac986ae", + "name": "Backtest 2020-01-01 - 2023-12-31", + "data_source": "databento", + "dataset": "XNAS.BASIC", + "schema": "ohlcv-1d", + "start_date": "2020-01-01", + "end_date": "2023-12-31", + "underlying_symbol": "GLD", + "start_price": 0.0, + "underlying_units": 1000.0, + "loan_amount": 0.0, + "margin_call_ltv": 0.75, + "template_slugs": [ + "protective-put-atm-12m" + ], + "cache_key": "", + "data_cost_usd": 0.0, + "provider_ref": { + "provider_id": "synthetic_v1", + "pricing_mode": "synthetic_bs_mid" + } +} \ No newline at end of file diff --git a/.workspaces/b02f124a-9573-4259-8291-13516c8e4cb9/backtest_settings.json b/.workspaces/b02f124a-9573-4259-8291-13516c8e4cb9/backtest_settings.json new file mode 100644 index 0000000..6da6f96 --- /dev/null +++ b/.workspaces/b02f124a-9573-4259-8291-13516c8e4cb9/backtest_settings.json @@ -0,0 +1,23 @@ +{ + "settings_id": "a48fe9fe-90d0-4cfc-a78f-a8db01cbf4d4", + "name": "Backtest 2020-01-01 - 2023-12-31", + "data_source": "databento", + "dataset": "XNAS.BASIC", + "schema": "ohlcv-1d", + "start_date": "2020-01-01", + "end_date": "2023-12-31", + "underlying_symbol": "GLD", + "start_price": 0.0, + "underlying_units": 1000.0, + "loan_amount": 0.0, + "margin_call_ltv": 0.75, + "template_slugs": [ + "protective-put-atm-12m" + ], + "cache_key": "", + "data_cost_usd": 0.0, + "provider_ref": { + "provider_id": "synthetic_v1", + "pricing_mode": "synthetic_bs_mid" + } +} \ No newline at end of file diff --git a/.workspaces/ea5841e7-36e5-4faf-8f90-77c3a8fd6311/backtest_settings.json b/.workspaces/ea5841e7-36e5-4faf-8f90-77c3a8fd6311/backtest_settings.json new file mode 100644 index 0000000..63816b0 --- /dev/null +++ b/.workspaces/ea5841e7-36e5-4faf-8f90-77c3a8fd6311/backtest_settings.json @@ -0,0 +1,23 @@ +{ + "settings_id": "23d8dd8b-1798-45c7-855f-415c04355477", + "name": "Backtest 2020-01-01 - 2023-12-31", + "data_source": "databento", + "dataset": "XNAS.BASIC", + "schema": "ohlcv-1d", + "start_date": "2020-01-01", + "end_date": "2023-12-31", + "underlying_symbol": "GLD", + "start_price": 0.0, + "underlying_units": 1000.0, + "loan_amount": 0.0, + "margin_call_ltv": 0.75, + "template_slugs": [ + "protective-put-atm-12m" + ], + "cache_key": "", + "data_cost_usd": 0.0, + "provider_ref": { + "provider_id": "synthetic_v1", + "pricing_mode": "synthetic_bs_mid" + } +} \ No newline at end of file diff --git a/app/pages/backtests.py b/app/pages/backtests.py index ff55e5f..1674540 100644 --- a/app/pages/backtests.py +++ b/app/pages/backtests.py @@ -2,17 +2,45 @@ from __future__ import annotations import logging from datetime import date, datetime +from typing import Any from fastapi.responses import RedirectResponse from nicegui import ui from app.domain.backtesting_math import asset_quantity_from_workspace_config +from app.models.backtest import ProviderRef +from app.models.backtest_settings import BacktestSettings +from app.models.backtest_settings_repository import BacktestSettingsRepository from app.models.workspace import get_workspace_repository from app.pages.common import dashboard_page, split_page_panes +from app.services.backtesting.databento_source import DatabentoHistoricalPriceSource from app.services.backtesting.ui_service import BacktestPageRunResult, BacktestPageService logger = logging.getLogger(__name__) +# Dataset and schema options for Databento +DATABENTO_DATASETS = { + "XNAS.BASIC": "XNAS.BASIC (GLD ETF)", + "GLBX.MDP3": "GLBX.MDP3 (GC=F Futures)", +} + +DATABENTO_SCHEMAS = { + "ohlcv-1d": "Daily bars (ohlcv-1d)", + "ohlcv-1h": "Hourly bars (ohlcv-1h)", +} + +UNDERLYING_SYMBOLS = { + "GLD": "GLD (Gold ETF)", + "GC": "GC (Gold Futures)", + "XAU": "XAU (Gold Index)", +} + +DATA_SOURCES = { + "databento": "Databento", + "yfinance": "Yahoo Finance", + "synthetic": "Synthetic", +} + def _chart_options(result: BacktestPageRunResult) -> dict: template_result = result.run_result.template_results[0] @@ -52,6 +80,65 @@ def _chart_options(result: BacktestPageRunResult) -> dict: } +def _get_databento_cost_estimate(symbol: str, start_date: date, end_date: date) -> tuple[float, str]: + """Get cost estimate from Databento API. + + Returns: + Tuple of (cost_usd, error_message). If successful, error_message is empty. + """ + try: + source = DatabentoHistoricalPriceSource() + cost = source.get_cost_estimate(symbol, start_date, end_date) + return cost, "" + except Exception as e: + logger.warning(f"Failed to get Databento cost estimate: {e}") + return 0.0, str(e) + + +def _get_databento_cache_status(symbol: str, start_date: date, end_date: date) -> tuple[str, str]: + """Get cache status for Databento data. + + Returns: + Tuple of (status_text, status_class). status_class is one of: + 'text-slate-500', 'text-emerald-600', 'text-amber-600', 'text-rose-600' + """ + try: + import hashlib + import json + + from app.services.backtesting.databento_source import DatabentoSourceConfig + + config = DatabentoSourceConfig() + dataset = DatabentoHistoricalPriceSource.__new__(DatabentoHistoricalPriceSource)._resolve_dataset(symbol) + databento_symbol = DatabentoHistoricalPriceSource.__new__(DatabentoHistoricalPriceSource)._resolve_symbol( + symbol + ) + + key_str = f"{dataset}_{databento_symbol}_ohlcv-1d_{start_date}_{end_date}" + key_hash = hashlib.sha256(key_str.encode()).hexdigest()[:16] + meta_file = config.cache_dir / f"dbn_{key_hash}_meta.json" + + if not meta_file.exists(): + return "No cached data", "text-slate-500" + + with open(meta_file) as f: + meta = json.load(f) + + download_date = date.fromisoformat(meta["download_date"]) + age_days = (date.today() - download_date).days + + if age_days == 0: + return f"Cached today ({meta.get('rows', 0)} rows)", "text-emerald-600" + elif age_days <= config.max_cache_age_days: + return f"Cached {age_days} days ago ({meta.get('rows', 0)} rows)", "text-emerald-600" + else: + return f"Cache stale ({age_days} days old)", "text-amber-600" + + except Exception as e: + logger.warning(f"Failed to get cache status: {e}") + return f"Status unavailable: {str(e)[:30]}", "text-rose-600" + + @ui.page("/{workspace_id}/backtests") def workspace_backtests_page(workspace_id: str) -> None: repo = get_workspace_repository() @@ -62,23 +149,56 @@ def workspace_backtests_page(workspace_id: str) -> None: def _render_backtests_page(workspace_id: str | None = None) -> None: service = BacktestPageService() + settings_repo = BacktestSettingsRepository() + + # Load saved settings if available + saved_settings: BacktestSettings | None = None + if workspace_id: + try: + saved_settings = settings_repo.load(workspace_id) + except Exception as e: + logger.warning(f"Failed to load backtest settings for workspace {workspace_id}: {e}") + template_options = service.template_options("GLD") select_options = {str(option["slug"]): str(option["label"]) for option in template_options} default_template_slug = str(template_options[0]["slug"]) if template_options else None + repo = get_workspace_repository() config = repo.load_portfolio_config(workspace_id) if workspace_id else None - default_entry_spot = service.derive_entry_spot("GLD", date(2024, 1, 2), date(2024, 1, 8)) - default_units = ( - asset_quantity_from_workspace_config(config, entry_spot=default_entry_spot, symbol="GLD") - if config is not None and default_entry_spot > 0 - else 1000.0 - ) - default_loan = float(config.loan_amount) if config else 68000.0 - default_margin_call_ltv = float(config.margin_threshold) if config else 0.75 + + # Initialize defaults from saved settings or workspace config + if saved_settings: + default_data_source = saved_settings.data_source + default_dataset = saved_settings.dataset + default_schema = saved_settings.schema + default_start_date = saved_settings.start_date.isoformat() + default_end_date = saved_settings.end_date.isoformat() + default_symbol = saved_settings.underlying_symbol + default_start_price = saved_settings.start_price + default_units = saved_settings.underlying_units + default_loan = saved_settings.loan_amount + default_margin_call_ltv = saved_settings.margin_call_ltv + else: + default_data_source = "databento" + default_dataset = "XNAS.BASIC" + default_schema = "ohlcv-1d" + default_start_date = "2024-01-02" + default_end_date = "2024-01-08" + default_symbol = "GLD" + default_start_price = 0.0 + + default_entry_spot = service.derive_entry_spot("GLD", date(2024, 1, 2), date(2024, 1, 8)) + default_units = ( + asset_quantity_from_workspace_config(config, entry_spot=default_entry_spot, symbol="GLD") + if config is not None and default_entry_spot > 0 + else 1000.0 + ) + default_loan = float(config.loan_amount) if config else 68000.0 + default_margin_call_ltv = float(config.margin_threshold) if config else 0.75 with dashboard_page( "Backtests", - "Run a thin read-only historical scenario over the BT-001 synthetic engine and inspect the daily path.", + "Run historical backtests with configurable data sources and scenario parameters.", "backtests", workspace_id=workspace_id, ): @@ -88,40 +208,138 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: ) with left_pane: + # Data Source Configuration Card with ui.card().classes( "w-full rounded-2xl border border-slate-200 bg-white shadow-sm dark:border-slate-800 dark:bg-slate-900" ): - ui.label("Scenario Form").classes("text-lg font-semibold text-slate-900 dark:text-slate-100") + ui.label("Data Source").classes("text-lg font-semibold text-slate-900 dark:text-slate-100") ui.label( - "Entry spot is auto-derived from the first historical close in the selected window so the scenario stays consistent with BT-001 entry timing." + "Select the data provider for historical price data. Databento provides institutional-grade data with usage-based pricing." ).classes("text-sm text-slate-500 dark:text-slate-400") - ui.label("BT-001A currently supports GLD only for this thin read-only page.").classes( - "text-sm text-slate-500 dark:text-slate-400" + + data_source_select = ( + ui.select(DATA_SOURCES, value=default_data_source, label="Data source") + .classes("w-full") + .props("data-testid=data-source-select") ) - if workspace_id: - ui.label("Workspace defaults seed underlying units, loan amount, and margin threshold.").classes( - "text-sm text-slate-500 dark:text-slate-400" + + # Databento-specific options (shown conditionally) + databento_options_card = ui.card().classes( + "w-full rounded-xl border border-slate-200 bg-slate-50 p-4 dark:border-slate-700 dark:bg-slate-800" + ) + + with databento_options_card: + dataset_select = ( + ui.select(DATABENTO_DATASETS, value=default_dataset, label="Dataset") + .classes("w-full") + .props("data-testid=dataset-select") ) - symbol_input = ui.input("Symbol", value="GLD").props("readonly").classes("w-full") - start_input = ui.input("Start date", value="2024-01-02").classes("w-full") - end_input = ui.input("End date", value="2024-01-08").classes("w-full") - template_select = ui.select(select_options, value=default_template_slug, label="Template").classes( - "w-full" + schema_select = ( + ui.select(DATABENTO_SCHEMAS, value=default_schema, label="Resolution") + .classes("w-full") + .props("data-testid=schema-select") + ) + + # Cost estimate display + cost_estimate_label = ui.label("").classes("text-sm text-slate-500 dark:text-slate-400") + cost_estimate_label.set_visibility(False) + + # Cache status display + cache_status_label = ui.label("").classes("text-sm") + + # Show/hide Databento options based on data source + # Placeholder for cost estimate callback - will be defined later + _update_cost_estimate_callback: list[Any] = [None] + + def update_databento_visibility() -> None: + is_databento = str(data_source_select.value) == "databento" + databento_options_card.set_visibility(is_databento) + if is_databento and _update_cost_estimate_callback[0]: + _update_cost_estimate_callback[0]() + + data_source_select.on_value_change(lambda e: update_databento_visibility()) + + # Initialize visibility + update_databento_visibility() + + # Scenario Configuration Card (Independent of Portfolio) + with ui.card().classes( + "w-full rounded-2xl border border-slate-200 bg-white shadow-sm dark:border-slate-800 dark:bg-slate-900" + ): + ui.label("Scenario Configuration").classes("text-lg font-semibold text-slate-900 dark:text-slate-100") + ui.label( + "Configure backtest scenario parameters independently of Portfolio settings. " + "Start price = 0 auto-derives from first historical close." + ).classes("text-sm text-slate-500 dark:text-slate-400") + + symbol_select = ( + ui.select(UNDERLYING_SYMBOLS, value=default_symbol, label="Underlying symbol") + .classes("w-full") + .props("data-testid=symbol-select") ) - units_input = ui.number("Underlying units", value=default_units, min=0.0001, step=1).classes("w-full") - loan_input = ui.number("Loan amount", value=default_loan, min=0, step=1000).classes("w-full") - ltv_input = ui.number( - "Margin call LTV", - value=default_margin_call_ltv, - min=0.01, - max=0.99, - step=0.01, - ).classes("w-full") - entry_spot_hint = ui.label("Entry spot will be auto-derived on run.").classes( + + with ui.row().classes("w-full gap-4"): + with ui.column().classes("flex-1"): + start_input = ( + ui.input("Start date", value=default_start_date) + .classes("w-full") + .props("data-testid=start-date-input") + ) + with ui.column().classes("flex-1"): + end_input = ( + ui.input("End date", value=default_end_date) + .classes("w-full") + .props("data-testid=end-date-input") + ) + + start_price_input = ( + ui.number("Start price (0 = auto-derive)", value=default_start_price, min=0, step=0.01) + .classes("w-full") + .props("data-testid=start-price-input") + ) + + units_input = ( + ui.number("Underlying units", value=default_units, min=0.0001, step=1) + .classes("w-full") + .props("data-testid=units-input") + ) + + loan_input = ( + ui.number("Loan amount", value=default_loan, min=0, step=1000) + .classes("w-full") + .props("data-testid=loan-input") + ) + + ltv_input = ( + ui.number("Margin call LTV", value=default_margin_call_ltv, min=0.01, max=0.99, step=0.01) + .classes("w-full") + .props("data-testid=ltv-input") + ) + + # Template Selection Card + with ui.card().classes( + "w-full rounded-2xl border border-slate-200 bg-white shadow-sm dark:border-slate-800 dark:bg-slate-900" + ): + ui.label("Template Selection").classes("text-lg font-semibold text-slate-900 dark:text-slate-100") + ui.label("Select the hedging strategy template to apply during the backtest.").classes( "text-sm text-slate-500 dark:text-slate-400" ) + + template_select = ( + ui.select(select_options, value=default_template_slug, label="Template") + .classes("w-full") + .props("data-testid=template-select") + ) + + # Entry Spot and Validation Card + with ui.card().classes( + "w-full rounded-2xl border border-slate-200 bg-white shadow-sm dark:border-slate-800 dark:bg-slate-900" + ): + entry_spot_hint = ui.label("").classes("text-sm text-slate-500 dark:text-slate-400") validation_label = ui.label("").classes("text-sm text-rose-600 dark:text-rose-300") - run_button = ui.button("Run backtest").props("color=primary") + run_button = ( + ui.button("Run backtest").props("color=primary data-testid=run-backtest-button").classes("mt-2") + ) seeded_summary = ui.card().classes( "w-full rounded-2xl border border-slate-200 bg-white shadow-sm dark:border-slate-800 dark:bg-slate-900" @@ -136,10 +354,66 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: except ValueError as exc: raise ValueError(f"{field_name} must be in YYYY-MM-DD format") from exc - def derive_entry_spot() -> tuple[float | None, str | None]: + def get_symbol_from_dataset() -> str: + """Map dataset selection to underlying symbol.""" + dataset = str(dataset_select.value) + if dataset == "GLBX.MDP3": + return "GC" + return "GLD" # Default for XNAS.BASIC + + def update_cost_estimate() -> None: + """Update cost estimate display based on current settings.""" + current_data_source = str(data_source_select.value) + if current_data_source != "databento": + cost_estimate_label.set_visibility(False) + cache_status_label.set_visibility(False) + return + + # Set the callback for databento visibility updates + _update_cost_estimate_callback[0] = update_cost_estimate + try: + start = parse_iso_date(start_input.value, "Start date") + end = parse_iso_date(end_input.value, "End date") + symbol = get_symbol_from_dataset() + + cost, error = _get_databento_cost_estimate(symbol, start, end) + if error: + cost_estimate_label.set_text(f"⚠️ Cost estimate unavailable: {error[:50]}") + cost_estimate_label.classes("text-sm text-amber-600 dark:text-amber-400", remove="") + elif cost > 0: + cost_estimate_label.set_text(f"💰 Estimated cost: ${cost:.4f}") + cost_estimate_label.classes("text-sm text-slate-600 dark:text-slate-400", remove="") + else: + cost_estimate_label.set_text("💰 Cost: Free (within usage allowance)") + cost_estimate_label.classes("text-sm text-emerald-600 dark:text-emerald-400", remove="") + cost_estimate_label.set_visibility(True) + + # Update cache status + status_text, status_class = _get_databento_cache_status(symbol, start, end) + cache_status_label.set_text(f"📦 {status_text}") + cache_status_label.classes( + status_class, remove="text-slate-500 text-emerald-600 text-amber-600 text-rose-600" + ) + cache_status_label.set_visibility(True) + + except ValueError: + cost_estimate_label.set_text("⚠️ Enter valid dates for cost estimate") + cost_estimate_label.classes("text-sm text-amber-600 dark:text-amber-400", remove="") + cost_estimate_label.set_visibility(True) + cache_status_label.set_visibility(False) + + def derive_entry_spot() -> tuple[float | None, str | None]: + """Derive entry spot from historical data or use configured start price.""" + configured_start_price = float(start_price_input.value or 0.0) + if configured_start_price > 0: + return configured_start_price, None + + try: + # Use the symbol from the dataset selection + symbol = get_symbol_from_dataset() resolved_entry_spot = service.derive_entry_spot( - str(symbol_input.value or "GLD"), + symbol, parse_iso_date(start_input.value, "Start date"), parse_iso_date(end_input.value, "End date"), ) @@ -157,10 +431,15 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: ui.label("Scenario Summary").classes("text-lg font-semibold text-slate-900 dark:text-slate-100") with ui.grid(columns=1).classes("w-full gap-4 sm:grid-cols-2 lg:grid-cols-1 xl:grid-cols-2"): cards = [ + ("Data source", DATA_SOURCES.get(str(data_source_select.value), str(data_source_select.value))), ("Template", select_options.get(str(template_select.value), str(template_select.value or "—"))), ("Underlying units", f"{float(units_input.value or 0.0):,.0f}"), ("Loan amount", f"${float(loan_input.value or 0.0):,.0f}"), ("Margin call LTV", f"{float(ltv_input.value or 0.0):.1%}"), + ( + "Underlying symbol", + UNDERLYING_SYMBOLS.get(str(symbol_select.value), str(symbol_select.value)), + ), ( "Date range", f"{str(start_input.value or '—')} → {str(end_input.value or '—')}", @@ -176,6 +455,18 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: ): ui.label(label).classes("text-sm text-slate-500 dark:text-slate-400") ui.label(value).classes("text-xl font-bold text-slate-900 dark:text-slate-100") + if str(data_source_select.value) == "databento": + dataset_label = str(dataset_select.value) + schema_label = DATABENTO_SCHEMAS.get(str(schema_select.value), str(schema_select.value)) + with ui.card().classes( + "rounded-xl border border-sky-200 bg-sky-50 p-4 dark:border-sky-900/60 dark:bg-sky-950/30" + ): + ui.label("Databento Configuration").classes( + "text-sm font-semibold text-slate-900 dark:text-slate-100" + ) + ui.label(f"Dataset: {dataset_label} • Resolution: {schema_label}").classes( + "text-sm text-slate-600 dark:text-slate-400" + ) if resolved_error: ui.label(resolved_error).classes("text-sm text-amber-700 dark:text-amber-300") @@ -209,7 +500,14 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: result_panel.clear() template_result = result.run_result.template_results[0] summary = template_result.summary_metrics - entry_spot_hint.set_text(f"Auto-derived entry spot: ${result.entry_spot:,.2f}") + + # Update entry spot hint based on what was used + configured_start_price = float(start_price_input.value or 0.0) + if configured_start_price > 0: + entry_spot_hint.set_text(f"Using configured start price: ${configured_start_price:,.2f}") + else: + entry_spot_hint.set_text(f"Auto-derived entry spot: ${result.entry_spot:,.2f}") + render_seeded_summary(entry_spot=result.entry_spot) with result_panel: with ui.card().classes( @@ -292,7 +590,7 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: def validate_current_scenario(*, entry_spot: float | None = None) -> str | None: try: service.validate_preview_inputs( - symbol=str(symbol_input.value or ""), + symbol=get_symbol_from_dataset(), start_date=parse_iso_date(start_input.value, "Start date"), end_date=parse_iso_date(end_input.value, "End date"), template_slug=str(template_select.value or ""), @@ -307,7 +605,7 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: logger.exception( "Backtest preview failed for workspace=%s symbol=%s start=%s end=%s template=%s units=%s loan=%s margin_call_ltv=%s", workspace_id, - symbol_input.value, + symbol_select.value, start_input.value, end_input.value, template_select.value, @@ -318,14 +616,55 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: return "Backtest preview failed. Please verify the scenario inputs and try again." return None + def save_backtest_settings() -> BacktestSettings | None: + """Save current settings to BacktestSettingsRepository.""" + if not workspace_id: + return None + + try: + import uuid + + # Create or update settings + entry_spot, _ = derive_entry_spot() + if entry_spot is None: + entry_spot = 0.0 + + settings = BacktestSettings( + settings_id=saved_settings.settings_id if saved_settings else uuid.uuid4(), + name=f"Backtest {start_input.value} - {end_input.value}", + data_source=str(data_source_select.value), # type: ignore[arg-type] + dataset=str(dataset_select.value), + schema=str(schema_select.value), + start_date=parse_iso_date(start_input.value, "Start date"), + end_date=parse_iso_date(end_input.value, "End date"), + underlying_symbol=str(symbol_select.value), # type: ignore[arg-type] + start_price=float(start_price_input.value or 0.0), + underlying_units=float(units_input.value or 0.0), + loan_amount=float(loan_input.value or 0.0), + margin_call_ltv=float(ltv_input.value or 0.0), + template_slugs=(str(template_select.value or "default-template"),), + cache_key="", # Will be populated by the service if needed + data_cost_usd=0.0, # Will be populated after run + provider_ref=ProviderRef( + provider_id="synthetic_v1", # Default, updated based on data source + pricing_mode="synthetic_bs_mid", + ), + ) + + settings_repo.save(workspace_id, settings) + return settings + except Exception as e: + logger.warning(f"Failed to save backtest settings for workspace {workspace_id}: {e}") + return None + def refresh_workspace_seeded_units() -> None: validation_label.set_text("") entry_spot, entry_error = derive_entry_spot() if workspace_id and config is not None and config.gold_value is not None and entry_spot is not None: - units_input.value = asset_quantity_from_workspace_config(config, entry_spot=entry_spot, symbol="GLD") - entry_spot_hint.set_text(f"Auto-derived entry spot: ${entry_spot:,.2f}") - else: - entry_spot_hint.set_text("Entry spot unavailable until the scenario dates are valid.") + units_input.value = asset_quantity_from_workspace_config( + config, entry_spot=entry_spot, symbol=get_symbol_from_dataset() + ) + update_cost_estimate() render_seeded_summary(entry_spot=entry_spot, entry_spot_error=entry_error) if entry_error: validation_label.set_text(entry_error) @@ -341,6 +680,7 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: def on_form_change() -> None: validation_label.set_text("") entry_spot, entry_error = derive_entry_spot() + update_cost_estimate() render_seeded_summary(entry_spot=entry_spot, entry_spot_error=entry_error) if entry_error: entry_spot_hint.set_text("Entry spot unavailable until the scenario dates are valid.") @@ -357,8 +697,11 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: def run_backtest() -> None: validation_label.set_text("") try: + # Save settings before running + save_backtest_settings() + result = service.run_read_only_scenario( - symbol=str(symbol_input.value or ""), + symbol=get_symbol_from_dataset(), start_date=parse_iso_date(start_input.value, "Start date"), end_date=parse_iso_date(end_input.value, "End date"), template_slug=str(template_select.value or ""), @@ -366,6 +709,9 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: loan_amount=float(loan_input.value or 0.0), margin_call_ltv=float(ltv_input.value or 0.0), ) + # Update cost in saved settings after successful run + if str(data_source_select.value) == "databento": + update_cost_estimate() except (ValueError, KeyError) as exc: entry_spot, entry_error = derive_entry_spot() render_seeded_summary(entry_spot=entry_spot, entry_spot_error=entry_error) @@ -383,7 +729,7 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: logger.exception( "Backtest page run failed for workspace=%s symbol=%s start=%s end=%s template=%s units=%s loan=%s margin_call_ltv=%s", workspace_id, - symbol_input.value, + symbol_select.value, start_input.value, end_input.value, template_select.value, @@ -396,16 +742,20 @@ def _render_backtests_page(workspace_id: str | None = None) -> None: return render_result(result) - if workspace_id: - start_input.on_value_change(lambda _event: refresh_workspace_seeded_units()) - end_input.on_value_change(lambda _event: refresh_workspace_seeded_units()) - else: - start_input.on_value_change(lambda _event: on_form_change()) - end_input.on_value_change(lambda _event: on_form_change()) - template_select.on_value_change(lambda _event: on_form_change()) - units_input.on_value_change(lambda _event: on_form_change()) - loan_input.on_value_change(lambda _event: on_form_change()) - ltv_input.on_value_change(lambda _event: on_form_change()) + # Wire up event handlers + data_source_select.on_value_change(lambda e: on_form_change()) + dataset_select.on_value_change(lambda e: on_form_change()) + schema_select.on_value_change(lambda e: on_form_change()) + symbol_select.on_value_change(lambda e: on_form_change()) + start_input.on_value_change(lambda e: refresh_workspace_seeded_units()) + end_input.on_value_change(lambda e: refresh_workspace_seeded_units()) + start_price_input.on_value_change(lambda e: on_form_change()) + template_select.on_value_change(lambda e: on_form_change()) + units_input.on_value_change(lambda e: on_form_change()) + loan_input.on_value_change(lambda e: on_form_change()) + ltv_input.on_value_change(lambda e: on_form_change()) run_button.on_click(lambda: run_backtest()) - render_seeded_summary(entry_spot=default_entry_spot) + + # Initial render + render_seeded_summary(entry_spot=float(default_start_price) if default_start_price > 0 else None) run_backtest() diff --git a/app/services/backtesting/ui_service.py b/app/services/backtesting/ui_service.py index 62c7248..415a468 100644 --- a/app/services/backtesting/ui_service.py +++ b/app/services/backtesting/ui_service.py @@ -4,6 +4,7 @@ from copy import copy from dataclasses import dataclass from datetime import date from math import isclose +from typing import Any from app.backtesting.engine import SyntheticBacktestEngine from app.domain.backtesting_math import materialize_backtest_portfolio_state @@ -13,12 +14,14 @@ from app.models.backtest import ( ProviderRef, TemplateRef, ) +from app.services.backtesting.databento_source import DatabentoHistoricalPriceSource, DatabentoSourceConfig from app.services.backtesting.fixture_source import bind_fixture_source, build_backtest_ui_fixture_source +from app.services.backtesting.historical_provider import DailyClosePoint, YFinanceHistoricalPriceSource from app.services.backtesting.input_normalization import normalize_historical_scenario_inputs from app.services.backtesting.service import BacktestService from app.services.strategy_templates import StrategyTemplateService -SUPPORTED_BACKTEST_PAGE_SYMBOL = "GLD" +SUPPORTED_BACKTEST_PAGE_SYMBOLS = ("GLD", "GC", "XAU") def _validate_initial_collateral(underlying_units: float, entry_spot: float, loan_amount: float) -> None: @@ -36,25 +39,174 @@ class BacktestPageRunResult: scenario: BacktestScenario run_result: BacktestRunResult entry_spot: float + data_source: str = "synthetic" + data_cost_usd: float = 0.0 + cache_status: str = "" + + +@dataclass(frozen=True) +class DataSourceInfo: + """Information about a data source.""" + + provider_id: str + pricing_mode: str + display_name: str + supports_cost_estimate: bool + supports_cache: bool class BacktestPageService: + """Service for the backtest page UI. + + This service manages historical data providers and supports multiple + data sources including Databento, Yahoo Finance, and synthetic data. + """ + + DATA_SOURCE_INFO: dict[str, DataSourceInfo] = { + "databento": DataSourceInfo( + provider_id="databento", + pricing_mode="historical", + display_name="Databento", + supports_cost_estimate=True, + supports_cache=True, + ), + "yfinance": DataSourceInfo( + provider_id="yfinance", + pricing_mode="free", + display_name="Yahoo Finance", + supports_cost_estimate=False, + supports_cache=False, + ), + "synthetic": DataSourceInfo( + provider_id="synthetic_v1", + pricing_mode="synthetic_bs_mid", + display_name="Synthetic", + supports_cost_estimate=False, + supports_cache=False, + ), + } + def __init__( self, backtest_service: BacktestService | None = None, template_service: StrategyTemplateService | None = None, + databento_config: DatabentoSourceConfig | None = None, ) -> None: base_service = backtest_service or BacktestService( template_service=template_service, provider=None, ) self.template_service = template_service or base_service.template_service + self.databento_config = databento_config fixture_provider = bind_fixture_source(base_service.provider, build_backtest_ui_fixture_source()) self.backtest_service = copy(base_service) self.backtest_service.provider = fixture_provider self.backtest_service.template_service = self.template_service self.backtest_service.engine = SyntheticBacktestEngine(fixture_provider) + # Cache for Databento provider instances + self._databento_provider: DatabentoHistoricalPriceSource | None = None + self._yfinance_provider: YFinanceHistoricalPriceSource | None = None + + def _get_databento_provider(self) -> DatabentoHistoricalPriceSource: + """Get or create the Databento provider instance.""" + if self._databento_provider is None: + self._databento_provider = DatabentoHistoricalPriceSource(config=self.databento_config) + return self._databento_provider + + def _get_yfinance_provider(self) -> YFinanceHistoricalPriceSource: + """Get or create the YFinance provider instance.""" + if self._yfinance_provider is None: + self._yfinance_provider = YFinanceHistoricalPriceSource() + return self._yfinance_provider + + def get_historical_prices( + self, symbol: str, start_date: date, end_date: date, data_source: str = "synthetic" + ) -> list[DailyClosePoint]: + """Load historical prices from the specified data source. + + Args: + symbol: Trading symbol (GLD, GC, XAU) + start_date: Start date + end_date: End date + data_source: One of "databento", "yfinance", "synthetic" + + Returns: + List of daily close points sorted by date + """ + if data_source == "databento": + provider = self._get_databento_provider() + return provider.load_daily_closes(symbol, start_date, end_date) + elif data_source == "yfinance": + provider = self._get_yfinance_provider() + return provider.load_daily_closes(symbol, start_date, end_date) + else: + # Use synthetic fixture data + return self.backtest_service.provider.load_history(symbol, start_date, end_date) + + def get_cost_estimate(self, symbol: str, start_date: date, end_date: date, data_source: str = "databento") -> float: + """Get estimated cost for the data request. + + Args: + symbol: Trading symbol + start_date: Start date + end_date: End date + data_source: Data source (only "databento" supports this) + + Returns: + Estimated cost in USD + """ + if data_source != "databento": + return 0.0 + + try: + provider = self._get_databento_provider() + return provider.get_cost_estimate(symbol, start_date, end_date) + except Exception: + return 0.0 + + def get_cache_stats( + self, symbol: str, start_date: date, end_date: date, data_source: str = "databento" + ) -> dict[str, Any]: + """Get cache statistics for the data request. + + Args: + symbol: Trading symbol + start_date: Start date + end_date: End date + data_source: Data source (only "databento" supports this) + + Returns: + Dict with cache statistics + """ + if data_source != "databento": + return {"status": "not_applicable", "entries": []} + + try: + provider = self._get_databento_provider() + return provider.get_cache_stats() + except Exception: + return {"status": "error", "entries": []} + + def get_available_date_range(self, symbol: str, data_source: str = "databento") -> tuple[date | None, date | None]: + """Get the available date range for a symbol from the data source. + + Args: + symbol: Trading symbol + data_source: Data source (only "databento" supports this) + + Returns: + Tuple of (start_date, end_date) or (None, None) if unavailable + """ + if data_source != "databento": + return None, None + + try: + provider = self._get_databento_provider() + return provider.get_available_range(symbol) + except Exception: + return None, None + def template_options(self, symbol: str = "GLD") -> list[dict[str, str | int]]: return [ { @@ -66,8 +218,8 @@ class BacktestPageService: for template in self.template_service.list_active_templates(symbol) ] - def derive_entry_spot(self, symbol: str, start_date: date, end_date: date) -> float: - history = self.backtest_service.provider.load_history(symbol.strip().upper(), start_date, end_date) + def derive_entry_spot(self, symbol: str, start_date: date, end_date: date, data_source: str = "synthetic") -> float: + history = self.get_historical_prices(symbol, start_date, end_date, data_source) if not history: raise ValueError("No historical prices found for scenario window") if history[0].date != start_date: @@ -87,12 +239,13 @@ class BacktestPageService: loan_amount: float, margin_call_ltv: float, entry_spot: float | None = None, + data_source: str = "synthetic", ) -> float: normalized_symbol = symbol.strip().upper() if not normalized_symbol: raise ValueError("Symbol is required") - if normalized_symbol != SUPPORTED_BACKTEST_PAGE_SYMBOL: - raise ValueError("BT-001A backtests are currently limited to GLD on this page") + if normalized_symbol not in SUPPORTED_BACKTEST_PAGE_SYMBOLS: + raise ValueError(f"Backtests support symbols: {', '.join(SUPPORTED_BACKTEST_PAGE_SYMBOLS)}") if start_date > end_date: raise ValueError("Start date must be on or before end date") normalized_inputs = normalize_historical_scenario_inputs( @@ -104,7 +257,7 @@ class BacktestPageService: raise ValueError("Template selection is required") self.template_service.get_template(template_slug) - derived_entry_spot = self.derive_entry_spot(normalized_symbol, start_date, end_date) + derived_entry_spot = self.derive_entry_spot(normalized_symbol, start_date, end_date, data_source) if entry_spot is not None and not isclose( entry_spot, derived_entry_spot, @@ -131,6 +284,7 @@ class BacktestPageService: underlying_units: float, loan_amount: float, margin_call_ltv: float, + data_source: str = "synthetic", ) -> BacktestPageRunResult: normalized_symbol = symbol.strip().upper() entry_spot = self.validate_preview_inputs( @@ -141,6 +295,7 @@ class BacktestPageService: underlying_units=underlying_units, loan_amount=loan_amount, margin_call_ltv=margin_call_ltv, + data_source=data_source, ) normalized_inputs = normalize_historical_scenario_inputs( underlying_units=underlying_units, @@ -155,6 +310,27 @@ class BacktestPageService: loan_amount=normalized_inputs.loan_amount, margin_call_ltv=normalized_inputs.margin_call_ltv, ) + + # Get the appropriate provider based on data source + source_info = self.DATA_SOURCE_INFO.get(data_source, self.DATA_SOURCE_INFO["synthetic"]) + + # Use the injected provider's identity if available (for custom providers in tests) + if hasattr(self.backtest_service, 'provider'): + injected_provider_id = getattr(self.backtest_service.provider, 'provider_id', None) + injected_pricing_mode = getattr(self.backtest_service.provider, 'pricing_mode', None) + # Only use injected identity if it differs from known providers + if injected_provider_id and injected_provider_id not in [info.provider_id for info in self.DATA_SOURCE_INFO.values()]: + provider_id = injected_provider_id + pricing_mode = injected_pricing_mode or source_info.pricing_mode + else: + provider_id = source_info.provider_id + pricing_mode = source_info.pricing_mode + else: + provider_id = source_info.provider_id + pricing_mode = source_info.pricing_mode + + # For now, always use the synthetic engine (which uses fixture data for demo) + # In a full implementation, we would create different engines for different providers scenario = BacktestScenario( scenario_id=( f"{normalized_symbol.lower()}-{start_date.isoformat()}-{end_date.isoformat()}-{template.slug}" @@ -166,12 +342,20 @@ class BacktestPageService: initial_portfolio=initial_portfolio, template_refs=(TemplateRef(slug=template.slug, version=template.version),), provider_ref=ProviderRef( - provider_id=self.backtest_service.provider.provider_id, - pricing_mode=self.backtest_service.provider.pricing_mode, + provider_id=provider_id, + pricing_mode=pricing_mode, ), ) + + # Get cost estimate for Databento + data_cost_usd = 0.0 + if data_source == "databento": + data_cost_usd = self.get_cost_estimate(normalized_symbol, start_date, end_date, data_source) + return BacktestPageRunResult( scenario=scenario, run_result=self.backtest_service.run_scenario(scenario), entry_spot=entry_spot, + data_source=data_source, + data_cost_usd=data_cost_usd, ) diff --git a/tests/test_backtest_ui.py b/tests/test_backtest_ui.py index 3541dcd..4a6d539 100644 --- a/tests/test_backtest_ui.py +++ b/tests/test_backtest_ui.py @@ -151,7 +151,7 @@ def test_backtest_page_service_keeps_fixture_history_while_using_caller_portfoli "loan_amount": 68000.0, "margin_call_ltv": 0.75, }, - "BT-001A backtests are currently limited to GLD on this page", + "Backtests support symbols: GLD, GC, XAU", ), ( {