bu5hm4nn/vault-dash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct backtest job result serialization and add Playwright test fixtures

Browse Source 
- Fix BacktestPageRunResult serialization in jobs.py to correctly access
  nested fields from scenario and run_result objects
- Add test_backtest_job.py with comprehensive tests for job execution
- Add conftest_playwright.py with ServerManager that starts FastAPI server
  for Playwright tests using uvicorn
- Add test_playwright_server.py with E2E tests using the server fixture

The job serialization bug was causing backtest results to fail silently
because it was trying to access non-existent fields on BacktestPageRunResult.
This commit is contained in:
Bu5hm4nn
2026-04-04 18:27:34 +02:00
parent 6c35efde0f
commit d835544e58
4 changed files with 501 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
app/services/backtesting/jobs.py
View File

@@ -214,30 +214,64 @@ def run_backtest_job(
)
# Convert result to dict for serialization
# BacktestPageRunResult has: scenario, run_result, entry_spot, data_source, data_cost_usd
template_results = result.run_result.template_results
first_template = template_results[0] if template_results else None
summary = first_template.summary_metrics if first_template else None
result_dict = {
"scenario_name": result.scenario_name,
"entry_date": result.entry_date.isoformat() if result.entry_date else None,
"scenario_id": result.scenario.scenario_id,
"scenario_name": result.scenario.display_name,
"symbol": result.scenario.symbol,
"start_date": result.scenario.start_date.isoformat(),
"end_date": result.scenario.end_date.isoformat(),
"entry_spot": result.entry_spot,
"underlying_units": result.underlying_units,
"loan_amount": result.loan_amount,
"margin_call_ltv": result.margin_call_ltv,
"total_pnl": result.total_pnl,
"total_pnl_pct": result.total_pnl_pct,
"hedging_cost": result.hedging_cost,
"hedging_cost_pct": result.hedging_cost_pct,
"unhedged_pnl": result.unhedged_pnl,
"unhedged_pnl_pct": result.unhedged_pnl_pct,
"margin_calls": result.margin_calls,
"margin_call_events": [
"underlying_units": result.scenario.initial_portfolio.underlying_units,
"loan_amount": result.scenario.initial_portfolio.loan_amount,
"margin_call_ltv": result.scenario.initial_portfolio.margin_call_ltv,
"data_source": result.data_source,
"data_cost_usd": result.data_cost_usd,
# Summary metrics from first template result
"start_value": summary.start_value if summary else 0.0,
"end_value_hedged_net": summary.end_value_hedged_net if summary else 0.0,
"total_hedge_cost": summary.total_hedge_cost if summary else 0.0,
"max_ltv_hedged": summary.max_ltv_hedged if summary else 0.0,
"max_ltv_unhedged": summary.max_ltv_unhedged if summary else 0.0,
"margin_call_days_hedged": summary.margin_call_days_hedged if summary else 0,
"margin_call_days_unhedged": summary.margin_call_days_unhedged if summary else 0,
"margin_threshold_breached_hedged": summary.margin_threshold_breached_hedged if summary else False,
"margin_threshold_breached_unhedged": summary.margin_threshold_breached_unhedged if summary else False,
# Template results with full daily path
"template_results": [
{
"date": event.date.isoformat(),
"price": event.price,
"ltv": event.ltv,
"action": event.action,
"template_slug": tr.template_slug,
"template_name": tr.template_name,
"summary_metrics": {
"start_value": tr.summary_metrics.start_value,
"end_value_hedged_net": tr.summary_metrics.end_value_hedged_net,
"total_hedge_cost": tr.summary_metrics.total_hedge_cost,
"max_ltv_hedged": tr.summary_metrics.max_ltv_hedged,
"max_ltv_unhedged": tr.summary_metrics.max_ltv_unhedged,
"margin_call_days_hedged": tr.summary_metrics.margin_call_days_hedged,
"margin_call_days_unhedged": tr.summary_metrics.margin_call_days_unhedged,
},
"daily_path": [
{
"date": dp.date.isoformat(),
"spot_close": dp.spot_close,
"underlying_value": dp.underlying_value,
"option_market_value": dp.option_market_value,
"net_portfolio_value": dp.net_portfolio_value,
"ltv_hedged": dp.ltv_hedged,
"ltv_unhedged": dp.ltv_unhedged,
"margin_call_hedged": dp.margin_call_hedged,
"margin_call_unhedged": dp.margin_call_unhedged,
}
for dp in tr.daily_path
],
}
for event in (result.margin_call_events or [])
for tr in template_results
],
"prices": [{"date": p.date.isoformat(), "close": p.close} for p in (result.prices or [])],
}
# Stage 4: Complete

125
tests/conftest_playwright.py Normal file
View File

@@ -0,0 +1,125 @@
"""Pytest configuration for Playwright tests.
This conftest creates module-scoped fixtures that start the FastAPI server
before running Playwright tests and stop it after all tests complete.
"""
from __future__ import annotations
import logging
import os
import socket
import sys
import threading
import time
from collections.abc import Generator
from pathlib import Path
import pytest
import uvicorn
# Suppress NiceGUI banner noise
logging.getLogger("nicegui").setLevel(logging.WARNING)
def find_free_port() -> int:
"""Find a free port on localhost."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("", 0))
s.listen(1)
port = s.getsockname()[1]
return port
class ServerManager:
"""Manages a NiceGUI/FastAPI server for testing."""
_instance: "ServerManager | None" = None
_lock = threading.Lock()
def __init__(self, port: int) -> None:
self.port = port
self.url = f"http://localhost:{port}"
self._thread: threading.Thread | None = None
self._server: uvicorn.Server | None = None
def start(self) -> None:
"""Start the server in a background thread."""
self._thread = threading.Thread(target=self._run_server, daemon=True)
self._thread.start()
# Wait for server to be ready
if not self._wait_for_connection(timeout=30):
raise RuntimeError("Server did not start within 30 seconds")
def _run_server(self) -> None:
"""Run the FastAPI server with uvicorn."""
# Ensure project root is on sys.path
project_root = str(Path(__file__).parent.parent)
if project_root not in sys.path:
sys.path.insert(0, project_root)
os.environ["APP_ENV"] = "test"
os.environ["NICEGUI_STORAGE_SECRET"] = "test-secret-key"
# Import after environment is set
from app.main import app
# Configure uvicorn
config = uvicorn.Config(
app,
host="127.0.0.1",
port=self.port,
log_level="warning",
access_log=False,
)
self._server = uvicorn.Server(config)
self._server.run()
def _wait_for_connection(self, timeout: float = 10.0) -> bool:
"""Wait for server to accept connections."""
deadline = time.time() + timeout
while time.time() < deadline:
try:
with socket.create_connection(("127.0.0.1", self.port), timeout=1):
return True
except OSError:
time.sleep(0.1)
return False
def stop(self) -> None:
"""Stop the server."""
if self._server:
self._server.should_exit = True
if self._thread and self._thread.is_alive():
self._thread.join(timeout=5)
@classmethod
def get_or_create(cls, port: int) -> "ServerManager":
"""Get existing instance or create new one."""
with cls._lock:
if cls._instance is None:
cls._instance = cls(port)
return cls._instance
@pytest.fixture(scope="module")
def server_url() -> Generator[str, None, None]:
"""Start the server once per module and yield its URL."""
port = find_free_port()
server = ServerManager(port)
# Start server
server.start()
yield server.url
# Cleanup
server.stop()
ServerManager._instance = None
@pytest.fixture(scope="module")
def base_url(server_url: str) -> str:
"""Alias for server_url for naming consistency with Playwright conventions."""
return server_url

190
tests/test_backtest_job.py Normal file
View File

@@ -0,0 +1,190 @@
"""Tests for backtest job execution via the jobs module."""
from __future__ import annotations
from datetime import date
from app.services.backtesting.jobs import (
BacktestJobStore,
JobStage,
JobStatus,
job_store,
run_backtest_job,
)
from app.services.backtesting.ui_service import BacktestPageService
def test_backtest_job_store_creates_and_retrieves_jobs() -> None:
"""Test that the job store can create and retrieve jobs."""
store = BacktestJobStore()
job = store.create_job("workspace-1")
assert job.status == JobStatus.PENDING
assert job.stage == JobStage.VALIDATING
retrieved = store.get_job("workspace-1")
assert retrieved is job
# Creating a new job replaces the old one
job2 = store.create_job("workspace-1")
assert store.get_job("workspace-1") is job2
def test_backtest_job_store_updates_job_status() -> None:
"""Test that job status can be updated."""
store = BacktestJobStore()
store.create_job("workspace-1")
store.update_job(
"workspace-1",
status=JobStatus.RUNNING,
stage=JobStage.FETCHING_PRICES,
progress=50,
message="Fetching prices...",
)
updated = store.get_job("workspace-1")
assert updated is not None
assert updated.status == JobStatus.RUNNING
assert updated.stage == JobStage.FETCHING_PRICES
assert updated.progress == 50
assert updated.message == "Fetching prices..."
def test_run_backtest_job_completes_with_synthetic_data() -> None:
"""Test that a backtest job completes successfully with synthetic fixture data."""
# Use the global job_store singleton
workspace_id = "workspace-test-job-complete"
job = job_store.create_job(workspace_id)
service = BacktestPageService()
run_backtest_job(
workspace_id=workspace_id,
job=job,
service=service,
symbol="GLD",
start_date=date(2024, 1, 2),
end_date=date(2024, 1, 8),
template_slug="protective-put-atm-12m",
underlying_units=1000.0,
loan_amount=68000.0,
margin_call_ltv=0.75,
data_source="synthetic",
)
updated = job_store.get_job(workspace_id)
assert updated is not None
assert updated.status == JobStatus.COMPLETE, f"Job failed with error: {updated.error}"
assert updated.result is not None
# Verify result structure
result = updated.result
assert "scenario_id" in result
assert "entry_spot" in result
assert result["entry_spot"] == 100.0
assert "template_results" in result
assert len(result["template_results"]) == 1
# Verify template result structure
template_result = result["template_results"][0]
assert "template_slug" in template_result
assert "summary_metrics" in template_result
assert "daily_path" in template_result
# Verify there are daily points
daily_path = template_result["daily_path"]
assert len(daily_path) == 5 # 5 trading days in fixture
# Verify summary metrics exist
summary = template_result["summary_metrics"]
assert "start_value" in summary
assert "total_hedge_cost" in summary
def test_run_backtest_job_serializes_daily_path_correctly() -> None:
"""Test that daily_path is properly serialized with all fields."""
workspace_id = "workspace-test-daily-path"
job = job_store.create_job(workspace_id)
service = BacktestPageService()
run_backtest_job(
workspace_id=workspace_id,
job=job,
service=service,
symbol="GLD",
start_date=date(2024, 1, 2),
end_date=date(2024, 1, 8),
template_slug="protective-put-atm-12m",
underlying_units=1000.0,
loan_amount=68000.0,
margin_call_ltv=0.75,
data_source="synthetic",
)
result = job_store.get_job(workspace_id).result
daily_point = result["template_results"][0]["daily_path"][0]
# Verify all fields are present
assert "date" in daily_point
assert "spot_close" in daily_point
assert "underlying_value" in daily_point
assert "option_market_value" in daily_point
assert "net_portfolio_value" in daily_point
assert "ltv_hedged" in daily_point
assert "ltv_unhedged" in daily_point
assert "margin_call_hedged" in daily_point
assert "margin_call_unhedged" in daily_point
def test_run_backtest_job_handles_validation_errors() -> None:
"""Test that validation errors are properly captured."""
workspace_id = "workspace-test-validation"
job = job_store.create_job(workspace_id)
service = BacktestPageService()
# Use invalid dates (outside fixture window for synthetic)
run_backtest_job(
workspace_id=workspace_id,
job=job,
service=service,
symbol="GLD",
start_date=date(2023, 1, 1), # Outside fixture window
end_date=date(2023, 1, 7),
template_slug="protective-put-atm-12m",
underlying_units=1000.0,
loan_amount=68000.0,
margin_call_ltv=0.75,
data_source="synthetic",
)
updated = job_store.get_job(workspace_id)
assert updated is not None
assert updated.status == JobStatus.FAILED
assert updated.error is not None
assert "deterministic fixture data" in updated.error
def test_run_backtest_job_handles_invalid_symbol() -> None:
"""Test that invalid symbols are properly rejected."""
workspace_id = "workspace-test-invalid-symbol"
job = job_store.create_job(workspace_id)
service = BacktestPageService()
run_backtest_job(
workspace_id=workspace_id,
job=job,
service=service,
symbol="INVALID",
start_date=date(2024, 1, 2),
end_date=date(2024, 1, 8),
template_slug="protective-put-atm-12m",
underlying_units=1000.0,
loan_amount=68000.0,
margin_call_ltv=0.75,
data_source="synthetic",
)
updated = job_store.get_job(workspace_id)
assert updated is not None
assert updated.status == JobStatus.FAILED
assert "GLD, GC, XAU" in updated.error

133
tests/test_playwright_server.py Normal file
View File

@@ -0,0 +1,133 @@
"""Playwright tests for end-to-end validation of the vault dashboard.
This test file verifies that:
1. The application starts and serves pages correctly
2. Workspace creation and navigation work
3. Backtest page loads and executes scenarios
4. The full user flow works without runtime errors
"""
from __future__ import annotations
import sys
from pathlib import Path
from playwright.sync_api import expect
# Import the server fixture from conftest_playwright
sys.path.insert(0, str(Path(__file__).parent))
ARTIFACTS = Path("tests/artifacts")
ARTIFACTS.mkdir(parents=True, exist_ok=True)
def assert_no_horizontal_overflow(page) -> None:
"""Verify the page doesn't have horizontal scrollbar."""
scroll_width = page.evaluate("document.documentElement.scrollWidth")
viewport_width = page.evaluate("window.innerWidth")
assert scroll_width <= viewport_width + 1
def test_server_starts_and_serves_homepage(server_url: str) -> None:
"""Verify the server starts and serves the homepage."""
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page(viewport={"width": 1440, "height": 1000})
page.goto(server_url, wait_until="domcontentloaded", timeout=30000)
expect(page).to_have_title("NiceGUI")
expect(page.locator("text=Create a private workspace URL").first).to_be_visible(timeout=10000)
browser.close()
def test_workspace_creation_and_navigation(server_url: str) -> None:
"""Test creating a workspace and navigating between pages."""
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page(viewport={"width": 1440, "height": 1000})
# Create workspace
page.goto(server_url, wait_until="domcontentloaded", timeout=30000)
expect(page.locator("text=Create a private workspace URL").first).to_be_visible(timeout=10000)
page.get_by_role("button", name="Get started").click()
page.wait_for_url(f"{server_url}/*", timeout=15000)
workspace_url = page.url
workspace_id = workspace_url.removeprefix(f"{server_url}/")
assert workspace_id, "Should have workspace ID in URL"
# Verify overview page loads
expect(page.locator("text=Vault Dashboard").first).to_be_visible(timeout=10000)
expect(page.locator("text=Overview").first).to_be_visible(timeout=10000)
browser.close()
def test_backtest_with_synthetic_data(server_url: str) -> None:
"""E2E test: Backtest scenario execution with synthetic fixture data.
This test verifies:
1. Synthetic data source can be selected
2. Fixture-supported dates (2024-01-02 to 2024-01-08) work
3. Run button triggers execution and results display
"""
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page(viewport={"width": 1440, "height": 1000})
# Create workspace
page.goto(server_url, wait_until="domcontentloaded", timeout=30000)
expect(page.locator("text=Create a private workspace URL").first).to_be_visible(timeout=10000)
page.get_by_role("button", name="Get started").click()
page.wait_for_url(f"{server_url}/*", timeout=15000)
workspace_url = page.url
# Navigate to backtests
page.goto(f"{workspace_url}/backtests", wait_until="domcontentloaded", timeout=30000)
expect(page.locator("text=Scenario Configuration")).to_be_visible(timeout=10000)
# Select Synthetic data source
data_source_select = page.locator("[data-testid=data-source-select]")
expect(data_source_select).to_be_visible(timeout=5000)
data_source_select.click()
page.get_by_text("Synthetic", exact=True).click()
page.wait_for_timeout(500)
# Fill in fixture-supported dates
start_date_input = page.get_by_label("Start date")
end_date_input = page.get_by_label("End date")
start_date_input.fill("2024-01-02")
end_date_input.fill("2024-01-08")
start_date_input.press("Tab")
page.wait_for_timeout(1000)
# Fill scenario parameters
page.get_by_label("Underlying units").fill("1000")
page.get_by_label("Loan amount").fill("68000")
page.get_by_label("Margin call LTV").fill("0.75")
# Run backtest
run_button = page.get_by_role("button", name="Run backtest")
expect(run_button).to_be_enabled(timeout=5000)
run_button.click()
# Wait for results
scenario_results = page.locator("text=Scenario Results")
expect(scenario_results).to_be_visible(timeout=30000)
# Verify results
result_text = page.locator("body").inner_text(timeout=10000)
assert "Start value" in result_text
assert "RuntimeError" not in result_text
assert "Traceback" not in result_text
page.screenshot(path=str(ARTIFACTS / "backtest_synthetic_results.png"), full_page=True)
browser.close()