diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..21df0b0 --- /dev/null +++ b/.env.example @@ -0,0 +1,12 @@ +# Copy to .env and adjust. All variables are optional; defaults are in quantcore/config.py. +QUANTCORE_DATA_DIR=./data # where the setup/download services persist everything +QUANTCORE_CUOPT_URL=http://127.0.0.1:5000 # cuOpt on this machine (Apptainer setup); a remote one: http://:5000 +QUANTCORE_CUOPT_TIME_LIMIT=120 # seconds, MILP time limit +QUANTCORE_BASE_CURRENCY=CHF +QUANTCORE_RISK_FREE_RATE=0.0 # annual, CHF +QUANTCORE_HISTORY_START=2004-12-01 # first date requested from Yahoo +QUANTCORE_REQUIRED_HISTORY_FROM=2005-01-31 # a stock must have prices on or before this date +QUANTCORE_N_US=500 +QUANTCORE_N_EUROPE=500 +QUANTCORE_LOG_LEVEL=INFO +QUANTCORE_ENABLE_UI=true # false: JSON API and /docs only, no test bench at /ui/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..214165a --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# data produced by the setup / download services lives outside version control +data/ +*.parquet +!quantcore/data/static/* +.env +__pycache__/ +*.pyc +.pytest_cache/ +.ruff_cache/ +*.egg-info/ +build/ +dist/ +.DS_Store +scratch/ +# LaTeX build files +*.aux +*.fdb_latexmk +*.fls +*.log +*.out +*.toc +*.synctex.gz diff --git a/README.md b/README.md index 5adb9d8..8c1fd49 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,250 @@ -# Portfolio_Risk_Management +# quantcore — Portfolio Risk Measurement & Optimisation -The API for risk management, data downloads and portfolio optimization for AISE502 \ No newline at end of file +The **deterministic quant core** of the AI-Augmented Portfolio Intelligence Platform (AISE502): +data, risk and performance measurement, stock-price models, Black–Litterman expected returns and +mean–CVaR portfolio optimisation in whole shares, solved as a MILP on **NVIDIA cuOpt** on the DGX Spark. + +Everything here is *deterministic* — same input, same output — and contains **no LLM call**. The AI +layer of the platform (news → insights → views) consumes these services through their contracts and +never computes a number itself. + +| Service | Module | What it does | +|---|---|---| +| **SetupService** | `quantcore.universe` | Builds the investment universe **once**: largest 500 US (S&P 500) and 500 European (STOXX 600; EU, CH, NO, UK) stocks, Yahoo metadata (name, country, listing currency, exchange, GICS sector/industry, market cap, shares outstanding), full price history since 2004, FX rates; keeps only stocks with prices since 2005. | +| **DataDownloadService** | `quantcore.data` | Fetches the newest Yahoo prices and FX rates (incremental with dividend-adjustment splicing, or full), snapshots for reproducible demos. | +| **MarketDataService** | `quantcore.data.market_data` | Aligned adjusted closes and log-returns, daily/weekly/monthly, in CHF or local currency; market caps, cap-weighted broad market index. | +| **RiskService** | `quantcore.measurement.risk` | Volatility, correlation, covariance, VaR and CVaR (95 %, 99 %) per asset and per share-based portfolio — moving windows and exponentially weighted (EWM), Gaussian and historical, for a day / week / month, as absolute CHF amounts. | +| **PerformanceService** | `quantcore.measurement.performance` | Annualised return, volatility, Sharpe, Sortino, drawdown, Calmar, skew/kurtosis, hit ratio; rolling and EWM series; portfolios in shares. | +| **Stock-price models** | `quantcore.models.simulation` | Geometric Brownian motion (multivariate normal) and the empirical-innovation model `eps_t = Sigma_{t-1}^{-1/2} r_t`, `r(j|T) = Sigma_T^{1/2} eps_j` with the latest rolling or EWM covariance. | +| **Expected returns** | `quantcore.models.black_litterman` | Market-cap equilibrium returns (broad-market average, market weights), Black–Litterman on a 10-year weekly covariance with **Idzorek confidence** views (diagonal Ω), sentiment → view helper. | +| **OptimizationService** | `quantcore.optimization` | Mean–CVaR MILP in integer share numbers: **new portfolio** (from cash) or **rebalancing** (existing holdings, transaction costs); constraints: max 10 % per stock, max per sector, no shorts, cost/turnover limits, optional cardinality; scenario sets with variance reduction (Sobol', moment matching, market-direction importance sampling) that stay identical when risk limits, constraints, costs or views change, tail-row generation for large sets, and an exact Gaussian (mean–variance-equivalent) mode; solved on cuOpt (GPU, over HTTP) with a HiGHS fallback (CPU, compiled into SciPy — runs in-process, nothing to deploy). | +| **HTTP API** | `quantcore.api` | FastAPI app (`/docs`) exposing all of the above; setup/update as background jobs. Runs as a container on the DGX Spark next to cuOpt. | + +Documentation: **[docs/USAGE.md](docs/USAGE.md) (start here if you are building on this package: the API, the +Python client, the `quantcore-api` command line, the conventions and the traps)**, +[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) (services, contracts, data flow, resilience), +[docs/METHODOLOGY.md](docs/METHODOLOGY.md) (every formula and convention), +[docs/optimization_model.pdf](docs/optimization_model.pdf) (derivation of the MILP from the Rockafellar–Uryasev +model in weights to whole shares with rebalancing and costs; sampling error of scenario sets, quasi-Monte Carlo, +moment matching, importance sampling, row generation, the exact Gaussian/mean–variance case and reproducibility; +source `docs/optimization_model.tex`), +[docs/measurement_and_models.pdf](docs/measurement_and_models.pdf) (risk and performance measures, rolling/EWM +estimators and shrinkage, GBM and empirical-innovation scenarios, Black–Litterman with Idzorek confidence; +source `docs/measurement_and_models.tex`), +[docs/SPARK_APPTAINER.md](docs/SPARK_APPTAINER.md) (**students: running it on your own DGX Spark** with +Apptainer, no root), [docs/SPARK_DEPLOYMENT.md](docs/SPARK_DEPLOYMENT.md) (the Docker setup on a Spark with root). + +--- + +## Quick start on your DGX Spark (students) + +The service and cuOpt run as two Apptainer containers on your Spark and talk over `127.0.0.1`. +No root rights, no Docker, nothing exposed to the network: + +```bash +git clone https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git +cd Portfolio_Risk_Management +deploy/apptainer/qc.sh install # once: cuOpt image, quantcore image, self-test +deploy/apptainer/qc.sh data-import quantcore_data.tar.gz # prepared data set (or: qc.sh setup, ~15 min) +deploy/apptainer/qc.sh start # -> http://127.0.0.1:5001/ui/ and /docs +deploy/apptainer/qc.sh api selftest # every endpoint, pass/fail +``` + +Your platform then calls the API at `http://127.0.0.1:5001`. The full guide, with settings, +several users on one Spark and troubleshooting, is [docs/SPARK_APPTAINER.md](docs/SPARK_APPTAINER.md). + +## Quick start (local) + +```bash +pip install -e ".[dev]" # Python 3.11+ +cp .env.example .env # optional: data dir, cuOpt URL, ... + +quantcore setup # once: universe + full history (~15 min, ~1100 Yahoo tickers) +quantcore status # what is stored, cuOpt health +quantcore update # later: newest prices / FX (incremental) +quantcore snapshot --name demo # freeze the data for a reproducible demo + +quantcore risk NESN.SW AAPL ASML.AS --horizon week --window 200 --lam 0.94 +quantcore performance NESN.SW AAPL --freq weekly +quantcore optimize --cash 1000000 --cvar-limit 40000 --confidence 0.95 --horizon week --max-assets 100 +quantcore serve # HTTP API on :5001 -> http://localhost:5001/docs + # and the test bench -> http://localhost:5001/ui/ +``` + +Against a *running* service -- from any machine, without the data directory -- use the API client +instead (`quantcore-api`, or `quantcore.client.QuantcoreClient` from Python): + +```bash +export QUANTCORE_API_URL=http://127.0.0.1:5001 # or the address of the machine that runs it +quantcore-api health +quantcore-api selftest # call every endpoint, pass/fail table, exit code +quantcore-api endpoints # the API surface, from its own OpenAPI schema +quantcore-api risk NESN.SW AAPL --horizon week --drift zero --space value +quantcore-api optimize --cash 1000000 --cvar-limit 40000 --horizon week --risk-drift zero +``` + +`quantcore setup` fetches the candidate lists from the web (Wikipedia S&P 500, iShares STOXX 600 +holdings); `--offline` uses the snapshots shipped in `quantcore/data/static/`. All data lands in +`QUANTCORE_DATA_DIR` (default `./data`): `universe/universe.csv`, `prices/adj_close.parquet`, +`fx/fx_chf.parquet`, `manifest.json`, `snapshots/`. + +## Using the services from Python + +```python +from quantcore import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.measurement.risk import RiskService +from quantcore.measurement.performance import PerformanceService +from quantcore.optimization.service import OptimizationService, OptimizationRequest + +settings = get_settings() +md = MarketDataService(settings) # or MarketDataService(settings, snapshot="demo") + +risk = RiskService(md) +risk.asset_risk(["NESN.SW", "AAPL"], horizon="week", window=200, lam=0.94) # per-asset table +risk.correlation(["NESN.SW", "AAPL", "ASML.AS"], method="ewm", lam=0.99) +risk.portfolio_risk({"NESN.SW": 500, "AAPL": 100}, cash=20_000, horizon="month", method="historical") + +PerformanceService(md).asset_performance(["NESN.SW", "AAPL"], freq="weekly") + +opt = OptimizationService(settings, md) +res = opt.optimize(OptimizationRequest( + mode="new", cash=1_000_000, max_assets=100, horizon="week", confidence=0.95, cvar_limit=40_000, + scenario_model="empirical", covariance={"method": "ewm", "lam": 0.99}, + views=[{"symbol": "NESN.SW", "sentiment": 0.7, "confidence": 0.6, "source": "insight-1234"}], +)) +res.positions_frame() # shares, value, weight, trade per stock +res.cvar, res.expected_pnl # CHF over the horizon +res.scenario_hash, res.reproducibility # same inputs -> same scenarios; 'exact' (HiGHS) | 'within_gap' (cuOpt) +res.scenario_key["id"] # varying the risk limit, constraints, costs or views keeps this id: one + # scenario set for a whole family of runs (and it is cached in the service) + +# scenario options: 4096 moment-matched Sobol' scenarios by default; importance sampling for the tail +# switches on automatically at beta = 0.99; large sets are solved by tail-row generation +res99 = opt.optimize(OptimizationRequest(cash=1_000_000, confidence=0.99, cvar_limit=60_000, n_scenarios=16_384)) +# the exact Gaussian (mean-variance-equivalent) solution: no scenarios, no sampling error +res_g = opt.optimize(OptimizationRequest(cash=1_000_000, cvar_limit=40_000, scenario_model="gaussian_exact")) +# n_scenarios defaults to 16 per candidate (4096-16384); the result warns if that is low for the +# candidate count or if the covariance estimate is ill-conditioned for it + +# rebalancing: existing holdings, transaction costs, turnover limit +res2 = opt.optimize(OptimizationRequest( + mode="rebalance", cash=res.cash_end, holdings={p["symbol"]: p["shares"] for p in res.positions}, + cvar_limit=30_000, costs={"proportional_buy": 0.001, "proportional_sell": 0.001, "fixed_per_trade": 10}, + constraints={"max_weight": 0.10, "max_sector_weight": 0.25, "max_turnover": 0.3}, +)) +``` + +## HTTP API (the platform's contract) + +| Method | Path | Purpose | +|---|---|---| +| GET | `/health` | store status, cuOpt health | +| GET | `/universe?region=§or=&country=` · `/universe/{symbol}` | universe with metadata | +| GET | `/prices?symbols=A,B&freq=weekly¤cy=base` · `/returns?...` | prices / log-returns | +| GET | `/market/index?freq=weekly` · `/market/weights` | cap-weighted market index, weights | +| POST | `/risk/assets` (`space: log｜value`, `drift: estimated｜zero`) · `/risk/series` · `/risk/covariance` · `/risk/correlation` · `/risk/portfolio` | risk measurement | +| POST | `/performance/assets` · `/performance/rolling` · `/performance/portfolio` | performance measurement | +| GET | `/data/symbols?region=§or=&country=&search=` | one table of the universe (metadata, market cap, price) plus the filter facets | +| POST | `/model/expected-returns` · `/model/covariance` · `/model/scenarios` | Black–Litterman posterior, the estimator the models use, scenario set (+ the P&L distribution of a portfolio) | +| POST | `/optimize` | mean–CVaR MILP (new / rebalance; a rebalancing also reports the starting portfolio under `before`) | +| POST | `/jobs/setup` · `/jobs/update?full=` · GET `/jobs/{id}` | background setup / data update | +| POST/GET | `/snapshots` | freeze / list data snapshots | + +Example: + +```bash +curl -s http://127.0.0.1:5001/optimize -H 'Content-Type: application/json' -d '{ + "mode": "new", "cash": 1000000, "max_assets": 100, "horizon": "week", + "confidence": 0.95, "cvar_limit": 40000, "scenario_model": "empirical", + "views": [{"symbol": "NESN.SW", "sentiment": 0.7, "confidence": 0.6}] +}' | python -m json.tool +``` + +## Test bench (browser UI) + +`http://127.0.0.1:5001/ui/` serves a single-page test bench — the +one place to drive every service by hand and judge what it produces. It is a pure client of the HTTP +API above: no numbers are computed in the browser. + +Because it is a client and not a part of the API, it can be left out of a deployment: +`QUANTCORE_ENABLE_UI=false` (read once at startup) serves the JSON API and `/docs` only, `/ui/` +answers 404 and `/` no longer redirects. The same image therefore also suits an installation whose +users build their own front end against these endpoints. + +The left panel holds the state every tab shares — the candidate stocks, the portfolio (new or +rebalancing, cash, holdings) and the models (frequency, horizon, stock-price model, scenario count, +seed, sampling, tail tilt, covariance estimator, expected-return model). Five tabs use it: + +| Tab | What it answers | +|---|---| +| **Stocks & data** | Which stocks exist (filter by region, sector, country, name) and what their history looks like: indexed prices, drawdowns, risk/return, market caps, and a per-stock table of volatility, VaR/CVaR (normal and historical), Sharpe, drawdown, skew and kurtosis. | +| **Risk & scenarios** | Per-stock scenario moments and quantiles in **price returns** (bounded below by −100 %) or, with one click, in the log-returns the model works in — a quarterly log-return of −2.0 is a price return of −86 %, and mixing the two is the easiest way to misread a simulation. The covariance the optimiser actually sees: correlation heat map, volatilities, eigenvalue spectrum, shrinkage and conditioning warnings. Then the scenario set: the horizon P&L distribution of a chosen portfolio (cap-weighted, equally weighted, your holdings or the last proposal) with VaR/CVaR markers, per-stock moments and quantiles, a single-stock distribution, the scenario hash, and a sampling-noise check across three seeds. | +| **Expected returns** | The market-implied (equilibrium) returns from reverse optimisation, the views you add (sentiment or explicit, with Idzorek confidence), and the posterior — side by side, with the change per stock and how each view was translated into `q` and `ω`. | +| **Constraints** | Weight, sector, cardinality, turnover, cost and investment-band limits, transaction costs, plus a feasibility check against the candidate set (are there enough names, do the sector caps leave room, is any stock unaffordable at one share?) and the sector composition against the caps. | +| **Optimisation** | Objective, confidence, CVaR limit, the risk convention (`risk_drift`), solver and time limit; the proposal in integer shares with trades and costs; for a rebalancing the **before and after** of the same portfolio on the same scenarios; the weights and sector exposure against the limits; the provenance (scenario key and hash, solver, MIP gap, row-generation passes, reproducibility); a run history and a risk-limit sweep that traces the risk/return trade-off. | + +A third: **the CVaR limit can stop constraining anything over a long horizon**. CVaR₉₉ ≈ −μₕ + 2.67 σₕ, +so with a one-year horizon and an expected return of 40 % against a portfolio volatility of ~13 %, the whole +1 % tail is a gain, the limit is slack and mean-CVaR degenerates into "buy the highest forecast returns up to +the weight caps". The result then warns, and `risk_drift="zero"` measures the CVaR on the centred scenarios +(the risk figure is then independent of the return forecast) while the objective keeps it. The `historical` +expected-return model makes this extreme: on this data it gives up to 821 % p.a. to a single stock. + +Two things to know while using it. The **time limit is the budget of the whole run**, not of each +solver call: row generation and the Gaussian outer approximation solve a sequence of MILPs, and each +one gets what is left of the budget (the default is 300 s; an empty field means no limit, and HiGHS +will then run until it can prove optimality -- hours on a large candidate set). And the service +answers **one request at a time** while a solve runs: the MILP holds the interpreter, so the other +tabs wait until it finishes. The progress panel says both, and a run can be cancelled. + +Typical checks it is built for: put 1 m or 50 m CHF into a new portfolio and see how the composition +changes with wealth (integer shares start to matter); restrict the universe to European stocks and +compare; add a view on one stock and watch the posterior and the proposal move; switch the +stock-price model from GBM to empirical innovations (or to the exact Gaussian mode, which has no +sampling error at all) and see the risk figures change; run the same request twice and confirm that +the scenario hash and the shares are identical. + +## On the DGX Spark + +There are two setups. Both run the same code and give the same results. + +* **Apptainer, no root** (the students' Sparks): `deploy/apptainer/qc.sh` runs cuOpt and the API as + two Apptainer instances of the user, bound to `127.0.0.1`. See + [docs/SPARK_APPTAINER.md](docs/SPARK_APPTAINER.md). +* **Docker Compose, with root** (the lecturer's Spark): `deploy/deploy_to_spark.sh --with-data` + adds a `quantcore` service next to the `cuopt` service of an existing compose file and reaches + cuOpt at `http://cuopt:5000`. See [docs/SPARK_DEPLOYMENT.md](docs/SPARK_DEPLOYMENT.md). + +## Tests + +```bash +pytest # 127 tests: reference values for VaR/CVaR, Idzorek closed form vs. numeric, brute-force MILP optimality, + # exact Gaussian mode vs. brute force, row generation vs. full MILP, sampling/moment matching/tilt, + # weighted quantiles and scenario distributions, API (incl. the before/after of a rebalancing), + # the HTTP client and every `quantcore-api` command against the app in-process +``` + +The cuOpt round-trip test runs only when the server is reachable. + +## Layout + +``` +quantcore/ + config.py settings (env QUANTCORE_*, .env) + universe/ candidate sources, Yahoo symbol mapping, SetupService + data/ YahooClient (timeouts/retry/breaker), FX, DataStore, DataDownloadService, MarketDataService, static/ + measurement/ returns, rolling & EWM estimators, risk (VaR/CVaR), performance + models/ CovarianceModel + matrix roots, GBM & empirical-innovation simulators, Black-Litterman/Idzorek + optimization/ MILP builder (problem.py), cuOpt/HiGHS solvers, OptimizationService + api/ FastAPI app + pydantic schemas + api/ui/ the browser test bench (static HTML/CSS/ES modules, no build step, no CDN); + QUANTCORE_ENABLE_UI=false serves the JSON API and /docs only + client.py QuantcoreClient -- the API as a Python object (needs only a URL) + cli.py `quantcore` command (in-process services, needs the data directory) + apicli.py `quantcore-api` command (HTTP client of a running service, incl. `selftest`) +deploy/apptainer/ qc.sh (install/start/stop/cli on a Spark without root), quantcore.def, pinned constraints.txt +deploy/ Dockerfile, compose service blocks (Spark + a bench-free one), deploy_to_spark.sh +docs/ USAGE.md, ARCHITECTURE.md, METHODOLOGY.md, SPARK_APPTAINER.md, SPARK_DEPLOYMENT.md +tests/ pytest suite (synthetic data store fixture) +``` diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..a0bc145 --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,24 @@ +# quantcore -- portfolio quant service for the DGX Spark (aarch64) and any x86-64 host. +# Build context is the repository root: docker build -f deploy/Dockerfile -t local/quantcore:0.1 . +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 \ + QUANTCORE_DATA_DIR=/data QUANTCORE_CUOPT_URL=http://cuopt:5000 \ + OPENBLAS_NUM_THREADS=4 OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 +# BLAS threading: the innovation filter runs thousands of small (N x N) eigendecompositions; +# OpenBLAS with all 20 cores of the Spark is ~50x slower on those than with 1-4 threads. + +RUN apt-get update && apt-get install -y --no-install-recommends curl tini \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY quantcore ./quantcore +RUN pip install . + +VOLUME ["/data"] +EXPOSE 5001 +HEALTHCHECK --interval=60s --timeout=10s --retries=2 CMD curl -sf http://localhost:5001/health >/dev/null || exit 1 + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["uvicorn", "quantcore.api.app:app", "--host", "0.0.0.0", "--port", "5001", "--workers", "1"] diff --git a/deploy/apptainer/constraints.txt b/deploy/apptainer/constraints.txt new file mode 100644 index 0000000..3e5a72a --- /dev/null +++ b/deploy/apptainer/constraints.txt @@ -0,0 +1,48 @@ +# Library versions of the quantcore image, taken from the container that runs on the lecturer's Spark +# (Python 3.12.14, aarch64, 2026-10-07). deploy/apptainer/quantcore.def installs with `pip install -c`, +# so a student's build in December gets the same SciPy/HiGHS, pandas and numpy as the verified one. +# To move to newer libraries: delete the pins, build, run the tests, and freeze again: +# apptainer exec quantcore.sif pip freeze --exclude quantcore +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +beautifulsoup4==4.15.0 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +click==8.5.0 +curl_cffi==0.16.3 +fastapi==0.141.1 +h11==0.16.0 +httptools==0.8.0 +idna==3.20 +lxml==6.1.3 +msgpack==1.2.2 +multitasking==0.0.13 +numpy==2.5.3 +pandas==3.0.6 +peewee==4.5.1 +platformdirs==4.11.12 +protobuf==7.36.2 +pyarrow==25.0.1 +pycparser==3.0 +pydantic==2.13.5 +pydantic-settings==2.15.0 +pydantic_core==2.46.5 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.3 +pytz==2026.3.post1 +PyYAML==6.0.3 +requests==2.34.2 +scipy==1.18.1 +six==1.17.0 +soupsieve==2.9.2 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.53.0 +uvloop==0.22.1 +watchfiles==1.3.0 +websockets==17.1 +yfinance==1.7.0 diff --git a/deploy/apptainer/qc.sh b/deploy/apptainer/qc.sh new file mode 100755 index 0000000..3641892 --- /dev/null +++ b/deploy/apptainer/qc.sh @@ -0,0 +1,576 @@ +#!/usr/bin/env bash +# qc.sh -- quantcore standalone on a DGX Spark with Apptainer: no root, no Docker, no Tailscale. +# +# cuOpt and quantcore each run in their own Apptainer container on this machine and talk over +# 127.0.0.1. The API and the browser test bench listen on 127.0.0.1 too, so they are used from this +# Spark (its desktop browser, a terminal, a notebook) and are not reachable from the network. +# +# qc.sh install once: config, cuOpt image, quantcore image, data directory +# qc.sh data-import FILE copy a prepared data set (.tar.gz or directory) ... +# qc.sh setup ... or build it from Yahoo Finance (~15 min) +# qc.sh start cuOpt + API -> http://127.0.0.1:5001/ui/ and /docs +# qc.sh status | logs [api|cuopt] [-f] | stop | restart +# qc.sh cli ... the quantcore CLI in the container, no API needed +# (e.g. qc.sh cli optimize --cash 1e6 --cvar-limit 40000) +# qc.sh api ... quantcore-api against the local API (e.g. qc.sh api selftest) +# qc.sh help all commands +# +# Settings: $QC_HOME/quantcore.env (QC_HOME defaults to ~/quantcore); see quantcore.env.example. +# Guide: docs/SPARK_APPTAINER.md. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +QC_HOME="${QC_HOME:-$HOME/quantcore}" +CONFIG="$QC_HOME/quantcore.env" +APPTAINER="${APPTAINER:-apptainer}" +API_INSTANCE="quantcore-api" +CUOPT_INSTANCE="quantcore-cuopt" +ME="$(id -un)" + +# --- settings: defaults < config file < environment ------------------------------------------ +_env_overrides="$(declare -p $(compgen -v | grep -E '^(QC_|QUANTCORE_)' || true) 2>/dev/null || true)" +QC_DATA="$QC_HOME/data" +QC_IMAGES="$QC_HOME/images" +QC_API_PORT=5001 +QC_CUOPT_PORT=5000 +QC_CUOPT_START=auto +QC_CUOPT_IMAGE="docker://nvidia/cuopt:26.8.0-cuda13.3-py3.14" +QC_IMAGE=build +QC_PYTHON_IMAGE="docker://python:3.12-slim" # base of the no-build fallback (same as quantcore.def) +if [[ -f "$CONFIG" ]]; then + set -a + # shellcheck disable=SC1090 + source "$CONFIG" + set +a +fi +[[ -n "$_env_overrides" ]] && eval "$_env_overrides" +unset _env_overrides +QC_API_URL="http://127.0.0.1:$QC_API_PORT" +QC_VENV="$QC_HOME/venv" +PY_SIF="$QC_IMAGES/python.sif" +CUOPT_URL="${QUANTCORE_CUOPT_URL:-http://127.0.0.1:$QC_CUOPT_PORT}" +CUOPT_EXTERNAL=0 # 1: QUANTCORE_CUOPT_URL names a cuOpt somewhere else -- never start our own +[[ -n "${QUANTCORE_CUOPT_URL:-}" ]] && CUOPT_EXTERNAL=1 + +# --- helpers ---------------------------------------------------------------------------------- +say() { printf '%s\n' "$*"; } +step() { printf '\n>> %s\n' "$*"; } +warn() { printf 'warning: %s\n' "$*" >&2; } +die() { printf 'error: %s\n' "$*" >&2; exit 1; } + +quantcore_sif() { if [[ "$QC_IMAGE" == *.sif && -f "$QC_IMAGE" ]]; then say "$QC_IMAGE"; else say "$QC_IMAGES/quantcore.sif"; fi; } + +# How quantcore runs: "sif" = the quantcore image (built or prebuilt); "venv" = the no-build fallback, +# a plain Python image plus a virtualenv in $QC_HOME/venv (for a Spark that allows no image builds). +qc_mode() { + if [[ "$QC_IMAGE" == "venv" ]]; then say venv; return; fi + if [[ -f "$(quantcore_sif)" ]]; then say sif; return; fi + if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" ]]; then say venv; return; fi + say none +} +qc_image() { if [[ "$(qc_mode)" == venv ]]; then say "$PY_SIF"; else quantcore_sif; fi; } +qc_bin() { if [[ "$(qc_mode)" == venv ]]; then say "$QC_VENV/bin/$1"; else say "$1"; fi; } +cuopt_sif() { if [[ -f "$QC_CUOPT_IMAGE" ]]; then say "$QC_CUOPT_IMAGE"; else say "$QC_IMAGES/cuopt.sif"; fi; } + +need_apptainer() { + command -v "$APPTAINER" >/dev/null 2>&1 \ + || die "'$APPTAINER' not found. This Spark needs Apptainer (ask the administrator), or set APPTAINER=singularity." +} + +need_image() { # path what + [[ -f "$1" ]] || die "$2 image missing ($1) -- run: $0 install" +} + +# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3 (both on DGX OS). +http_get() { + local url="$1" timeout="${2:-3}" + if command -v curl >/dev/null 2>&1; then + curl -sf --max-time "$timeout" "$url" + elif command -v python3 >/dev/null 2>&1; then + python3 -c 'import sys, urllib.request as u; sys.stdout.write(u.urlopen(sys.argv[1], timeout=float(sys.argv[2])).read().decode())' \ + "$url" "$timeout" 2>/dev/null + else + return 2 + fi +} + +port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } + +free_port() { # first free port from $1 upwards + local p="$1" + while port_busy "$p"; do p=$((p + 1)); done + say "$p" +} + +instance_running() { + "$APPTAINER" instance list 2>/dev/null | awk 'NR > 1 {print $1}' | grep -qx "$1" +} + +instance_log() { # name out|err -> path of the instance log (Apptainer keeps them per host and user) + local f + for f in "$HOME/.apptainer/instances/logs/"*"/$ME/$1.$2"; do + [[ -f "$f" ]] && { say "$f"; return 0; } + done + return 1 +} + +show_log_tail() { # name + local f + for kind in err out; do + if f="$(instance_log "$1" "$kind")"; then + say "--- last lines of $f" + tail -n 25 "$f" >&2 || true + fi + done +} + +wait_for() { # url seconds label + local url="$1" limit="$2" label="$3" waited=0 + printf ' waiting for %s ' "$label" + until http_get "$url" 2 >/dev/null; do + if (( waited >= limit )); then printf ' timed out after %ss\n' "$limit"; return 1; fi + printf '.'; sleep 2; waited=$((waited + 2)) + done + printf ' up (%ss)\n' "$waited" +} + +# Hand the settings to a container through APPTAINERENV_* (unlike --env these survive commas, +# e.g. in no_proxy). The containers start with --cleanenv, so nothing else from the host leaks in: +# no PYTHONPATH, no activated virtualenv or conda, no LD_LIBRARY_PATH. +export_quantcore_env() { + local v + for v in $(compgen -v | grep '^QUANTCORE_' || true); do + export "APPTAINERENV_$v=${!v}" + done + export APPTAINERENV_QUANTCORE_DATA_DIR=/data + export APPTAINERENV_QUANTCORE_CUOPT_URL="$CUOPT_URL" + export APPTAINERENV_QUANTCORE_API_HOST=127.0.0.1 + export APPTAINERENV_QUANTCORE_API_PORT="$QC_API_PORT" + export APPTAINERENV_QUANTCORE_API_URL="$QC_API_URL" + # what quantcore.def's %environment sets, for the venv fallback (an explicit value wins) + export APPTAINERENV_PYTHONNOUSERSITE=1 APPTAINERENV_PYTHONUNBUFFERED=1 APPTAINERENV_PYTHONDONTWRITEBYTECODE=1 + export APPTAINERENV_OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-4}" APPTAINERENV_OMP_NUM_THREADS="${OMP_NUM_THREADS:-4}" + export APPTAINERENV_MKL_NUM_THREADS="${MKL_NUM_THREADS:-4}" + for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY TZ; do + [[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}" + done + return 0 +} + +# Run a command in the quantcore container (the CLI needs no running API). +qc_binds() { # bind options for the quantcore container + local b=(--bind "$QC_DATA:/data") + [[ "$(qc_mode)" == venv ]] && b+=(--bind "$QC_VENV") + printf '%s\n' "${b[@]}" +} + +run_qc() { # command [args]: quantcore | quantcore-api | python ... + need_apptainer + [[ "$(qc_mode)" != none ]] || die "no quantcore image yet -- run: $0 install" + mkdir -p "$QC_DATA" + export_quantcore_env + local cmd="$1"; shift + local binds; mapfile -t binds < <(qc_binds) + exec "$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin "$cmd")" "$@" +} + +data_is_empty() { [[ -z "$(ls -A "$QC_DATA" 2>/dev/null | grep -v -x -e jobs -e '.*\.log')" ]]; } + +# --- install ---------------------------------------------------------------------------------- +cmd_install() { + need_apptainer + local force=0; [[ "${1:-}" == "--force" ]] && force=1 + step "Apptainer" + say " $("$APPTAINER" --version)" + "$APPTAINER" instance run --help >/dev/null 2>&1 \ + || die "this Apptainer has no 'instance run' (needs Apptainer >= 1.1); ask the administrator to update it" + + step "Directories and settings ($QC_HOME)" + mkdir -p "$QC_HOME" "$QC_DATA" "$QC_IMAGES" + if [[ -f "$CONFIG" ]]; then + say " keeping $CONFIG" + else + cp "$SCRIPT_DIR/quantcore.env.example" "$CONFIG" + say " wrote $CONFIG" + fi + + step "GPU" + if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then + say " $(nvidia-smi -L | head -n 1)" + else + warn "no GPU visible (nvidia-smi). cuOpt will not start; optimisations will use HiGHS on the CPU." + fi + local free_gb; free_gb=$(( $(df -Pk "$QC_IMAGES" | awk 'NR == 2 {print $4}') / 1024 / 1024 )) + say " free disk under $QC_IMAGES: ${free_gb} GB (the cuOpt pull needs ~12 GB while it runs, 3.3 GB after)" + + if [[ "$QC_CUOPT_START" != "no" && "$CUOPT_EXTERNAL" == 0 ]]; then + cmd_pull_cuopt $([[ $force == 1 ]] && echo --force) + fi + step "quantcore image" + get_quantcore_image "$force" + step "Self-test of the quantcore container ($(qc_mode) mode)" + qc_selftest + + step "Data ($QC_DATA)" + if data_is_empty; then + say " empty. Either copy a prepared data set: $0 data-import quantcore_data.tar.gz" + say " or build it from Yahoo Finance (~15 min): $0 setup" + else + say " present ($(du -sh "$QC_DATA" | cut -f1))" + fi + step "Done. Next: $0 start" +} + +cmd_pull_cuopt() { + need_apptainer + local dst="$QC_IMAGES/cuopt.sif" + if [[ -f "$QC_CUOPT_IMAGE" ]]; then + step "cuOpt image: $QC_CUOPT_IMAGE (provided on this machine, not pulled)" + return 0 + fi + if [[ -f "$dst" && "${1:-}" != "--force" ]]; then + step "cuOpt image present: $dst" + return 0 + fi + step "Pulling cuOpt: $QC_CUOPT_IMAGE -> $dst (~3.3 GB, several minutes)" + mkdir -p "$QC_IMAGES" + rm -f "$dst.part" + "$APPTAINER" pull "$dst.part" "$QC_CUOPT_IMAGE" + mv -f "$dst.part" "$dst" +} + +qc_selftest() { + ( export_quantcore_env + local binds; mapfile -t binds < <(qc_binds) + "$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin python)" -c ' +import importlib.resources as r, scipy, quantcore, quantcore.api.app as app +from scipy.optimize import milp +assert (r.files("quantcore.api") / "ui" / "index.html").is_file(), "test bench missing" +print(" quantcore", quantcore.__version__, "| scipy", scipy.__version__, "| UI mounted:", app.UI_ENABLED)' ) +} + +get_quantcore_image() { # force(0|1) + local dst="$QC_IMAGES/quantcore.sif" + case "$QC_IMAGE" in + build) + if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst (rebuild: $0 build)"; return 0; fi + if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" && "$1" != 1 ]]; then + say " present: virtualenv $QC_VENV (no-build fallback; reinstall: $0 build)"; return 0 + fi + build_sif || { + say "" + say " The image build is not possible on this Spark without root. Falling back to the no-build" + say " setup: a plain Python image plus a virtualenv in $QC_VENV (same libraries, same results)." + build_venv + } ;; + venv) + if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" && "$1" != 1 ]]; then say " present: $QC_VENV"; else build_venv; fi ;; + docker://*|oras://*|library://*|http://*|https://*) + if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst"; else + say " pulling $QC_IMAGE -> $dst" + rm -f "$dst.part"; "$APPTAINER" pull "$dst.part" "$QC_IMAGE"; mv -f "$dst.part" "$dst" + fi ;; + *) + [[ -f "$QC_IMAGE" ]] || die "QC_IMAGE=$QC_IMAGE is neither 'build', a URL, nor an existing .sif file" + say " using $QC_IMAGE" ;; + esac +} + +build_sif() { # build quantcore.sif from the definition; non-zero when this Apptainer cannot build + local dst="$QC_IMAGES/quantcore.sif" + mkdir -p "$QC_IMAGES" + say " building $dst from $REPO_DIR (a few minutes)" + rm -f "$dst.part" + # %files paths in the definition are relative to the repository root. Without root Apptainer + # builds through --fakeroot (user namespaces); a Spark may not allow that. + if ! (cd "$REPO_DIR" && "$APPTAINER" build "$dst.part" deploy/apptainer/quantcore.def); then + rm -f "$dst.part" + return 1 + fi + mv -f "$dst.part" "$dst" + say " built $dst ($(du -h "$dst" | cut -f1))" +} + +build_venv() { # the no-build fallback: pull a plain Python image, install quantcore into a virtualenv + mkdir -p "$QC_IMAGES" "$QC_HOME" + if [[ ! -f "$PY_SIF" ]]; then + say " pulling $QC_PYTHON_IMAGE -> $PY_SIF" + rm -f "$PY_SIF.part"; "$APPTAINER" pull "$PY_SIF.part" "$QC_PYTHON_IMAGE"; mv -f "$PY_SIF.part" "$PY_SIF" + fi + say " installing quantcore into $QC_VENV (pinned versions from deploy/apptainer/constraints.txt)" + local work; work="$(mktemp -d "${TMPDIR:-/tmp}/quantcore-src.XXXXXX")" + # install from a copy: pip writes build/ and *.egg-info next to the sources + cp -r "$REPO_DIR/pyproject.toml" "$REPO_DIR/README.md" "$REPO_DIR/quantcore" "$work/" + "$APPTAINER" exec --cleanenv --bind "$QC_HOME" --bind "$work" "$PY_SIF" python -m venv --clear "$QC_VENV" + "$APPTAINER" exec --cleanenv --env PIP_DISABLE_PIP_VERSION_CHECK=1 --bind "$QC_HOME" --bind "$work" \ + --bind "$REPO_DIR/deploy/apptainer/constraints.txt:/tmp/qc-constraints.txt" "$PY_SIF" \ + "$QC_VENV/bin/pip" install --quiet --no-cache-dir -c /tmp/qc-constraints.txt "$work" + rm -rf "$work" + say " installed: $("$APPTAINER" exec --cleanenv --bind "$QC_VENV" "$PY_SIF" "$QC_VENV/bin/quantcore" --help >/dev/null && echo ok)" +} + +cmd_build() { # rebuild after a code change, in the mode in use + need_apptainer + local mode; mode="$(qc_mode)" + step "Rebuilding quantcore ($([[ $mode == venv ]] && echo "virtualenv $QC_VENV" || echo "image $QC_IMAGES/quantcore.sif"))" + if [[ "$mode" == venv ]]; then + build_venv + elif [[ "$QC_IMAGE" == build ]]; then + build_sif || die "the image build failed (see above). Without root it needs Apptainer's --fakeroot. Use the + no-build fallback instead: set QC_IMAGE=venv in $CONFIG and run: $0 build" + else + die "QC_IMAGE=$QC_IMAGE is a prebuilt image; nothing to build" + fi + if instance_running "$API_INSTANCE"; then say " the running API still uses the old code: $0 restart"; fi +} + +# --- run -------------------------------------------------------------------------------------- +start_cuopt() { + if [[ "$CUOPT_EXTERNAL" == 1 ]]; then + if http_get "$CUOPT_URL/cuopt/health" >/dev/null; then say "cuOpt: using $CUOPT_URL (QUANTCORE_CUOPT_URL)" + else warn "cuOpt at $CUOPT_URL (QUANTCORE_CUOPT_URL) does not answer -- optimisations will use HiGHS"; fi + return 0 + fi + if [[ "$QC_CUOPT_START" == "no" ]]; then + say "cuOpt: not started (QC_CUOPT_START=no) -- optimisations use HiGHS on the CPU" + return 0 + fi + if http_get "$CUOPT_URL/cuopt/health" >/dev/null; then + if instance_running "$CUOPT_INSTANCE"; then + say "cuOpt: running ($CUOPT_INSTANCE, $CUOPT_URL)" + elif [[ "$QC_CUOPT_START" == "yes" ]]; then + die "QC_CUOPT_START=yes, but another cuOpt already serves port $QC_CUOPT_PORT. Use it (QC_CUOPT_START=auto) + or pick another port: QC_CUOPT_PORT=$(free_port $((QC_CUOPT_PORT + 100))) in $CONFIG" + else + say "cuOpt: using the server that already answers at $CUOPT_URL" + fi + return 0 + fi + if instance_running "$CUOPT_INSTANCE"; then # still starting, or hung + wait_for "$CUOPT_URL/cuopt/health" 180 "cuOpt ($CUOPT_INSTANCE)" && return 0 + show_log_tail "$CUOPT_INSTANCE" + die "cuOpt instance runs but does not answer; try: $0 restart" + fi + if port_busy "$QC_CUOPT_PORT"; then + die "port $QC_CUOPT_PORT is taken by something that is not cuOpt. + Set QC_CUOPT_PORT=$(free_port $((QC_CUOPT_PORT + 100))) in $CONFIG and run: $0 start" + fi + local sif; sif="$(cuopt_sif)" + need_image "$sif" cuOpt + say "cuOpt: starting instance $CUOPT_INSTANCE on 127.0.0.1:$QC_CUOPT_PORT (GPU)" + # NVIDIA's entrypoint sets HOME=/opt/cuopt. That is writable under Docker but read-only in a SIF, + # and the solver process dies on its first job when CuPy cannot write its kernel cache there. + # So the server runs with HOME on a writable directory of ours (CuPy, Numba and CUDA caches). + local cuopt_home="$QC_HOME/cuopt-home" + mkdir -p "$cuopt_home" + ( [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]] && export APPTAINERENV_CUDA_VISIBLE_DEVICES="$CUDA_VISIBLE_DEVICES" + cd / + # the image's entrypoint runs its arguments: the cuOpt REST server, on localhost only + "$APPTAINER" instance run --nv --cleanenv --bind "$cuopt_home" "$sif" "$CUOPT_INSTANCE" \ + env HOME="$cuopt_home" python -m cuopt_server.cuopt_service \ + -i 127.0.0.1 -p "$QC_CUOPT_PORT" -l "${QC_CUOPT_LOG_LEVEL:-info}" >/dev/null ) + if ! wait_for "$CUOPT_URL/cuopt/health" 180 "cuOpt"; then + show_log_tail "$CUOPT_INSTANCE" + warn "cuOpt did not come up -- the API starts anyway and optimises with HiGHS" + fi +} + +start_api() { + if instance_running "$API_INSTANCE"; then + if http_get "$QC_API_URL/health" >/dev/null; then say "API: running ($API_INSTANCE, $QC_API_URL)"; return 0; fi + wait_for "$QC_API_URL/health" 60 "API ($API_INSTANCE)" && return 0 + show_log_tail "$API_INSTANCE" + die "the API instance runs but does not answer; try: $0 restart" + fi + if port_busy "$QC_API_PORT"; then + die "port $QC_API_PORT is taken (another user's quantcore?). + Set QC_API_PORT=$(free_port $((QC_API_PORT + 10))) in $CONFIG and run: $0 start" + fi + local mode; mode="$(qc_mode)" + [[ "$mode" != none ]] || die "no quantcore image yet -- run: $0 install" + mkdir -p "$QC_DATA" + say "API: starting instance $API_INSTANCE on $QC_API_URL ($mode mode)" + ( export_quantcore_env + local binds; mapfile -t binds < <(qc_binds) + # start from /: pydantic-settings reads a .env from the working directory, and the container's + # / has none (instances take the caller's directory; they have no --cwd option) + cd / + if [[ "$mode" == venv ]]; then # the Python image's runscript executes its arguments + "$APPTAINER" instance run --cleanenv "${binds[@]}" "$PY_SIF" "$API_INSTANCE" "$QC_VENV/bin/quantcore" serve >/dev/null + else # quantcore.def's startscript: quantcore serve + "$APPTAINER" instance start --cleanenv "${binds[@]}" "$(quantcore_sif)" "$API_INSTANCE" >/dev/null + fi ) + if ! wait_for "$QC_API_URL/health" 90 "API"; then + show_log_tail "$API_INSTANCE" + die "the API did not come up" + fi +} + +cmd_start() { + need_apptainer + step "Starting quantcore (user $ME, settings $([[ -f "$CONFIG" ]] && echo "$CONFIG" || echo "defaults"))" + start_cuopt + start_api + data_is_empty && warn "no market data yet: $0 data-import or $0 setup" + local ui=""; [[ "${QUANTCORE_ENABLE_UI:-true}" != "false" ]] && ui="$QC_API_URL/ui/ (test bench)" + step "Ready -- on this Spark:" + [[ -n "$ui" ]] && say " $ui" + say " $QC_API_URL/docs (API, Swagger)" + say " Python: QuantcoreClient(\"$QC_API_URL\") CLI: $0 api health" +} + +cmd_stop() { + need_apptainer + local name stopped=0 + for name in "$API_INSTANCE" "$CUOPT_INSTANCE"; do + if instance_running "$name"; then + "$APPTAINER" instance stop "$name" >/dev/null && say "stopped $name" && stopped=1 + fi + done + [[ $stopped == 1 ]] || say "nothing running" +} + +cmd_status() { + need_apptainer + step "Instances of $ME" + "$APPTAINER" instance list + step "cuOpt $CUOPT_URL" + local h + if h="$(http_get "$CUOPT_URL/cuopt/health")"; then say " up: $h"; else say " not answering"; fi + step "API $QC_API_URL" + if h="$(http_get "$QC_API_URL/health")"; then + if command -v python3 >/dev/null 2>&1; then + printf '%s' "$h" | python3 -c ' +import json, sys +h = json.load(sys.stdin); s = h.get("store", {}); c = h.get("cuopt", {}) +print(" up: version", h.get("version"), "| base currency", h.get("base_currency")) +print(" data:", {k: s[k] for k in list(s)[:6]}) +print(" cuOpt seen by the API:", "reachable" if c.get("reachable") else "NOT reachable -> HiGHS")' + else + say " up: $h" + fi + else + say " not answering" + fi + step "Files" + say " settings: $CONFIG $([[ -f "$CONFIG" ]] || echo '(missing: defaults in use)')" + say " data: $QC_DATA ($(du -sh "$QC_DATA" 2>/dev/null | cut -f1 || echo -))" + say " quantcore: $(qc_mode) mode, $(qc_image)$([[ $(qc_mode) == venv ]] && echo " + $QC_VENV")" + say " cuOpt: $(cuopt_sif)" +} + +cmd_logs() { + local which="${1:-api}" follow="" + [[ "${2:-}" == "-f" || "${1:-}" == "-f" ]] && follow=1 + [[ "$which" == "-f" ]] && which=api + local name="$API_INSTANCE"; [[ "$which" == "cuopt" ]] && name="$CUOPT_INSTANCE" + local out err + out="$(instance_log "$name" out || true)"; err="$(instance_log "$name" err || true)" + [[ -n "$out$err" ]] || die "no logs for $name yet" + if [[ -n "$follow" ]]; then exec tail -n 50 -F ${out:+"$out"} ${err:+"$err"}; fi + tail -n 50 ${out:+"$out"} ${err:+"$err"} +} + +# --- data ------------------------------------------------------------------------------------- +data_job() { # setup|update [args]: through the running API (which then reloads), else in-process + local job="$1"; shift + if http_get "$QC_API_URL/health" >/dev/null; then + say "the API is running: the $job job runs there, and the API reloads the data when it is done" + run_qc quantcore-api --url "$QC_API_URL" "$job" --wait "$@" + else + run_qc quantcore "$job" "$@" + fi +} + +cmd_data_import() { + local src="${1:-}" force="${2:-}" + [[ -n "$src" ]] || die "usage: $0 data-import [--force]" + [[ -e "$src" ]] || die "$src not found" + mkdir -p "$QC_DATA" + if ! data_is_empty && [[ "$force" != "--force" ]]; then + die "$QC_DATA already holds data; add --force to overwrite it" + fi + if [[ -d "$src" ]]; then + cp -a "$src/." "$QC_DATA/" + else + tar -xzf "$src" -C "$QC_DATA" + fi + say "imported into $QC_DATA ($(du -sh "$QC_DATA" | cut -f1))" + if instance_running "$API_INSTANCE"; then say "the running API still holds the old data: $0 restart"; fi +} + +cmd_data_export() { + local dst="${1:-quantcore_data.tar.gz}" + [[ "$dst" = /* ]] || dst="$PWD/$dst" + tar -czf "$dst" -C "$QC_DATA" --exclude './jobs' --exclude '*.log' . + say "wrote $dst ($(du -h "$dst" | cut -f1))" +} + +cmd_env() { + cat < [options] + The startscript serves the HTTP API: apptainer instance start quantcore.sif quantcore-api + -> http://127.0.0.1:5001/ui/ (test bench) and /docs (Swagger); bound to localhost only. + + Bind a writable data directory to /data (QUANTCORE_DATA_DIR). cuOpt is expected at + QUANTCORE_CUOPT_URL (default http://127.0.0.1:5000); without it every optimisation uses HiGHS. + All QUANTCORE_* variables of quantcore/config.py can be passed with --env. + +%files + pyproject.toml /opt/quantcore-src/pyproject.toml + README.md /opt/quantcore-src/README.md + quantcore /opt/quantcore-src/quantcore + deploy/apptainer/constraints.txt /opt/quantcore-src/constraints.txt + +%post + set -eu + export PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_ROOT_USER_ACTION=ignore + # constraints.txt pins the library versions the image was verified with (SciPy decides the HiGHS + # version, and with it which of several equally good integer solutions HiGHS returns). + python -m pip install -c /opt/quantcore-src/constraints.txt /opt/quantcore-src + rm -rf /opt/quantcore-src + find /usr/local/lib -name '__pycache__' -type d -prune -exec rm -rf {} + 2>/dev/null || true + mkdir -p /data + python -c "import quantcore, scipy; print('quantcore', quantcore.__version__, '| scipy', scipy.__version__)" + +%environment + export PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 + # a student's own pip --user packages must not shadow the image's libraries + export PYTHONNOUSERSITE=1 + unset PYTHONPATH PYTHONHOME VIRTUAL_ENV + # small-matrix linear algebra (the innovation filter) is ~50x slower with all 20 cores + export OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-4}" OMP_NUM_THREADS="${OMP_NUM_THREADS:-4}" + export MKL_NUM_THREADS="${MKL_NUM_THREADS:-4}" + export QUANTCORE_DATA_DIR="${QUANTCORE_DATA_DIR:-/data}" + export QUANTCORE_CUOPT_URL="${QUANTCORE_CUOPT_URL:-http://127.0.0.1:5000}" + export QUANTCORE_API_HOST="${QUANTCORE_API_HOST:-127.0.0.1}" + export QUANTCORE_API_PORT="${QUANTCORE_API_PORT:-5001}" + +%runscript + exec quantcore "$@" + +%startscript + exec quantcore serve + +%test + set -eu + python - <<'PY' +import importlib.resources as r +import quantcore, quantcore.api.app as app # imports FastAPI app, all services and the UI mount +from scipy.optimize import milp # HiGHS, the CPU fallback solver +assert (r.files("quantcore.api") / "ui" / "index.html").is_file(), "test bench missing from the wheel" +print("quantcore", quantcore.__version__, "ok; UI mounted:", app.UI_ENABLED) +PY diff --git a/deploy/apptainer/quantcore.env.example b/deploy/apptainer/quantcore.env.example new file mode 100644 index 0000000..d92303c --- /dev/null +++ b/deploy/apptainer/quantcore.env.example @@ -0,0 +1,40 @@ +# Settings of deploy/apptainer/qc.sh -- quantcore standalone on a DGX Spark with Apptainer. +# `qc.sh install` copies this file to $QC_HOME/quantcore.env (QC_HOME defaults to ~/quantcore). +# qc.sh reads it on every call; after a change run `qc.sh restart`. A variable that is also set in +# the shell environment wins over this file (e.g. `QC_API_PORT=5011 qc.sh start`). + +# --- where things live ------------------------------------------------------------------------ +#QC_DATA=$HOME/quantcore/data # market data: universe, prices, FX, snapshots (~40 MB) +#QC_IMAGES=$HOME/quantcore/images # quantcore.sif (~0.2 GB) and cuopt.sif (~3.3 GB) + +# --- network: nothing leaves this machine ----------------------------------------------------- +# Both servers listen on 127.0.0.1 only. Ports are shared by every user of this Spark: if +# `qc.sh start` reports a port as taken, pick another one here (it suggests a free one). +QC_API_PORT=5001 # quantcore API + test bench: http://127.0.0.1:5001/ui/ +QC_CUOPT_PORT=5000 # cuOpt server + +# --- cuOpt: the GPU MILP solver, in its own container ----------------------------------------- +# auto: use a cuOpt that already answers on QC_CUOPT_PORT (another user's or a shared one), +# otherwise start our own instance on the GPU +# yes: always start our own instance (fails if the port is taken) +# no: no cuOpt -- every optimisation runs on HiGHS (CPU, same model, same answer) +QC_CUOPT_START=auto +# Official NVIDIA image (arm64 + amd64), pulled once without root. If the Spark already provides a +# cuOpt image, put the path of its .sif file here instead. +QC_CUOPT_IMAGE=docker://nvidia/cuopt:26.8.0-cuda13.3-py3.14 + +# --- the quantcore image ---------------------------------------------------------------------- +# build: build it from this repository (Apptainer's unprivileged --fakeroot build); if this +# Spark does not allow builds, install falls back to 'venv' by itself +# venv: no build: a plain Python image + a virtualenv in $QC_HOME/venv (same libraries) +# /path/x.sif: a prebuilt image (e.g. handed out by the lecturer) -- used in place, not copied +# docker://... or oras://... to pull a published one +QC_IMAGE=build + +# --- passed unchanged into the quantcore container (see quantcore/config.py) ------------------- +QUANTCORE_CUOPT_TIME_LIMIT=300 # seconds per MILP +QUANTCORE_ENABLE_UI=true # false: JSON API and /docs only, no test bench at /ui/ +QUANTCORE_LOG_LEVEL=INFO +#QUANTCORE_BASE_CURRENCY=CHF +#QUANTCORE_CUOPT_URL=http://127.0.0.1:5000 # set only to use a cuOpt at another address; then + # qc.sh never starts its own diff --git a/deploy/deploy_to_spark.sh b/deploy/deploy_to_spark.sh new file mode 100755 index 0000000..343bcd0 --- /dev/null +++ b/deploy/deploy_to_spark.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Deploy the quant service to the DGX Spark: +# 1. rsync the repository to ~/services/work/Portfolio_Risk_Optimization +# 2. optionally rsync an already built data directory (saves the 15-minute setup on the Spark) +# 3. add the `quantcore` service block to ~/services/docker-compose.yml (idempotent) +# 4. build the image and start the container +# +# Usage: deploy/deploy_to_spark.sh [--with-data] [--host user_fh_fhgr@spark-6d4a] +set -euo pipefail + +HOST="user_fh_fhgr@spark-6d4a" +WITH_DATA=0 +for arg in "$@"; do + case "$arg" in + --with-data) WITH_DATA=1 ;; + --host=*) HOST="${arg#--host=}" ;; + --host) shift; HOST="$1" ;; + esac +done + +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" +REMOTE_SERVICES="services" +REMOTE_REPO="$REMOTE_SERVICES/work/Portfolio_Risk_Optimization" +REMOTE_DATA="$REMOTE_SERVICES/work/quantcore_data" + +echo ">> rsync repository -> $HOST:$REMOTE_REPO" +ssh "$HOST" "mkdir -p $REMOTE_REPO $REMOTE_DATA" +rsync -az --delete \ + --exclude '.git' --exclude '/data/' --exclude '__pycache__' --exclude '*.pyc' --exclude '.pytest_cache' \ + --exclude '*.egg-info' --exclude '.ruff_cache' --exclude 'scratch/' \ + "$REPO_DIR/" "$HOST:$REMOTE_REPO/" + +if [[ "$WITH_DATA" == "1" && -d "$REPO_DIR/data" ]]; then + echo ">> rsync data directory -> $HOST:$REMOTE_DATA" + rsync -az --exclude 'jobs/' --exclude '*.log' "$REPO_DIR/data/" "$HOST:$REMOTE_DATA/" +fi + +echo ">> register the service in ~/services/docker-compose.yml (if missing)" +ssh "$HOST" bash -s <<'REMOTE' +set -euo pipefail +cd ~/services +if ! grep -q '^ quantcore:' docker-compose.yml; then + cp docker-compose.yml "docker-compose.yml.bak.$(date +%Y%m%d%H%M%S)" + # append the service block (everything below "services:" of the snippet) + awk 'f{print} /^services:/{f=1}' work/Portfolio_Risk_Optimization/deploy/docker-compose.quantcore.yml >> docker-compose.yml + echo " added quantcore service block" +else + echo " quantcore service already present" +fi +docker compose build quantcore +docker compose up -d quantcore +sleep 3 +docker compose ps quantcore +curl -s http://localhost:5001/health | head -c 600; echo +REMOTE + +echo ">> done. API: http://spark-6d4a:5001/docs" diff --git a/deploy/docker-compose.quantcore.yml b/deploy/docker-compose.quantcore.yml new file mode 100644 index 0000000..5a4ebc2 --- /dev/null +++ b/deploy/docker-compose.quantcore.yml @@ -0,0 +1,26 @@ +# Service block for the DGX Spark's ~/services/docker-compose.yml (deploy/deploy_to_spark.sh +# appends it). Standalone use: docker compose -f deploy/docker-compose.quantcore.yml up -d +# +# The service reaches cuOpt through the compose network (http://cuopt:5000) and keeps all its +# data under ~/services/work/quantcore_data on the host (survives rebuilds and restarts). +services: + quantcore: + build: + context: ./work/Portfolio_Risk_Optimization # the rsynced repository + dockerfile: deploy/Dockerfile + image: local/quantcore:0.1 + container_name: quantcore + restart: unless-stopped + ports: ["5001:5001"] + environment: + QUANTCORE_DATA_DIR: /data + QUANTCORE_CUOPT_URL: http://cuopt:5000 + QUANTCORE_CUOPT_TIME_LIMIT: "300" + QUANTCORE_LOG_LEVEL: INFO + QUANTCORE_ENABLE_UI: "true" # "false" for an installation whose users bring their own front end + OPENBLAS_NUM_THREADS: "4" # small-matrix linear algebra is much slower with 20 threads + OMP_NUM_THREADS: "4" + volumes: + - ./work/quantcore_data:/data + depends_on: + - cuopt diff --git a/deploy/docker-compose.students.yml b/deploy/docker-compose.students.yml new file mode 100644 index 0000000..2762f21 --- /dev/null +++ b/deploy/docker-compose.students.yml @@ -0,0 +1,30 @@ +# quantcore for a platform team: the JSON API and /docs, no test bench. +# +# docker compose -f deploy/docker-compose.students.yml up -d --build +# curl http://localhost:5001/health +# +# The MILP solver is optional. Point QUANTCORE_CUOPT_URL at a cuOpt server if you have one (on a DGX +# Spark: http://:5000); if it is unreachable, every optimisation falls back to HiGHS on the CPU +# -- slower on large problems, same model, same answer, and the result says which solver produced it. +# +# The data directory is mounted, not baked in: run the setup once (POST /jobs/setup, ~15 min) or copy +# a prepared ./data next to this file. +services: + quantcore: + build: + context: .. + dockerfile: deploy/Dockerfile + image: quantcore:0.1 + container_name: quantcore + restart: unless-stopped + ports: ["5001:5001"] + environment: + QUANTCORE_DATA_DIR: /data + QUANTCORE_CUOPT_URL: ${QUANTCORE_CUOPT_URL:-http://cuopt:5000} + QUANTCORE_CUOPT_TIME_LIMIT: "300" + QUANTCORE_ENABLE_UI: "false" # the test bench is not part of the deliverable + QUANTCORE_LOG_LEVEL: INFO + OPENBLAS_NUM_THREADS: "4" # small-matrix linear algebra is much slower with all cores + OMP_NUM_THREADS: "4" + volumes: + - ../data:/data diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..3033e02 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,125 @@ +# Architecture + +`quantcore` is the **deterministic core** of the platform sketched in the AISE502 project brief: +a set of services with plain contracts, no LLM inside, resilient against external data outages, +and reproducible through pinned data snapshots. The AI layer (news ingestion, `ResearchAgent`, +`AdvisorAgent`, …) sits *on top* and only reads these services through their contracts. + +``` + ┌──────────────────────────────────────────────────────────────────┐ + HTTP/JSON │ quantcore.api (FastAPI) – thin, no logic, background jobs │ + └───────┬───────────────┬───────────────┬──────────────┬───────────┘ + │ │ │ │ + ┌─────────────────▼──┐ ┌─────────▼─────────┐ ┌──▼──────────┐ ┌▼────────────────────┐ + │ SetupService │ │ RiskService │ │ Performance │ │ OptimizationService │ + │ DataDownloadService │ │ (measurement.risk) │ │ Service │ │ (optimization) │ + └─────────┬───────────┘ └─────────┬─────────┘ └──┬──────────┘ └┬───────────┬────────┘ + │ │ │ │ │ + ┌─────────▼───────────┐ ┌─────────▼───────────────▼──────────┐ │ ┌───────▼────────────┐ + │ YahooClient (ext.) │ │ MarketDataService │◄──┘ │ models: │ + │ timeout · retry · │ │ prices · FX → CHF · returns · │ │ CovarianceModel │ + │ circuit breaker │ │ market caps · cap-weighted index │ │ GBM / innovations │ + └─────────┬───────────┘ └─────────┬──────────────────────────┘ │ Black-Litterman │ + │ │ └───────┬────────────┘ + ┌─────────▼────────────────────────▼──────────┐ ┌────────────────▼────────────┐ + │ DataStore (parquet/csv under data_dir, │ │ MeanCVaRProblem → MILPModel │ + │ manifest, snapshots) │ │ CuOptSolver ⇄ HighsSolver │ + └─────────────────────────────────────────────┘ └──────────────┬──────────────┘ + │ REST + ┌─────────────▼─────────────┐ + │ cuOpt server (DGX Spark) │ + └───────────────────────────┘ +``` + +## Bounded contexts → modules + +| Context | Module | Contract (inputs → outputs) | +|---|---|---| +| Universe | `quantcore.universe` | web candidate lists → `universe.csv` (symbol, name, region, country, currency, exchange, GICS sector, Yahoo sector/industry, market cap local & CHF, shares outstanding, first price date, coverage) | +| Market data | `quantcore.data` | Yahoo → `adj_close.parquet` (date × symbol, listing currency), `fx_chf.parquet` (date × currency); `MarketDataService.prices/returns(symbols, start, end, currency, freq)` → aligned frames | +| Measurement | `quantcore.measurement` | returns frame → risk tables, series, matrices, `PortfolioRisk`; performance tables | +| Models | `quantcore.models` | returns frame → covariance, `ScenarioSet` (S × N horizon log-returns, optional probabilities, fingerprint; `sampling`: Sobol'/moment matching/importance sampling), `BlackLittermanResult` | +| Optimisation | `quantcore.optimization` | `OptimizationRequest` → `OptimizationResult` (shares, trades, cash, costs, CVaR, exposures, provenance of every model input, scenario hash, reproducibility level); `problem` (MILP, weighted scenarios or Gaussian cuts), `rowgen` (tail-row generation), `gaussian_exact` (outer approximation), `solvers` | +| Interface | `quantcore.api`, `quantcore.cli` | JSON/CLI surfaces of the above; no computation of their own | +| Test bench | `quantcore/api/ui` | a static single-page client of the JSON API (`/ui/`): stock data, covariance and scenarios, views, constraints, optimisation with before/after. Static HTML + ES modules, no build step and no CDN, so it also runs in the offline container. It formats and charts what the services return and computes nothing itself | + +Dependencies point downwards only: `api → optimization → models → measurement → data → config`. +`measurement` and `models` never touch storage or the network; they operate on DataFrames handed +to them, which is what makes them pure and testable with reference vectors. + +## Deterministic by construction + +* Every stochastic step takes an explicit `seed` (scenario generation: scrambled Sobol' points, + moment matched — the same request yields the same scenario matrix, fingerprinted as + `scenario_hash`; bit-identical within one environment, while across environments the *key* matches + and the digest need not (§9 of `docs/METHODOLOGY.md` measures why); the *scenario key* lists + exactly what it depends on — risk limits, constraints, + costs, objective, cash, holdings and views are not part of it, so a family of runs is compared on + one set, which the service also caches) and the same portfolio with HiGHS (`reproducibility="exact"`); on cuOpt the + figures agree within its MIP gap of 1e-4 (`"within_gap"`). `scenario_model="gaussian_exact"` needs + no scenarios at all. +* Every result carries the *provenance* of its inputs: covariance model, scenario model and seed, + expected-return model (incl. the views and their calibrated Ω), constraints, costs, solver, status, + gap, timings. An advisor can cite “CVaR 95 % = CHF 38 210 (OptimizationService, empirical model, + EWM λ=0.99, seed 42)”. +* Pinned snapshots (`DataStore.create_snapshot`, `MarketDataService(settings, snapshot=...)`) freeze + universe, prices and FX so that demos and grading are reproducible even if Yahoo misbehaves. + +## Where the solving happens + +The MILP is built in `quantcore.optimization.problem` and handed to one of two solvers, both fed the +same `MILPModel`: + +* **cuOpt** on the GPU, over HTTP to the cuOpt container on the same Spark (`QUANTCORE_CUOPT_URL`): + `http://127.0.0.1:5000` in the Apptainer setup, where both containers share the host's network, + and `http://cuopt:5000` inside the Docker compose network. The only GPU component; an external + dependency, hence the timeouts, retries and circuit breaker. +* **HiGHS** on the CPU, *in the same process as the service* — it is compiled into SciPy + (`scipy.optimize.milp`), so there is nothing to deploy or start, and no network hop. In production + that means inside the `quantcore` container (SciPy 1.18.1 → HiGHS 1.12.0, 4 threads). + +`solver="auto"` tries cuOpt and falls back to HiGHS; the result names the solver that produced it and +how reproducible it is. See `docs/METHODOLOGY.md` §9 for versions and the measured comparison. + +## Resilience on every external call + +| External dependency | Pattern | Fallback | +|---|---|---| +| Yahoo Finance (`YahooClient`) | per-request timeout, 4 retries with exponential back-off, circuit breaker (5 failures → 60 s open) | the stored data / a snapshot; the API keeps serving from the store | +| Candidate lists (Wikipedia, iShares) | timeout, one source after the other | static snapshots shipped in `quantcore/data/static/` | +| cuOpt (`CuOptSolver`) | submit with retry/back-off, polling with client-side deadline, circuit breaker, results deleted server-side | local HiGHS MILP (scipy) — the result says `solver: "highs"` and carries a warning | + +## Data model conventions + +* prices: adjusted close (splits & dividends), one column per Yahoo symbol, union calendar of all + exchanges, holidays forward-filled (≤ 5 days); pence-quoted London lines converted to GBP. +* FX: cross rates against the base currency, repaired for quoting-unit changes (`clean_fx`) — + a rate returned 100× off for a stretch of days would otherwise be read as a 100× move of + every stock in that currency (see `docs/METHODOLOGY.md` §1). +* returns: natural logarithms; CHF return = local return + FX return (additive). +* frequencies: daily, weekly (Friday close), monthly (month end); horizons: day = 1, week = 5, + month = 21 trading days (or 1 / 4 weeks at weekly frequency). +* money: everything in the base currency (CHF); positions are **integer share numbers**. + +## Where the AI layer plugs in + +* `ResearchAgent` insights (ticker, sentiment ∈ [−1, 1], degree of belief) become Black–Litterman + views through `view_from_sentiment` / the `views` field of `/optimize` — the numbers (view return, + Ω, posterior) are computed here, the agent only supplies the assessment and its provenance. +* `RiskAgent` / `OptimizationAgent` call `/risk/*` and `/optimize` and *explain* the returned figures. +* The ontology guard can validate tickers and sectors against `/universe`. + +## Judging the system by hand + +The test bench at `/ui/` exists so that every model choice can be exercised without writing code: +the same scenario set can be seen as a distribution, as per-stock moments and as the risk figures of +an optimised portfolio, and a run's provenance (scenario key and hash, solver, gap, reproducibility) +is shown next to its result. It is also the quickest way to see the invariants hold — two identical +requests share a scenario hash and produce identical shares, and a family of runs that varies only +the risk limit or the constraints reuses one scenario set. + +## Operations + +* `quantcore setup` (once), `quantcore update` (scheduled, e.g. nightly), `quantcore snapshot`. +* On the Spark the API runs as the `quantcore` container next to `cuopt`, `gurobi`, `jupyter` + (see `docs/SPARK_DEPLOYMENT.md`); data lives in `~/services/work/quantcore_data`. diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md new file mode 100644 index 0000000..d323b44 --- /dev/null +++ b/docs/METHODOLOGY.md @@ -0,0 +1,437 @@ +# Methodology + +Every formula the services implement, with the conventions and default parameters. Section +numbers follow the project brief ("Portfolio Risk Measurement and Optimization"). + +## 1. Investment universe (`quantcore.universe`) + +* **Candidates.** US: the S&P 500 constituents (Wikipedia). Europe (EU, CH, NO, UK): the STOXX + Europe 600 constituents, read from the iShares STOXX Europe 600 UCITS ETF holdings file, which + gives issuer ticker, exchange, listing currency, country and GICS sector. Domiciles outside + EU/CH/NO/UK (e.g. Bermuda, Israel) are excluded. Both lists are snapshotted into + `quantcore/data/static/` for offline use. +* **Yahoo symbols.** `` per exchange (`.SW .L .DE .PA .AS .MI .MC .ST .CO .OL .HE + .BR .VI .WA .IR .LS .AT`), share classes with a dash (`NOVO-B.CO`, `BRK-B`), London codes lose + the trailing dot (`RR.` → `RR.L`). All 1 117 symbols were verified against Yahoo, and the + European prices were cross-checked against the ETF file (after FX) — zero mismatches. +* **Metadata** from `Ticker.info`: long name, domicile, listing currency (GBp → GBP with a price + scale of 100), exchange, Yahoo sector/industry, market cap, shares outstanding, first trade date. +* **Selection.** One listing per company (share classes collapsed on a normalised name, largest cap + kept) → rank by market cap in CHF within each region → keep the largest 500 US and 500 European + → drop stocks whose first price is after `required_history_from` (2005-01-31) or whose coverage + since then is below 90 % of union-calendar days. The result has fewer than 1 000 names. +* **Market capitalisation** (for index weights) at any date: shares outstanding (today's, held + constant) × adjusted close × FX to CHF. +* **Price cleaning** (`quantcore.data.cleaning`, applied by the setup and download services and + re-runnable with `quantcore clean`): non-positive adjusted closes (broken dividend/rights + adjustments, e.g. ABI.BR before 2008) → NaN; *spike blocks* — at most three days whose entry + move `|r| > 0.4` is reversed within the block (stale pre-split ticks on exchange holidays, e.g. + 2× ticks in NOVO-B.CO) → NaN. Genuine jumps are not reversed and therefore untouched. Stocks + whose cleaned series has a gap longer than `max_missing_run` (20 business days) after + 2005-01-31 are excluded from the universe. On the 2026-09-21 data: 1 347 non-positive values in + 2 symbols, 28 spike ticks in 8 symbols; final universe 745 stocks (382 US, 363 European). +* **FX quoting-unit repair** (`clean_fx`, same module and the same two entry points): Yahoo returns + a cross rate in the wrong unit for whole stretches of days. NOK/CHF came back **100× too small on + 96 observations** — 2016-08-15 to 2017-01-09 in three runs, plus 2022-11-04 — which put the + measured volatility of the *currency* at 337 % a year instead of 12 %, and with it every Norwegian + stock (a CHF return is the local return plus the FX return, so EQNR.OL measured 338 % against its + true 31 %, and its Black–Litterman implied return came out above 100 % a year). A day-on-day + log-change within 0.15 of `k·ln(10)` is treated as a unit change, accumulated, and the series is + put back on the scale that most of its observations share (ties go to the scale it ends on). + The tolerance is tight on purpose: a unit change is *exactly* a power of ten apart from the day's + own move, while a loose tolerance swallows real events — at 0.5 it flagged ABVX.PA's +510 % + trial-result day (ratio 6.1) as a decimal shift. A scan of all 1 117 price series with the tight + rule finds one further case, ROSE.L (11 observations quoted in pence rather than GBP), which is + not in the investable universe. +* **FX spike ticks** (same function, thresholds an order of magnitude tighter than for prices: + entry move `|r| > 0.05` reversed within 0.02 over at most three days). Five ticks in twenty-one + years: EUR, GBP and USD on 2009-02-06 (one bad CHF quote hits every cross that day), SEK on + 2010-09-13, NOK on 2022-10-10. Removing them takes the lag-1 autocorrelation of daily FX + log-returns to about zero — EUR −0.053 → −0.005, GBP −0.040 → −0.001, USD −0.034 → −0.002 — which + is what a clean daily close series looks like. The one violent day in the sample, 2015-01-16 + (the SNB abandons the euro floor, every CHF cross moves ~18 %), does not reverse and is kept. + +**Why the rates are taken directly against CHF and not crossed through USD.** USD pairs are the most +liquid instruments, so triangulating `CUR/CHF = (USD/CHF) / (USD/CUR)` looks attractive; measured on +this feed it is worse, and the package therefore uses the direct pair wherever it has history (the +USD cross remains the fallback, and is what PLN uses, since `PLNCHF=X` only starts in 2026): + +| | direct `CURCHF=X` | crossed through USD | +|---|---|---| +| EUR volatility p.a. / lag-1 AC | 7.49 % / −0.005 | 9.99 % / **−0.219** | +| NOK volatility p.a. / lag-1 AC | 11.45 % / −0.047 | 17.14 % / **−0.330** | +| GBP volatility p.a. | 10.10 % | 10.35 % | +| reversing bad ticks, 2005–2026 | 5 over all currencies | 8, five of them in `EUR=X` alone | + +The cross combines two series snapped at different times, and the non-synchronicity shows up as a +strongly negative lag-1 autocorrelation and inflated variance — a portfolio risk figure built on it +would be overstated. It also has its own bad ticks (`NOK=X` falls 33 % on 2020-03-20 and rises 42 % +back on 2020-03-23; `EUR=X` moves −16 % then +14 % on 2008-12-08/09). Triangulation buys only the +removal of stale quotes (SEK 3.9 % of days unchanged, DKK 1.6 %), which costs far less than it adds. + +## 2. Prices, FX and log-returns (`quantcore.data`, `quantcore.measurement.returns`) + +* Adjusted closes `P_i(t)` in the listing currency on the union calendar of all exchanges, holidays + forward-filled (≤ 5 days; leading NaNs untouched). +* FX frame `X_c(t)` = CHF per unit of currency `c`, from Yahoo `cCHF=X`, crossed through USD when + the direct pair lacks history (`c/CHF = (USD/CHF)/(USD/c)`, e.g. PLN). +* Log-returns `r_i(t) = ln P_i(t) − ln P_i(t−1)`; CHF returns add the FX log-return: + `r_i^CHF(t) = r_i(t) + ln X_{c(i)}(t) − ln X_{c(i)}(t−1)`. +* Weekly returns: Friday-to-Friday closes; monthly: month-end to month-end. +* Horizon `h` in periods: day = 1, week = 5, month = 21 (daily data); at weekly frequency week = 1, + month = 4. Returns are assumed i.i.d., so `h`-period returns are sums. +* Portfolio log-return with weights `w` fixed at the start of the period (exact aggregation): + `r_p = ln(1 + Σ_i w_i (e^{r_i} − 1))`. + +## 3. Estimators (`quantcore.measurement.estimators`) + +Two families, both centred (the corresponding mean is subtracted): + +* **Moving window** of `w` observations (default 200 days): sample mean, standard deviation + (ddof 1), covariance `np.cov`, correlation. +* **Exponentially weighted** with decay `λ` (defaults 0.94 for measurement, 0.99 for the slow + optimisation covariance): weight of the observation `k` periods back ∝ `λ^k`, normalised to one + (pandas `ewm(alpha=1−λ, adjust=True)`); covariance + `Σ = Σ_t ω_t (r_t − m)(r_t − m)'`, `m = Σ_t ω_t r_t`. Effective window ≈ `1/(1−λ)`. +* Streamed sequences `Σ_{t−1}` for every `t` (needed by the innovation filter): the rolling + estimator recomputes the window, the EWM estimator uses the exact centred recursion + `m_t = m_{t−1} + α d`, `Σ_t = λ Σ_{t−1} + α d (r_t − m_t)'`, `d = r_t − m_{t−1}`, `α = 1 − λ`. +* Linear shrinkage towards the diagonal `(1−s)Σ + s·diag(Σ)` with `s` either given or `"auto"`: + the Schäfer–Strimmer / Ledoit–Wolf intensity + `s* = Σ_{i≠j} Var̂(s_ij) / Σ_{i≠j} s_ij²` (clipped to [0, 1]) computed on the effective estimation + window (`w` observations, or `3/(1−λ)` for EWM). It is ≈ 0 for a long, strongly correlated + sample and → 1 when `N` approaches the number of observations — exactly when the sample + covariance becomes ill-conditioned. `"auto"` is the default of the optimisation covariance + (≈ 0.05 for 100 stocks / λ = 0.99, ≈ 0.11 for the full universe). +* Annualisation: means × periods per year (252 / 52 / 12), volatilities × √(periods per year). + +## 4. Risk measures (`quantcore.measurement.risk`) + +Losses are positive. For a log-return `r` over the horizon the loss is `−r` in return space and +`V₀(1 − e^{r})` in CHF (exact, monotone mapping — the CHF VaR/CVaR are obtained by applying it to +the return-space figures, or directly from money P&L for scenario sets). + +* **Gaussian (GBM)**: with per-period mean `μ` and volatility `σ` from either estimator, + `μ_h = hμ`, `σ_h = √h σ`, + `VaR_β = −μ_h + z_β σ_h`, `CVaR_β = −μ_h + σ_h φ(z_β)/(1−β)`. + Reference: `σ = 1 %`, `μ = 0`: VaR₉₅ = 1.645 %, CVaR₉₅ = 2.063 %, VaR₉₉ = 2.326 %, CVaR₉₉ = 2.665 %. +* **Historical**: `VaR_β = −q_{1−β}(r_h)`, `CVaR_β = −E[r_h | r_h ≤ q_{1−β}]` on the last + `max(5w, 750)` observations aggregated to overlapping `h`-period sums. +* **Time series** (`risk_series`): rolling / EWM volatilities and the Gaussian VaR/CVaR built from + them, plus rolling historical quantiles and tail means. +* **Portfolio** (`portfolio_risk`): holdings in shares (+ cash) → value weights `w` at today's CHF + prices; Gaussian with `σ_p² = w'Σw`, historical on the realised portfolio return, or on a scenario + set (`method="scenarios"`). +* **Scenario CVaR** (`scenario_cvar`): with `S` equiprobable losses sorted descending and + `k = (1−β)S`, `CVaR_β = (Σ_{s≤⌊k⌋} L_(s) + (k−⌊k⌋) L_(⌊k⌋+1)) / k` — exactly what the + Rockafellar–Uryasev LP reproduces; `VaR_β = L_(⌈k⌉)`. + +## 5. Performance (`quantcore.measurement.performance`) + +Cumulative return `e^{Σr} − 1`; annualised log-return `ppy · mean(r)` and its geometric version +`e^{ppy·mean} − 1`; annualised volatility; Sharpe `(ann. return − r_f)/ann. vol` with the CHF +risk-free rate (`QUANTCORE_RISK_FREE_RATE`, default 0); Sortino (downside deviation); drawdown of +the cumulative value path, max drawdown, Calmar; skewness, excess kurtosis, hit ratio, best/worst +period. Rolling (window) and EWM (λ) versions as time series. Portfolios in shares: constant-mix +(today's value weights) or buy-and-hold. + +## 6. Stock-price models (`quantcore.models.simulation`) + +Both assume i.i.d. one-period log-returns; `Σ_T` is the latest matrix of the chosen +`CovarianceModel` (e.g. 200-day equally weighted, or EWM λ = 0.99). + +* **Geometric Brownian motion**: `r_h ~ N(hm, hΣ_T)`, sampled with `A z`, `A A' = Σ_T` + (eigen-root with eigenvalues descending by default, so the first draw coordinate drives the market + factor; symmetric and Cholesky roots optional). +* **Empirical innovations** (filtered historical simulation): + `ε_t = Σ_{t−1}^{−1/2} r_t` with `Σ_{t−1}` estimated from data up to `t−1` (streamed sequence, + symmetric root `V Λ^{−1/2} V'` — Cholesky optional for large `N`); forward simulation + `r(j | T) = hm + Σ_T^{1/2} Σ_{k=1..h} ε_{j_k}` with the dates `j_k` drawn with replacement from + the history (scrambled Sobol' points on the market-factor ranks by default, `"mc"` uniform or + `"balanced"` optional). Innovations are centred so the scenario mean equals the supplied drift. + **Re-standardisation**: a lagged volatility filter leaves the innovations with variance above + one (the forecast is too low right after every shock — 1.1 for 10 stocks, 1.5 for 100 with + λ = 0.99), so the centred sample is rescaled before resampling: `standardize="diagonal"` + (default) divides each asset's innovations by their standard deviation, `"full"` whitens with + the innovations' sample covariance (needs `T ≥ 3N`). The scenario covariance then reproduces + `hΣ_T` (per-asset volatility within ±2 %, exactly with full whitening) while skewness, fat + tails and non-linear co-movements of the data are preserved. Innovations beyond ±8 standard + deviations are winsorised (a numerical guard affecting ≈ 0.2 % of the values); ill-conditioned + `Σ_{t−1}` for large `N` is handled by the automatic shrinkage of Section 3. +* **Sampling and moment matching** (`quantcore.models.sampling`): the optimiser is a sample-average + approximation and the sample mean of a plain Monte-Carlo set is off by `σ/√S` per asset — 3–13 % + p.a. for `S = 2000` — which the optimiser maximises (measured: two seeds give portfolios differing + by 100 % of wealth). Therefore: scrambled Sobol' points (`S` rounded up to a power of two), + **moment matching** `z̃ = C^{-1/2}(z − z̄)` so that the (probability-weighted) scenario mean and + covariance are *exactly* `hm` and `hΣ_T`, and the model expected returns in the optimiser's + objective. Only the shape of the tail remains sampled. +* **Drift convention**: the expected-return model delivers *arithmetic* expected returns `μ` + (per period, annual / 252). The log-drift is the Itô-corrected `m_i = ln(1 + hμ_i)/h − σ_i²/2`, + so that `E[e^{r_h} − 1] = hμ` exactly (`drift_convention="arithmetic"`); `"log"` uses `μ` as the + log-drift and adds `σ²/2` to the expected simple return (5 % p.a. for a typical stock, 34 % for + the most volatile one — an unintended tilt to volatile stocks). +* **Importance sampling** (`importance_sampling="auto"` = `"market"`): the loss direction `d` of + the *cap-weighted candidate portfolio*; two blocks of 25 % of the scenarios are shifted along it + by `z_0.95` and `z_0.99` (GBM: `z ↦ z + c_k d`; empirical: dates drawn with exponentially tilted + probabilities `q_t ∝ exp(ϑ_k d'ε_t)`), so one set serves both confidence levels. Every scenario + carries the self-normalised likelihood ratio of the defensive mixture (bounded by + `1/(1 − Σ mixture)`) as its probability `p_s`. Tail noise falls 2–4×; 2 048 tilted scenarios ≈ + 8 192 plain ones at β = 0.99. The shift is driven by the *covariance*: in return space it is + `−c_k Σv/√(v'Σv)`, i.e. each asset moves by its covariance with the reference portfolio (beta), + not by its own volatility — on the top 30 the correlation of the shift with beta is −1.00 versus + −0.93 with own volatility, and a negatively correlated stock is shifted *up*. The weighted + tilted set still reproduces `hΣ` and all correlations exactly (the unweighted one does not). The + gain a given portfolio gets scales with the correlation between its loss and the reference's + (`tilt_alignment` in the result: 5.3× at 1.00, 2.3× at 0.90 — the usual value for the optimiser's + solutions — and nothing below ~0.3, where a warning is raised). `"adaptive"` tilts towards a first-pass solution (sharper at 99 %, + but the set then depends on the risk parameters); `"off"` disables the tilt. +* **One scenario set per family of runs**: the scenario *key* (`OptimizationService.scenario_key`, + reported with every result) contains only model inputs — candidates, as-of date, frequency and + horizon, covariance model, scenario model, `n_scenarios`, sampling, moment matching, seed, tilt + mode, innovation settings. The CVaR limit, confidence, objective, constraints, costs, cash, + holdings and the expected-return model are *not* in it: varying them reuses the same scenarios + (cached in the service), so differences between runs are the effect of the parameter and not + sampling noise. The drift is applied after the random part (`ScenarioSet.with_drift`), so a new + view shifts the scenarios without touching the shocks (`shock_hash` is unchanged). +* `ScenarioSet` reports `expected_simple` (the expectation the optimiser uses), `probabilities`, + the effective sample size and a SHA-256 fingerprint `hash` of the matrix. + +## 7. Expected returns — Black–Litterman with Idzorek confidence (`quantcore.models.black_litterman`) + +Weekly CHF log-returns of the candidate stocks over the last 10 years (`bl_years`): + +* `Σ` = 10-year sample covariance of weekly returns; `w` = current market-cap weights of the + candidates; `μ_M` = average weekly log-return of the **cap-weighted broad market index** of the + whole universe (weights drift with prices between rebalancings, today's shares held constant); + `r_f` = CHF risk-free rate per week. +* Risk aversion `δ = (μ_M − r_f)/(w'Σw)`; equilibrium excess returns `π = δ Σ w` (so the market + portfolio earns exactly `μ_M − r_f`). +* Views `P μ = Q + ε`, `ε ~ N(0, Ω)`, Ω diagonal. A view is `{assets: {ticker: weight}, q, confidence}`; + `q` is the annual total return of the view portfolio (converted to weekly excess internally; + the risk-free part cancels for relative views). +* **Idzorek confidence** `C_k ∈ (0, 1]`: the view's uncertainty is the `ω_k` for which the + unconstrained posterior weights move the fraction `C_k` of the way from `w` to the weights implied + by the view at 100 % confidence. For a diagonal Ω that calibration has the closed form + `ω_k = (1 − C_k)/C_k · p_k (τΣ) p_k'`; the tilt-matching minimisation of the paper is implemented + as `idzorek_omega_numeric` and the tests show both agree to 1e-4 (Idzorek 2005; the closed form is + due to Walters 2014). +* Posterior `μ_BL = π + τΣP'(PτΣP' + Ω)^{-1}(Q − Pπ)`, + `Σ_BL = Σ + τΣ − τΣP'(PτΣP' + Ω)^{-1}PτΣ`; `τ` default 0.05. +* Sentiment → view (`view_from_sentiment`): an assessment `s ∈ [−1, 1]` with degree of belief `C` + on stock `i` becomes the absolute view `q = π_i^{ann} + r_f + s · strength · σ_i^{ann}`, + `strength` = 0.25 by default (a fully positive view = +¼ annual volatility above equilibrium). +* Output: annual and per-period posterior total returns (used as scenario drift), Ω, δ, `μ_M`. + Alternatives: `expected_returns="historical"` (10-year means) or `"zero"`. + +## 8. Mean–CVaR optimisation as a MILP (`quantcore.optimization.problem`) + +Scenarios `R_{s,i}` = simple horizon returns `e^{r_{s,i}} − 1` in CHF (Section 6, with the drift of +Section 7), prices `P_i`, wealth `W = cash₀ + Σ_i n⁰_i P_i`. + +Variables: shares `n_i ∈ ℤ₊` (no shorts); rebalancing adds buys `b_i`, sells `s_i ∈ ℤ₊` with +`n_i = n⁰_i + b_i − s_i`; `α`, `u_s ≥ 0` bounded by the worst gain/loss attainable within the share +caps (free variables make the CVaR objective numerically hard — cuOpt's concurrent method aborts on +such models; the client then retries once with its dual simplex); binaries `y_i` (cardinality / minimum position) and +`t_i` (fixed cost per trade) only when those options are used. + +Transaction costs `TC = Σ_i (c_b P_i b_i + c_s P_i s_i) + F Σ_i t_i` (defaults 10 bp, no fixed cost). + +Loss in scenario `s`: `L_s = TC − Σ_i P_i R_{s,i} n_i`. CVaR (Rockafellar–Uryasev): + +``` +u_s ≥ L_s − α (S rows) +CVaR_β = α + 1/(1−β) Σ_s p_s u_s (p_s = 1/S unless importance sampling is used) +``` + +Objectives: `max_return` — maximise `E[P&L] = Σ_i P_i μ_i n_i − TC` (model expected simple returns +`μ_i` over the horizon; the probability-weighted scenario mean if none are given) s.t. `CVaR_β ≤ L` +(the absolute CHF risk budget for the period, e.g. "CVaR₉₅ over a week ≤ 40 000"); `min_cvar` +s.t. `E[P&L] ≥ target`; `utility` — maximise `E[P&L] − λ·CVaR_β`. + +Constraints: budget `cash₀ + Σ P_i s_i − Σ P_i b_i − TC ≥ 0` (cash may stay uninvested); single-stock +`P_i n_i ≤ w_max W` (10 %, implemented as tight variable bounds `n_i ≤ ⌊w_max W / P_i⌋`); sector +`Σ_{i∈k} P_i n_i ≤ w_k W` (default 25 %, per-sector dict possible); `TC ≤ TC_max`; turnover +`Σ P_i (b_i + s_i) ≤ τ W`; min/max invested fraction; cardinality `Σ y_i ≤ K`, `n_i ≤ M y_i`; +minimum position `P_i n_i ≥ v_min y_i`; `b_i + s_i ≤ M t_i`. + +The reported expected P&L (from the model means), VaR and CVaR of the decoded integer portfolio are +recomputed from the weighted scenario set and agree with the solver's values (`tests/test_problem.py`, +including a brute-force optimality check). Note that `α` is only *a* minimiser of the RU function — +it equals the VaR only when the CVaR constraint is tight. + +Sizes: `S + N(+2N) + 1` variables, `S + N + K + O(1)` rows, `≈ S·(N+2)` non-zeros; with the default +`N = 100`, `S = 4096` about 420 k non-zeros, solved by cuOpt in ~10 s. Solve times grow faster than +linearly in `S` (HiGHS LP ≈ `S^1.9`, cuOpt ≈ `S^1.2–1.6`): 2 000 → 100 000 scenarios costs ≈ 45× on +cuOpt. + +**Tail-row generation** (`quantcore.optimization.rowgen`, `row_generation="auto"` from 8 192 +scenarios on): only tail scenarios have `u_s > 0` at the optimum, so the MILP is solved on a working +set (the `2(1−β)S` worst scenarios of a reference portfolio), the losses of all scenarios are +checked, violated ones (`L_s > α*`) are added, and the loop stops when none is violated — the +restricted problem is a relaxation, so the final solution is optimal for the full set (within the +MIP gap). Measured: working sets ≈ 11 % of `S`, 2–3 iterations, 6–7× faster at 32 k–100 k scenarios. + +**Exact Gaussian mode** (`scenario_model="gaussian_exact"`, `quantcore.optimization.gaussian_exact`): +with normal horizon returns `R ~ N(m, C)` the CVaR is `TC − m'v + k_β √(v'Cv)` (`v = P∘n`, +`k_β = φ(z_β)/(1−β)`); the mean–CVaR problem then coincides with the mean–variance problem (the +CVaR bound corresponds to an implicit variance bound `σ̄ = (Λ + m'v*)/k_β`). In integer shares this is +a convex MISOCP, solved by outer approximation: tangent cuts `t ≥ TC + (k_β C v_k/σ_k − m)'v` on the +same MILP with an epigraph variable `t` (`cvar`), a new cut at each solution until the true CVaR equals +`t` (20–70 cuts; identical to Gurobi's direct MISOCP solution to the cent). No scenarios, no sampling +error; the reference for the sampling tests. + +## 9. Solvers (`quantcore.optimization.solvers`) + +Both solvers receive the *same* `MILPModel` (objective, CSR matrix, row and variable bounds, +variable types); `solve_with_fallback` picks between them and the result always names the one that +produced it. + +**cuOpt** — NVIDIA's GPU optimisation library, used through its self-hosted REST server +(`nvidia/cuopt`, version 26.08 here) running as the `cuopt` container on the DGX Spark: +`POST /cuopt/request` with a msgpack body, then poll `GET /cuopt/solution/{id}`; `"inf"/"ninf"` for +infinite bounds, `variable_types`, a mandatory `time_limit`. Statuses `Optimal` / `FeasibleFound` +(time limit — check `mip_gap`) / `Infeasible`. Docs: . It is the +only part of the pipeline that uses the GPU. + +**HiGHS** — the open-source LP/MILP/QP solver of ERGO-Code, University of Edinburgh (MIT licence, +, source , reference: Huangfu & Hall, +*Parallelizing the dual revised simplex method*, Math. Prog. Comp. 10, 119–142, 2018). quantcore +neither installs nor starts it separately: **HiGHS ships compiled inside SciPy**, and +`scipy.optimize.milp()` is a thin wrapper around it. It therefore runs **in-process, on the CPU, in +whichever process calls quantcore** — no network, no daemon, no separate container — and its version +follows the installed SciPy: + +| where | Python | SciPy | HiGHS | notes | +|---|---|---|---|---| +| `quantcore` container on the Spark (production) | 3.12.14 | 1.18.1 | 1.12.0 (`4f96ee8`) | aarch64 Linux, 4 threads (`OMP/OPENBLAS/MKL_NUM_THREADS=4`); numpy 2.5.3, pandas 3.0.6 | +| developer machine used for this work | 3.11.11 | 1.13.1 | 1.2.0 | arm64 macOS; numpy 1.26.4, pandas 2.1.4 | + +The container tracks the newest releases (nothing is pinned beyond the lower bounds in +`pyproject.toml`), so a rebuild can move it a long way from the developer machine — the build of +2026-09-23 jumped it to pandas 3, where the month-end resampling alias is `'ME'` and the old `'M'` +is gone (`quantcore.measurement.returns.MONTH_END` resolves this at import time, and is the reason +monthly frequency works on both). + +Verify it in any environment with +`python -c "import scipy.optimize as so, numpy as np, scipy.sparse as sp; print(so.milp(np.array([-3.,-2.]), constraints=so.LinearConstraint(sp.csr_matrix([[2.,3.],[4.,1.]]), [-np.inf]*2, [12.,10.]), integrality=np.ones(2), bounds=so.Bounds([0,0],[np.inf]*2), options={'disp': True}))"`, +which prints the HiGHS banner with its version. + +HiGHS is used in four places: `solver="highs"`; as the fallback when cuOpt is unreachable or errors +(`solver="auto"`); for the cut MILPs of the exact Gaussian mode, which are small and numerous; and as +the oracle in the tests (brute-force optimality checks). + +**Scenario count and candidate count**: `n_scenarios` defaults to 16 per candidate within +[4 096, 16 384]. The reported figures (expected P&L, CVaR) are stable at any size from ~2 000 on +(sd < 0.1 %), but the *composition* moves between near-equivalent portfolios in proportion to N/S — +3.5 % of wealth between seeds at N = 30, S = 2 048 against 12.5 % at N = 100 — and ~64 per candidate +is needed to hold it near 5 %. The result warns below 8 per candidate. + +**Covariance rank**: an estimator with `T_eff` effective observations (rolling: the window; EWM: +`3/(1−λ)`) cannot support more than `T_eff` assets. A 200-day window with N = 300 gives 101 zero +eigenvalues and a portfolio with *negative* estimated variance; `shrinkage="auto"` (the default) +raises the intensity with N/T (0.033 at N = 30 → 0.128 at N = 745) and keeps the smallest eigenvalue +above 2 % of the average variance in every case. `CovarianceModel.conditioning()` reports the +figures and the optimisation warns when the estimate is singular or ill-conditioned for the size of +the candidate set. Long-only with a 10 % cap limits the damage in any case: the optimised portfolio's +out-of-sample/in-sample volatility ratio exceeds the cap-weighted portfolio's by only 0.89–1.18× +across N = 30…745, with or without shrinkage. + +**cuOpt transport**: the model travels as one msgpack body (9.9 M non-zeros = 108 MB at N = 300, +S = 32 768), accepted at ~10 MB/s, so transport is not the constraint. The client uses a payload-scaled +timeout and a *fresh connection per attempt* — retrying on a pooled connection appends a second body and +the server rejects it with "unpack received extra data" (422) — and `solver="auto"` falls back to HiGHS +on failure. cuOpt's concurrent method also aborts on some CVaR-objective models; the client retries once +with its dual simplex (`method=2`). With row generation only the working set is sent. + +**Which models go to cuOpt.** cuOpt 26.08 aborts on one specific combination: the CVaR in the +*objective* (`objective="min_cvar"`) **and** probability-weighted scenarios. The cause is the scaling +of the objective vector. In the min-CVaR formulation the coefficient on each excess `u_s` is +`p_s/(1−β)`, so the scenario probabilities *are* the objective coefficients; importance sampling +spreads them over about six orders of magnitude. Measured on a 4 137-variable / 4 106-row model +(40 candidates, 4 096 scenarios, β = 99 %): + +| scenarios | objective `|c|` range | ratio | cuOpt | +|---|---|---|---| +| importance-sampled | 1.1e-06 … 1.0 | 8.8e5 | aborts in ~5 s | +| uniform (`importance_sampling="off"`) | 2.4e-02 … 1.0 | 4.1e1 | **Optimal in 3.8 s**, 5 runs out of 5, matching HiGHS to 7e-5 | + +Relaxing the integrality does not help — the pure LP aborts too, so it is the LP and not the branch +and bound — and multiplying the objective by a constant does not either, since that leaves the ratio +unchanged. In the `max_return` formulation the same probabilities sit in a *constraint row* instead +(the objective is then just 40 expected returns, range 1e3), where cuOpt's row scaling copes, and it +solves at either weighting. Its methods are not interchangeable either: `method=1` (PDLP) returned the +weighted optimum once in 4.9 s and then aborted three times on the identical model, while `method=2` +and `method=3` always abort. + +So `solver="auto"` diverts exactly that combination to HiGHS (1.7 s for that model) and the result +says why — otherwise a green cuOpt health pill next to `solver: "highs"` looks like the GPU being +ignored. Everything else keeps the GPU path. An explicit `solver="cuopt"` is always honoured and +returns cuOpt's own answer, abort included; an abort under `"auto"` is retried down +`CUOPT_RETRY_METHODS = (1, 2)` before the fallback. + +**cuOpt vs HiGHS, both on the Spark** (end-to-end through the API: scenario generation, model build, +solve, decode; GBM model, weekly, β = 0.95, no tilt, 600 s solver time limit): + +| case | cuOpt (GPU) | HiGHS (CPU, 4 threads) | +|---|---|---| +| N = 30, S = 8 192 | 15.6 s | 14.5 s | +| N = 30, S = 32 768 | 171.4 s | 171.0 s | +| N = 100, S = 4 096 | 10.0 s | 8.9 s | +| N = 100, S = 32 768 | 606.9 s (time limit, `FeasibleFound`) | 350.7 s (`Optimal`) | +| N = 300, S = 8 192 (row generation) | 139 s | 64 s | + +HiGHS is at least as fast in every case measured and 2× faster on the two largest. That is not a +statement about GPUs in general: cuOpt's published speedups are for *LP* with its PDLP first-order +method, while [its own documentation](https://docs.nvidia.com/cuopt/user-guide/latest/milp-features.html) +states that for MILP the GPU runs primal heuristics and **branch and bound runs on the CPU**, that the +MIP solver is beta, and that "proving feasible solutions optimal remains under active development". +Our models have an easy LP relaxation and a small integrality gap, so the work sits almost entirely in +proving optimality — the part cuOpt does not accelerate — while HiGHS' dual simplex warm-starts across +branch-and-bound nodes. HiGHS is also a strong reference point: in the [Mittelmann MILP benchmark](https://plato.asu.edu/ftp/milp.html) +(7 July 2026) HiGHS 1.15.1 solves 179 of 240 instances at a scaled shifted geometric mean of 5.44, +against 1.72 for COPT 8.0.3 and 9.93 for SCIP. Caveats: cuOpt's `time_limit` covers only its internal +solve, it ignores the MIP-gap settings sent over REST (so it may prove optimality to a tighter +tolerance), and this is one problem family on one machine, not a benchmark. + +**Reproducibility**: the scenario set depends only on the seed (`scenario_hash` in every result), and +the hash is identical for two runs **in the same environment**. Across environments it is not, and the +reason is the fingerprint rather than the mathematics. Measured between the developer Mac (numpy 1.26.4, +pandas 2.1.4, scipy 1.13.1) and the Spark container (numpy 2.5.3, pandas 3.0.6, scipy 1.18.1) on the +same 100 stocks and the same stored prices: scrambled Sobol' points and the numpy RNG stream are +bit-identical, the estimated covariance agrees to `max |dC| = 1.7e-18` (entries ~5e-4), its +eigenvalues to 1.4e-17 and the columns of `pca_root` to 4e-13 relative — no sign flips, no rotation. +The scenario matrices therefore agree to 13–14 significant digits. But `scenario_hash` fingerprints +`np.round(scenarios, 12)`, and on a 4096 × 100 matrix of weekly log-returns a *relative* perturbation +of 1e-15 already pushes ~3 entries of 409 600 across a rounding boundary (1e-14 pushes ~47), which +changes the digest. The figures are unaffected: the same request optimised on both machines gave +E[P&L] 1 888.65 against 1 888.76 and CVaR 39 999.71 against 39 999.82 on a CHF 1 M portfolio. + +The portable identity is the **scenario key**, not the hash: it is built from the request and the +model description, so two environments that agree on the inputs produce the same `key` (verified: +`7162939ddf75ff74` on both machines for the run above) even where the digest of the realised matrix +differs. Read `scenario_hash` as "these two results used bit-identical scenarios" — a statement about +one machine — and `scenario_key` as "these two results were asked for the same scenarios". +HiGHS' branch-and-bound is deterministic → identical shares for the same inputs *and the same HiGHS +build* (`reproducibility="exact"`); across SciPy versions (1.2.0 on the developer machine against +1.12.0 in the container) the search can end on a different optimal vertex among ties, with the same +objective. cuOpt 26.08 +ignores the MIP-gap settings sent over REST and stops within its default relative gap of 1e-4: the +objective, expected P&L and CVaR of two runs agree to that gap (≈ 0.2 CHF/week on CHF 1 M) while +near-tied share counts can differ by a few shares (`reproducibility="within_gap"`). The Gaussian exact +mode runs on HiGHS by default. + +## 10. Default parameters + +| Parameter | Default | Where | +|---|---|---| +| history start / required first price | 2004-12-01 / ≤ 2005-01-31, coverage ≥ 90 % | `Settings` | +| base currency, risk-free | CHF, 0 % | `Settings` | +| measurement window / λ | 200 periods / 0.94 | `RiskService` | +| optimisation covariance | EWM λ = 0.99 (≈ 100-day effective window) | `OptimizationRequest.covariance` | +| scenarios | empirical innovations, 4 096 scrambled Sobol' draws on the market-factor ranks, moment matched, arithmetic drift, last 2 500 days, diagonal re-standardisation, seed 42 | `OptimizationRequest` | +| importance sampling / row generation | `auto` = market tilt (blocks at z₀.₉₅, z₀.₉₉, 25 % each); row generation from 8 192 scenarios | `OptimizationRequest` | +| MIP gap | 1e-4 relative (HiGHS; cuOpt uses its own default) | `Settings.mip_rel_gap` | +| Black–Litterman | 10 years weekly, τ = 0.05, sentiment strength 0.25 | `OptimizationRequest`, `view_from_sentiment` | +| constraints | 10 % per stock, 25 % per sector, no shorts, 10 bp costs | `OptimizationConstraints`, `TransactionCosts` | +| CVaR limit | explicit CHF amount or fraction (5 % of wealth if nothing is given) | `OptimizationRequest` | +| cuOpt time limit | 120 s (`QUANTCORE_CUOPT_TIME_LIMIT`) | `Settings` | diff --git a/docs/SPARK_APPTAINER.md b/docs/SPARK_APPTAINER.md new file mode 100644 index 0000000..a3910cf --- /dev/null +++ b/docs/SPARK_APPTAINER.md @@ -0,0 +1,254 @@ +# quantcore on your DGX Spark (Apptainer, no root) + +This guide sets up the quant service of the project on your own DGX Spark. The service covers +market data, risk measurement, Black–Litterman expected returns and the mean-CVaR optimiser. +Everything runs on the Spark itself: + +- **cuOpt**, NVIDIA's GPU solver, runs in its own Apptainer container. +- **quantcore**, this repository, runs in a second Apptainer container. It calls cuOpt at + `http://127.0.0.1:5000`. +- The API and the browser test bench listen on `http://127.0.0.1:5001`. You reach them from the + Spark: its desktop browser, a terminal, a notebook, or the platform you build. They are not + reachable from the network. + +You need no root rights, no Docker and no VPN. One script, `deploy/apptainer/qc.sh`, does +everything. + +## What you need + +| Requirement | Check | +|---|---| +| Apptainer 1.1 or newer | `apptainer --version` | +| The GPU | `nvidia-smi -L` lists the NVIDIA GB10. Without it, optimisation runs on HiGHS on the CPU. | +| About 5 GB in your home directory | The cuOpt image is 3.3 GB, quantcore 0.2 GB, the data 40 MB. Pulling cuOpt needs about 12 GB for a few minutes. | +| Internet access | Docker Hub for the images, PyPI for the Python libraries, Yahoo Finance for market data. | + +## Quick start + +```bash +git clone https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git +cd Portfolio_Risk_Management + +deploy/apptainer/qc.sh install # once, ~10 min: settings, cuOpt image, quantcore image, self-test +deploy/apptainer/qc.sh data-import /path/to/quantcore_data.tar.gz # the prepared data set +# or build the data set from Yahoo Finance yourself (~15 min): deploy/apptainer/qc.sh setup +deploy/apptainer/qc.sh start # starts cuOpt and the API +``` + +Then open **http://127.0.0.1:5001/ui/** in the Spark's browser. The interactive API +documentation is at **http://127.0.0.1:5001/docs**. + +Working on the Spark over SSH from your laptop? Forward the port, then open the same address in +the laptop's browser. Nothing else on the network can reach the service: + +```bash +ssh -L 5001:127.0.0.1:5001 @ +``` + +A shorter command saves typing. Put this line in your `~/.bashrc`: + +```bash +alias qc="$HOME/Portfolio_Risk_Management/deploy/apptainer/qc.sh" +``` + +The rest of this guide writes `qc` for the script. + +## Daily use + +```bash +qc start # cuOpt + API; after a reboot, run it again +qc status # what runs, whether cuOpt is reachable, how much data there is +qc logs # API log; `qc logs cuopt` for the solver, add -f to follow +qc update # the newest prices and FX rates from Yahoo Finance +qc stop # stop both containers +``` + +The containers keep running after you log out. They stop when the Spark reboots. + +## Using the service + +**In the browser.** The test bench at `http://127.0.0.1:5001/ui/` shows every service: the +stock universe, risk and scenarios, expected returns and views, constraints, and the +optimisation with a before/after comparison. + +**From your own code.** This is how the platform you build uses the service. The API speaks +JSON over HTTP, so any language works. Your code reaches it at `http://127.0.0.1:5001`. That +address also works inside your own Apptainer containers, because Apptainer containers share +the Spark's network. For Python there is a ready-made client: + +```python +from quantcore.client import QuantcoreClient + +qc = QuantcoreClient("http://127.0.0.1:5001") +print(qc.health()["cuopt"]["reachable"]) +risk = qc.asset_risk(["NESN.SW", "AAPL"], horizon="week") +``` + +Install the client into your own environment with +`pip install "git+https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git"`. +[USAGE.md](USAGE.md) describes every endpoint and the request fields. + +**From the command line, through the API.** `qc api` runs the API client against your +service: + +```bash +qc api health +qc api selftest # calls every endpoint, pass/fail table +qc api risk NESN.SW AAPL --horizon week +qc api optimize --cash 1000000 --cvar-limit 40000 --horizon week +``` + +**From the command line, without the API.** `qc cli` runs the services directly in the +container. The API does not need to be running: + +```bash +qc cli status +qc cli risk NESN.SW AAPL --horizon week +qc cli optimize --cash 1000000 --cvar-limit 40000 --horizon week --max-assets 50 +``` + +## Settings + +`qc install` writes the settings file `~/quantcore/quantcore.env`. Edit it, then run +`qc restart`. A value set in the shell wins over the file, for example +`QC_API_PORT=5011 qc start`. + +| Setting | Default | Meaning | +|---|---|---| +| `QC_API_PORT` | `5001` | Port of the API and the test bench. | +| `QC_CUOPT_PORT` | `5000` | Port of the cuOpt server. | +| `QC_CUOPT_START` | `auto` | `auto` uses a cuOpt that already answers on the port, or starts one. `yes` always starts its own. `no` never starts one, so HiGHS solves every model. | +| `QC_CUOPT_IMAGE` | `docker://nvidia/cuopt:26.8.0-cuda13.3-py3.14` | Where the cuOpt image comes from. Set the path of a `.sif` file if your Spark already provides one. | +| `QC_IMAGE` | `build` | How quantcore gets its container; see the next section. | +| `QUANTCORE_*` | | Passed into the container unchanged. Examples are `QUANTCORE_CUOPT_TIME_LIMIT` and `QUANTCORE_ENABLE_UI`. All of them are listed in `quantcore/config.py`. | + +**Several users on one Spark.** Ports belong to the whole machine. If someone else already uses +port 5001, `qc start` stops and suggests a free port to put into your settings file. A cuOpt +that another user already runs on port 5000 is simply used: cuOpt keeps no state between +requests. + +## How quantcore gets its container + +`QC_IMAGE=build` is the default. `qc install` then builds the image `quantcore.sif` from this +repository with Apptainer's unprivileged build. Some Spark installations do not allow +unprivileged builds. In that case `qc install` falls back by itself to a setup that needs no +build. It pulls a plain Python image and installs quantcore into a virtual environment in +`~/quantcore/venv`. The libraries and the results are the same, because both setups install the +pinned versions in `deploy/apptainer/constraints.txt`. You can choose the fallback directly +with `QC_IMAGE=venv`. + +`QC_IMAGE` also accepts the path of a prebuilt `quantcore.sif`, or a `docker://` or `oras://` +address to pull one from. + +**After `git pull`**, rebuild and restart: + +```bash +git pull && qc build && qc restart +``` + +## How it works + +``` + DGX Spark, your user account, no root + ┌────────────────────────────────────────────────────────────────────────────┐ + │ browser / your platform / qc api │ + │ │ HTTP │ + │ ▼ │ + │ 127.0.0.1:5001 instance "quantcore-api" quantcore.sif │ + │ FastAPI + test bench ~/quantcore/data -> /data │ + │ │ HTTP (msgpack) │ + │ ▼ │ + │ 127.0.0.1:5000 instance "quantcore-cuopt" cuopt.sif, started with --nv │ + │ cuOpt server ──► GB10 GPU │ + └────────────────────────────────────────────────────────────────────────────┘ +``` + +- An Apptainer **instance** is a container that runs in the background, like a service. + `apptainer instance list` shows yours. +- Apptainer containers use the Spark's network directly, with no port mapping. A server inside a + container that listens on `127.0.0.1` is therefore reachable from the Spark, and only from it. +- The images are read-only. Everything that changes lives in `~/quantcore/data`, which is + mounted into the container as `/data`. +- The containers start with `--cleanenv`. Variables from your shell, such as an activated + virtual environment or `PYTHONPATH`, cannot interfere. `qc.sh` passes only the settings above, + plus proxy variables if you have them. +- NVIDIA's cuOpt image sets its home directory to `/opt/cuopt`. That folder is writable under + Docker but read-only in an Apptainer image, and the solver would crash on its first job when + CuPy writes its kernel cache there. `qc.sh` therefore starts the cuOpt server with its home + directory in `~/quantcore/cuopt-home`. +- When cuOpt is not reachable, or stops during a solve, every optimisation falls back to HiGHS on + the CPU. The result says which solver produced it. + +| Path | Content | +|---|---| +| `~/quantcore/quantcore.env` | your settings | +| `~/quantcore/data/` | universe, prices, FX, snapshots | +| `~/quantcore/images/cuopt.sif` | the cuOpt image | +| `~/quantcore/images/quantcore.sif` | the quantcore image; in the fallback, `python.sif` plus `~/quantcore/venv/` | +| `~/quantcore/cuopt-home/` | cuOpt's caches (CuPy kernels, CUDA) | +| `~/.apptainer/instances/logs/` | instance logs, also shown by `qc logs` | +| `~/.apptainer/cache/` | downloaded image layers; `qc clean-cache` frees the space | + +## Troubleshooting + +| Symptom | Cause and fix | +|---|---| +| `port 5001 is taken` | Another user or program uses the port. Set the suggested `QC_API_PORT` in `~/quantcore/quantcore.env`, then run `qc start`. | +| `qc status` shows cuOpt as not answering | Run `qc logs cuopt`. Check that `nvidia-smi` works. Then `qc restart`. Optimisations keep working on HiGHS in the meantime. | +| The cuOpt log says `Read-only file system: '/opt/cuopt/.cupy'` | cuOpt was started without `qc.sh`. Start it with `qc start`, or give it a writable home directory as described in "Running cuOpt by hand". | +| `/optimize` answers 503, "no market data yet" | Run `qc data-import ` or `qc setup`. | +| Yahoo errors during `qc setup` or `qc update` | Yahoo limits how fast it answers. Run the command again later; stored data stays intact. | +| `qc install` stops with "no 'instance run'" | The Spark's Apptainer is older than 1.1. Ask the administrator to update it. | +| The cuOpt pull fails with "no space left on device" | Apptainer unpacks images in `/tmp`. Point it at a larger directory: `APPTAINER_TMPDIR=~/quantcore/tmp qc pull-cuopt`. | +| Model status `FeasibleFound` instead of `Optimal` | The time limit was reached. Raise `time_limit` in the request or `QUANTCORE_CUOPT_TIME_LIMIT`, or reduce `n_scenarios` or `max_assets`. | + +## Running cuOpt by hand + +`qc start` does this for you. If you want to run the cuOpt image yourself, for example for another +project, start it like this: + +```bash +apptainer pull cuopt.sif docker://nvidia/cuopt:26.8.0-cuda13.3-py3.14 +mkdir -p ~/cuopt-home +apptainer instance run --nv --cleanenv --bind ~/cuopt-home cuopt.sif my-cuopt \ + env HOME=$HOME/cuopt-home python -m cuopt_server.cuopt_service -i 127.0.0.1 -p 5000 +curl http://127.0.0.1:5000/cuopt/health +apptainer instance stop my-cuopt +``` + +## For the lecturer + +**Data set for the students.** Many students running `qc setup` at the same time from one +network can hit Yahoo's rate limits. A prepared data set avoids that, and every student then +works on the same data. Pack it on a machine that has the data: + +```bash +qc data-export quantcore_data.tar.gz # or, from any data directory: +tar -czf quantcore_data.tar.gz -C --exclude ./jobs --exclude '*.log' . +``` + +The archive is about 30 MB. It is not part of the public repository: Yahoo's terms do not allow +redistributing their data. + +**Prebuilt image.** Build `quantcore.sif` on any aarch64 Linux machine where you can build +images, then hand out the file: + +```bash +apptainer build quantcore.sif deploy/apptainer/quantcore.def # from the repository root +``` + +Students set `QC_IMAGE=/path/to/quantcore.sif` in their settings file. A copy built and tested on +the lecturer's Spark is described in the verification note below. + +**The lecturer's own Spark** runs the Docker setup in [SPARK_DEPLOYMENT.md](SPARK_DEPLOYMENT.md). +That setup needs root and Docker. Students do not need it. + +**Verification.** The setup was tested end to end on 7 October 2026 on a DGX Spark: NVIDIA GB10, +driver 580.159, Ubuntu 24.04, Apptainer 1.5.4, as a user without root. That Spark allowed no +unprivileged image builds, so `qc install` took the virtualenv fallback by itself. The image mode +was then tested with a prebuilt `quantcore.sif`. In both modes: + +- `qc api selftest --full` passed on every endpoint, and the optimisation ran on cuOpt on the GPU. +- `qc update` fetched the newest Yahoo prices through the running API. +- A taken API port stopped `qc start` with a suggested free port, and that port worked. +- The test suite passed inside the container, including the round trip to the live cuOpt server. diff --git a/docs/SPARK_DEPLOYMENT.md b/docs/SPARK_DEPLOYMENT.md new file mode 100644 index 0000000..ddc0987 --- /dev/null +++ b/docs/SPARK_DEPLOYMENT.md @@ -0,0 +1,110 @@ +# Running the quant service on the DGX Spark (Docker, with root) + +> This is the setup of the lecturer's Spark, which has root rights, Docker Compose and Tailscale. +> **Students** run the service on their own Spark with Apptainer and without root: +> see [SPARK_APPTAINER.md](SPARK_APPTAINER.md). + +The Spark (`spark-6d4a`, reachable over Tailscale) already runs cuOpt, Gurobi and JupyterLab as +Docker containers defined in `~/services/docker-compose.yml`. The quant service joins that +compose file as the container **`quantcore`** on port **5001**; it reaches cuOpt over the compose +network (`http://cuopt:5000`) and keeps its data on the host in `~/services/work/quantcore_data`. + +| Service | Address | What it is | +|---|---|---| +| **quantcore** | `http://spark-6d4a:5001` (`/docs` for Swagger, `/ui/` for the test bench) | portfolio data, risk/performance measurement, models, mean-CVaR MILP | +| cuOpt | `http://spark-6d4a:5000` | the GPU MILP solver the service uses | + +## Deploy (from the Mac) + +```bash +deploy/deploy_to_spark.sh --with-data # first time: also ships the locally built data set +deploy/deploy_to_spark.sh # later: code only +``` + +The script rsyncs the repository to `~/services/work/Portfolio_Risk_Optimization`, appends the +service block from `deploy/docker-compose.quantcore.yml` to the compose file (once; a backup of the +compose file is kept), builds the image (`python:3.12-slim`, aarch64) and starts the container. + +The block is appended **only if `quantcore:` is not already in `~/services/docker-compose.yml`**, so +later edits to `deploy/docker-compose.quantcore.yml` (a new environment variable, a changed port) do +not travel with a redeploy — apply them to the Spark's compose file by hand, or delete the block +there and re-run the script. Code and image changes need no such step. + +Without `--with-data` the container starts empty; run the setup once through the API: + +```bash +curl -X POST http://spark-6d4a:5001/jobs/setup # ~15 min: 1 117 tickers' metadata + history +curl http://spark-6d4a:5001/jobs/ # status: queued | running | done | failed +``` + +## Operate + +```bash +ssh user_fh_fhgr@spark-6d4a +cd ~/services +docker compose ps quantcore # status (healthcheck on /health) +docker compose logs -f quantcore # live logs +docker compose restart quantcore +docker compose up -d --build quantcore # rebuild after a code change (or re-run the deploy script) + +# inside the container: the CLI works against the same data volume +docker exec -it quantcore quantcore status +docker exec -it quantcore quantcore update # newest prices (incremental) +docker exec -it quantcore quantcore snapshot --name demo +``` + +Nightly data refresh (host cron on the Spark), e.g. `crontab -e`: + +``` +15 23 * * 1-5 curl -s -X POST http://localhost:5001/jobs/update >/dev/null +``` + +## Use from anywhere on the tailnet + +```bash +curl http://spark-6d4a:5001/health +curl "http://spark-6d4a:5001/universe?region=Europe§or=Health%20Care" +curl -s http://spark-6d4a:5001/risk/assets -H 'Content-Type: application/json' \ + -d '{"symbols": ["NESN.SW", "ROP.SW", "AAPL"], "horizon": "week"}' +curl -s http://spark-6d4a:5001/optimize -H 'Content-Type: application/json' \ + -d '{"cash": 1000000, "max_assets": 100, "cvar_limit": 40000, "horizon": "week"}' +``` + +The test bench is served from the same container (`QUANTCORE_ENABLE_UI`, default `true`, in the +compose block). Set it to `"false"` and rebuild for a deployment that exposes the JSON API and +`/docs` only — for users who build their own front end against these endpoints. +`deploy/docker-compose.students.yml` is that deployment, standalone and with the bench already +off; `docs/USAGE.md` is the guide its users need. + +Python clients simply point `QUANTCORE_CUOPT_URL` / the API base URL at the Spark; the package +itself also runs on a laptop (`quantcore serve`) and uses the Spark's cuOpt remotely +(`QUANTCORE_CUOPT_URL=http://spark-6d4a:5000`), falling back to HiGHS — which needs no service of its +own: it is compiled into SciPy inside the `quantcore` image (SciPy 1.18.1 → HiGHS 1.12.0) and runs on +the container's CPU — when the +Spark is unreachable. + +## Sizing + +| Candidates | Scenarios | MILP size (vars / rows / nnz) | cuOpt solve | +|---|---|---|---| +| 100 | 2 000 | 2 101 / 2 013 / 206 k | ~6–10 s | +| 300 | 2 000 | 2 301 / 2 013 / 607 k | ~12 s | +| 745 (all) | 1 000 | 1 746 / 1 013 / 749 k | ~17 s | +| 100, rebalancing with fixed costs | 2 000 | 2 401 / 2 215 / 808 k (400 integers) | ~33 s | + +The time limit (`QUANTCORE_CUOPT_TIME_LIMIT`, request field `time_limit`) bounds the solve; a +`FeasibleFound` status with a small `mip_gap` is a valid, slightly sub-optimal answer. + +BLAS threads are pinned to 4 (`OPENBLAS_NUM_THREADS`): the empirical-innovation filter performs +thousands of small eigendecompositions, which OpenBLAS runs ~50× slower with all 20 cores. + +## Troubleshooting + +| Symptom | Cause / fix | +|---|---| +| `/health` shows `cuopt.reachable: false` | cuOpt container down or unhealthy: `docker compose restart cuopt` (see the Spark README §5). Optimisations fall back to HiGHS and say so in `solver` / `warnings`. | +| `/optimize` answers 503 "no market data yet" | run `POST /jobs/setup` or deploy with `--with-data`. | +| Yahoo errors in the setup/update job | rate limiting; the client retries with back-off and opens a circuit after 5 failures. Re-run the job later; stored data stays intact. | +| `FeasibleFound` instead of `Optimal` | time limit hit; raise `time_limit` or reduce `n_scenarios` / `max_assets`. | +| `Infeasible` | the constraints contradict each other (e.g. `min_invested` with a tiny CVaR limit, or a rebalancing turnover cap that prevents reaching the risk budget). | +| Tailscale SSH asks for a browser check | `ssh user_fh_fhgr@spark-6d4a` prints a login URL; open it once, then re-run the deploy script. | diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000..350a711 --- /dev/null +++ b/docs/USAGE.md @@ -0,0 +1,342 @@ +# Using quantcore + +`quantcore` is the **deterministic quant core** your advisory platform builds on: market data, risk +and performance measurement, stock-price models, Black–Litterman expected returns and mean–CVaR +portfolio optimisation in whole shares. + +It contains **no LLM call and makes no decisions**. It computes numbers from data and from the inputs +you hand it, and every answer carries the provenance of those inputs. Your platform supplies the +judgement — which news matters, how strongly to believe it, what to tell the client — and calls in +here for every figure it quotes. + +You never have to modify this package. Treat it as a service with a contract. + +--- + +## 1. Two ways to use it + +| | **HTTP API** | **Python library** | +|---|---|---| +| you need | a URL | the package *and* the data directory on the same machine | +| import | nothing — `requests`, or `quantcore.client` | `from quantcore.measurement.risk import RiskService` | +| language | any | Python | +| good for | the normal case: your platform calls a running service | notebooks, experiments, running your own backtest loop | + +Start with the API. Everything in this guide is available both ways, and the library classes are +documented in `README.md` and in `docs/ARCHITECTURE.md` if you need them. + +## 2. Install + +```bash +git clone https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git +cd Portfolio_Risk_Management +pip install -e . # Python 3.11+ +``` + +On a DGX Spark you do not need this step to run the service: `deploy/apptainer/qc.sh` runs it in +containers (see §3 and [SPARK_APPTAINER.md](SPARK_APPTAINER.md)). Install the package only where your +own code imports it, for example for `quantcore.client`. + +This gives you two commands: + +| command | what it is | +|---|---| +| `quantcore` | runs the services **in this process**; needs the data directory locally | +| `quantcore-api` | talks to a **running** service over HTTP; needs only its URL | + +## 3. Point at a service, or run your own + +**On your DGX Spark** (the usual case): the service and cuOpt run as Apptainer containers of your +user, reachable at `127.0.0.1` on the Spark only. [SPARK_APPTAINER.md](SPARK_APPTAINER.md) has the details: + +```bash +deploy/apptainer/qc.sh install && deploy/apptainer/qc.sh data-import quantcore_data.tar.gz +deploy/apptainer/qc.sh start # API at http://127.0.0.1:5001 (the client's default URL) +deploy/apptainer/qc.sh api health +``` + +**Someone else runs it:** point the client at that machine. + +```bash +export QUANTCORE_API_URL=http://:5001 +quantcore-api health +``` + +**You run it yourself.** The data has to be built once, which downloads about 1 100 tickers from +Yahoo Finance and takes ~15 minutes: + +```bash +quantcore setup # once +quantcore serve # HTTP API on :5001, interactive docs at /docs +``` + +Or as a container, which is how it runs in production: + +```bash +docker build -f deploy/Dockerfile -t quantcore:0.1 . +docker run -p 5001:5001 -v $PWD/data:/data \ + -e QUANTCORE_CUOPT_URL=http://:5000 quantcore:0.1 +``` + +Configuration is environment variables with a `QUANTCORE_` prefix (see `.env.example`); the ones you +are likely to touch: + +| variable | default | meaning | +|---|---|---| +| `QUANTCORE_API_URL` | `http://127.0.0.1:5001` | which service `quantcore-api` and `QuantcoreClient` talk to | +| `QUANTCORE_DATA_DIR` | `./data` | where the prices and the universe live | +| `QUANTCORE_CUOPT_URL` | `http://127.0.0.1:5000` | the GPU MILP solver (a cuOpt container on the same machine); unreachable is fine, see §9 | +| `QUANTCORE_BASE_CURRENCY` | `CHF` | reporting currency of every figure | +| `QUANTCORE_ENABLE_UI` | `true` | `false` serves the JSON API and `/docs` only | + +## 4. Check that it works + +```bash +quantcore-api selftest +``` + +This calls **every endpoint** with a small request and prints what each one answered: + +``` + endpoint result ms evidence +0 GET /health ok 260 745 stocks, last 2026-09-21, cuOpt up +5 GET /prices ok 28 5667 dates x 3 symbols +9 POST /risk/assets ok 10 3 rows, cvar95_hist 0.1259 +21 POST /optimize (gaussian_exact) ok 63 Optimal on highs, 7 positions, CVaR 44,551 +... +24 ok, 0 failed, 3 skipped in 1.0 s against http://127.0.0.1:8765 +``` + +Exit code 0 means the deployment is sound. Run this first whenever something behaves oddly — it +separates "the service is broken" from "my request is wrong" in one second. The three skipped probes +write to the store or download data; `--write` includes them. `--full` optimises on a real scenario +set instead of the fast Gaussian mode. + +`quantcore-api endpoints` lists the API surface, and `/docs` is the interactive reference with +every field of every request. **That is the authoritative contract** — this guide is the tour. + +## 5. Conventions you have to know + +These are not negotiable per request; they are how the whole package thinks. + +* **Returns are natural logarithms.** `ln(P_t / P_{t-1})`. A CHF return of a foreign stock is the + local log-return plus the FX log-return — they add. +* **Money is CHF**, always, whatever the listing currency. +* **Positions are integer share numbers**, never percentages. That is the point of the MILP: you + cannot buy 0.3 of a share, and at CHF 1 m the difference matters. +* **A horizon is `"day"`, `"week"`, `"month"`, `"quarter"`, `"year"`, or an integer number of + periods** of the chosen frequency. `"week"` at daily frequency is 5 trading days. +* **Risk is CVaR** (expected shortfall) at 95 % and 99 %, expressed as an absolute amount over the + horizon. VaR comes along for reference. +* **Two knobs on every parametric risk figure**, and the defaults are not always what you want: + * `drift`: `"estimated"` subtracts the window's mean return; `"zero"` uses volatility only. Use + `"zero"` for anything longer than a week — over a quarter the estimated mean is the noisiest + input in the calculation and it can halve the reported risk. + * `space`: `"log"` reports the figure as a log-return; `"value"` as a fraction of the position's + value (`1 - e^{-x}`), which cannot exceed 100 %. Use `"value"` in anything a client reads. + +## 6. The endpoints + +| method | path | what it answers | +|---|---|---| +| GET | `/health` | is the service up, what data does it hold, is the GPU solver reachable | +| GET | `/universe`, `/universe/{symbol}` | the investable stocks and their metadata | +| GET | `/data/symbols` | the universe as one table plus filter facets — feeds a stock picker | +| GET | `/prices`, `/returns` | adjusted closes and log-returns, daily/weekly/monthly, CHF or local | +| GET | `/market/index`, `/market/weights` | the cap-weighted broad market and its weights | +| POST | `/risk/assets` | per-stock volatility, VaR, CVaR (normal and historical, rolling and EWM) | +| POST | `/risk/series` | those figures through time, for a chart | +| POST | `/risk/covariance`, `/risk/correlation` | the matrices as measured | +| POST | `/risk/portfolio` | VaR/CVaR of a portfolio held in shares | +| POST | `/performance/assets`, `/performance/rolling`, `/performance/portfolio` | return, Sharpe, Sortino, drawdown, Calmar, skew, kurtosis | +| POST | `/model/expected-returns` | Black–Litterman: equilibrium π, your views, the posterior | +| POST | `/model/covariance` | the covariance the models use, with its conditioning | +| POST | `/model/scenarios` | a scenario set, its moments, a portfolio's P&L distribution | +| POST | `/optimize` | the mean–CVaR MILP: a new portfolio or a rebalancing | +| POST | `/jobs/setup`, `/jobs/update`, GET `/jobs`, `/jobs/{id}` | background data work | +| GET/POST | `/snapshots` | freeze the data so a demo stays reproducible | + +## 7. From Python + +```python +from quantcore.client import QuantcoreClient + +qc = QuantcoreClient("http://127.0.0.1:5001") # or omit: $QUANTCORE_API_URL, else this default + +qc.health()["store"]["last_price_date"] # '2026-09-21' + +# what can I invest in? +swiss = qc.universe(country="Switzerland") +health = [s["symbol"] for s in qc.universe(region="Europe", sector="Health Care")] + +# what is one stock's risk over a week, as a client would read it? +row = qc.asset_risk(["NESN.SW"], horizon="week", drift="zero", space="value")[0] +row["cvar95_normal_ewm"] # 0.0665 -> 6.7 % of the position in the worst 5 % + +# what is my client's current portfolio risking? +qc.portfolio_risk({"NESN.SW": 500, "AAPL": 100}, cash=20_000, horizon="month") + +# the research agent found something positive on Apple, with middling confidence +views = [{"symbol": "AAPL", "sentiment": 0.6, "confidence": 0.4}] +post = qc.expected_returns(max_assets=100, views=views) + +# propose a portfolio for CHF 1 m that risks at most CHF 30 000 in the worst 5 % of weeks +res = qc.optimize(cash=1_000_000, horizon="week", confidence=0.95, cvar_limit=30_000, + views=views, risk_drift="zero", time_limit=300) + +res["status"] # 'Optimal' +res["cvar"] # 30000.00 -- the limit binds exactly +for p in res["positions"]: + print(p["symbol"], p["shares"], round(p["weight"], 4)) +``` + +Every method passes its keyword arguments through to the API unchanged, so the field names in +`/docs` are the field names here. A non-2xx answer raises `QuantcoreError` with the API's own +message; an unreachable server raises it too, with `unreachable = True`. + +## 8. From the command line + +The same surface, for looking at things and for scripting: + +```bash +quantcore-api universe --region Europe --sector "Health Care" +quantcore-api prices NESN.SW AAPL --freq weekly --tail 5 +quantcore-api risk NESN.SW AAPL --horizon quarter --drift zero --space value +quantcore-api correlation NESN.SW NOVN.SW ROG.SW +quantcore-api risk-portfolio --holdings '{"NESN.SW": 500, "AAPL": 100}' --cash 20000 +quantcore-api expected-returns --max-assets 50 --view AAPL:0.6:0.4 +quantcore-api scenarios NESN.SW AAPL --model gbm --n-scenarios 4096 +quantcore-api optimize --cash 1000000 --cvar-limit 30000 --horizon week --risk-drift zero +quantcore-api optimize --request my_request.json --json > result.json +quantcore-api update --wait # refresh prices and follow the job +``` + +`--view SYMBOL:SENTIMENT[:CONFIDENCE[:STRENGTH]]` is the shorthand for a sentiment view, repeatable. +`--json` prints the raw answer (pipe it into `jq`, or save it as a fixture); it works before or after +the subcommand. `--request FILE` takes a full JSON request, which is the right way to keep a +complicated setup reproducible — its fields win over the flags. + +## 9. The five things the project actually needs + +**Pick a universe.** `/data/symbols` gives you rows *and* the counts per region, sector and country +in one call, so a picker needs no second request. Pass the resulting symbols as `symbols` to +`/optimize` to restrict the candidates; leave it out and the largest `max_assets` by market cap are +used. + +**Measure what the client holds.** `/risk/portfolio` with their share numbers and cash. Use +`method="historical"` alongside the default `"normal"` and show both — where they disagree, the +distribution is not Gaussian, and that is worth saying out loud. + +**Turn an assessment into a view.** A view is a statement about an *annual expected return* with a +degree of belief. Either explicit (`{"assets": {"NESN.SW": 1.0}, "q": 0.09}`: "Nestlé will return +9 %") or from sentiment (`{"symbol": "NESN.SW", "sentiment": 0.6, "confidence": 0.4}`). Sentiment in +[−1, 1] scales `strength` (default 25 % p.a. at full sentiment); `confidence` in (0, 1] is the +Idzorek confidence that sets Ω. Call `/model/expected-returns` *before* optimising to see what your +view did to the posterior — that is the number you will have to defend. + +**Propose a new portfolio.** `/optimize` with `mode="new"` and `cash`. Set `cvar_limit` (absolute +CHF) or `cvar_limit_fraction` (of wealth). Constraints live in `constraints`, costs in `costs`: + +```json +{"mode": "new", "cash": 1000000, "horizon": "week", "confidence": 0.95, "cvar_limit": 30000, + "risk_drift": "zero", + "constraints": {"max_weight": 0.10, "max_sector_weight": 0.25, "max_positions": 25, + "min_position_value": 5000, "min_invested": 0.95}, + "costs": {"proportional_buy": 0.001, "proportional_sell": 0.001, "fixed_per_trade": 0}} +``` + +**Rebalance an existing one.** `mode="rebalance"` with `holdings` (shares) and the `cash` available. +Transaction costs are charged and the result carries a `before` block, so you can show the client +both states and what the trade bought them: + +```python +res = qc.optimize(mode="rebalance", holdings={"NESN.SW": 500, "AAPL": 100}, cash=50_000, + horizon="week", cvar_limit=30_000, risk_drift="zero", + constraints={"max_turnover": 200_000}) +res["before"]["cvar"], res["cvar"] # risk before and after +res["total_cost"], res["turnover"] # what the change costs +[p for p in res["positions"] if p["trade"] != 0] # the trades to place +``` + +## 10. Reading a result + +Every optimisation answers with its own provenance. The fields you will quote: + +| field | meaning | +|---|---| +| `status` | `Optimal` \| `FeasibleFound` (time limit — check the gap) \| `Infeasible` \| `Error` | +| `solver` | `cuopt` (GPU) or `highs` (CPU) — which one actually produced this | +| `reproducibility` | `exact` (HiGHS) \| `within_gap` (cuOpt, 1e-4) \| `none` | +| `expected_pnl`, `expected_return` | over the horizon, in CHF and as a fraction | +| `var`, `cvar`, `cvar_limit`, `confidence` | the risk figures and the limit they had to respect | +| `positions` | per stock: `shares`, `shares_before`, `trade`, `value`, `weight`, `cost`, `expected_return_ann` | +| `sector_exposure`, `invested`, `cash_end`, `total_cost`, `turnover` | the resulting portfolio | +| `model` | covariance estimator, scenario model and seed, expected-return model, the views and the Ω that was calibrated from them | +| `scenario_key`, `scenario_hash` | what the scenarios depend on, and a fingerprint of the draws | +| `warnings` | **read these** — see below | +| `timings` | seconds per stage | + +So a sentence in your advisory output can be backed end to end: *"CVaR 95 % = CHF 29 999 over one +week (mean–CVaR MILP, EWM λ = 0.99, GBM scenarios, seed 42, solved to optimality on HiGHS)."* + +**`warnings` is not decoration.** It is where the service tells you that the answer is technically +correct and economically doubtful: a risk limit that could not bind, a covariance estimate too +singular for the number of candidates, too few scenarios per candidate, an expected return above +60 % p.a. on a stock it decided to hold. Surface them or handle them; do not drop them. + +**Determinism.** The same request gives the same scenarios (`scenario_hash`) and the same portfolio, +on the same machine. Across machines — a different numpy or pandas — the `scenario_key` still +matches while the digest may not, because floating-point results agree to 13–14 digits rather than +bit for bit; the figures agree to well under a franc. Compare runs by `scenario_key`. For a demo +that must be identical next week, freeze the data with a snapshot (`POST /snapshots`). + +## 11. Things that will bite you + +* **The service answers one request at a time while a solve runs.** The MILP holds the interpreter. + Do not build a UI that fires an optimisation and then polls something else — it will look hung. +* **`time_limit` is the budget of the whole request**, not of each solver call (row generation and + the Gaussian mode solve a *sequence* of MILPs). Without it, a large candidate set on HiGHS can run + for hours. Set it. 300 s is a sane default. +* **The GPU solver is optional.** `solver="auto"` uses cuOpt if it answers and HiGHS otherwise; the + result says which, and adds a warning. A cuOpt outage degrades speed, never correctness. +* **`risk_drift="model"` (the default) measures the CVaR on scenarios that include the expected + return.** At long horizons the drift grows with *h* while the spread grows with *√h*, so a bullish + forecast can push the whole loss tail above zero and the risk limit stops binding — the optimiser + will then happily propose something absurd. Use `risk_drift="zero"` for anything beyond a few + weeks; the objective still maximises expected return, but the risk measurement no longer believes + the forecast. +* **Scenario count drives stability.** The default is 16 per candidate (within 4 096…16 384). The + reported figures are stable from ~2 000 on, but the *composition* moves between near-equivalent + portfolios; about 64 per candidate pins it. The result warns below 8. +* **`scenario_model="gaussian_exact"` is fast but bounds the CVaR from below.** It replaces the + scenarios with linear cuts, and until those converge the limit can be overshot by a hair (a franc + or two on a million, reported in `warnings`). Use it to explore; use `"gbm"` or `"empirical"` when + the limit has to hold exactly. +* **`Infeasible` means your constraints contradict each other**, most often `min_invested` against a + tight `cvar_limit`, or a `max_turnover` that cannot reach the risk budget. Relax one and re-run. + +## 12. Troubleshooting + +| symptom | what it means | +|---|---| +| `cannot reach the quant-core API at ...` | no service there. `quantcore serve`, or fix `--url` / `QUANTCORE_API_URL` | +| `503 no market data yet` | the store is empty: `POST /jobs/setup` (~15 min) or mount a prepared data directory | +| `422` with a field name | your request does not match the schema; the message names the field. `/docs` has the valid values | +| `/health` shows `cuopt.reachable: false` | the GPU solver is down; optimisations fall back to HiGHS and say so | +| `FeasibleFound` instead of `Optimal` | the time limit hit; raise `time_limit`, or lower `n_scenarios` / `max_assets` | +| an optimisation "seems stuck" | it is solving, and the service is single-threaded. `timings` afterwards shows where the time went | +| numbers differ slightly from a colleague's | different environment, same inputs: compare `scenario_key`, not `scenario_hash` (§10) | +| Yahoo errors during a data job | rate limiting; the client retries and opens a circuit. Re-run later, the stored data is intact | + +Everything failing at once? `quantcore-api selftest` tells you whether it is the service or you. + +## 13. Where the details are + +| document | what is in it | +|---|---| +| `/docs` | every endpoint and field, interactive — the contract | +| `README.md` | the services, the library API, quick start | +| `docs/ARCHITECTURE.md` | how the pieces fit, what depends on what, where the AI layer plugs in | +| `docs/METHODOLOGY.md` | every formula, estimator and convention, with the measured numbers | +| `docs/optimization_model.pdf` | the MILP derived from Rockafellar–Uryasev to whole shares, and the sampling theory | +| `docs/measurement_and_models.pdf` | risk and performance measures, GBM and empirical innovations, Black–Litterman with Idzorek confidence | diff --git a/docs/measurement_and_models.pdf b/docs/measurement_and_models.pdf new file mode 100644 index 0000000..5da216e Binary files /dev/null and b/docs/measurement_and_models.pdf differ diff --git a/docs/measurement_and_models.tex b/docs/measurement_and_models.tex new file mode 100644 index 0000000..d1d9f45 --- /dev/null +++ b/docs/measurement_and_models.tex @@ -0,0 +1,807 @@ +% !TEX encoding = UTF-8 Unicode +% --------------------------------------------------------------------------------------------- +% quantcore -- documentation of the measurement layer, the scenario models and the +% expected-return model, as implemented in quantcore/measurement and quantcore/models. +% Build: latexmk -pdf docs/measurement_and_models.tex +% --------------------------------------------------------------------------------------------- +\documentclass[11pt,a4paper]{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage[british]{babel} +\usepackage{lmodern} +\usepackage{microtype} +\usepackage[margin=2.4cm]{geometry} +\usepackage{amsmath,amssymb,amsthm,mathtools} +\usepackage{booktabs,array} +\usepackage{enumitem} +\usepackage{xcolor} +\usepackage[colorlinks=true,linkcolor=blue!50!black,citecolor=blue!50!black,urlcolor=blue!50!black]{hyperref} + +\newtheorem{theorem}{Theorem}[section] +\newtheorem{proposition}[theorem]{Proposition} +\theoremstyle{definition} +\newtheorem{definition}[theorem]{Definition} +\newtheorem{example}[theorem]{Example} +\theoremstyle{remark} +\newtheorem{remark}[theorem]{Remark} + +\DeclareMathOperator{\CVaR}{CVaR} +\DeclareMathOperator{\VaR}{VaR} +\DeclareMathOperator{\E}{\mathbb{E}} +\DeclareMathOperator{\Var}{Var} +\DeclareMathOperator{\Cov}{Cov} +\DeclareMathOperator{\diag}{diag} +\DeclareMathOperator{\tr}{tr} +\newcommand{\R}{\mathbb{R}} +\newcommand{\pos}[1]{\left[#1\right]^{+}} +\newcommand{\code}[1]{\texttt{\hyphenchar\font=-1 #1}} +\newcommand{\bs}{\boldsymbol} +\setlength{\parskip}{4pt} +\setlength{\parindent}{0pt} + +\title{Risk and Performance Measurement, Scenario Generation\\ and the Expected-Return Model\\[1ex] +\large From log-returns to VaR/CVaR, from the covariance estimators to simulated horizon +scenarios, and from market capitalisations to Black--Litterman expected returns with +Idzorek confidence views, as implemented in \code{quantcore}} +\author{AISE502 -- AI-Augmented Portfolio Intelligence Platform\\ \small Deterministic quant core, modules \code{quantcore.measurement} and \code{quantcore.models}} +\date{September 2026} + +\begin{document} +\maketitle + +\begin{abstract} +\noindent +This note is the companion of \emph{Mean--CVaR Portfolio Optimisation in Whole Shares} +(the file \code{optimization\_model.pdf} in \code{docs/}). It documents the three model layers that feed the +optimiser and the platform's risk and performance figures: (i) the \emph{measurement} layer --- +log-returns in CHF, moving-window and exponentially weighted estimators of mean, volatility and +covariance, Value-at-Risk and Conditional Value-at-Risk at 95\,\% and 99\,\% for a day, a week +or a month, and the performance statistics; (ii) the \emph{stock-price models} that turn the +latest covariance into a set of horizon scenarios --- geometric Brownian motion and the +empirical-innovation model $\varepsilon_t=\Sigma_{t-1}^{-1/2}r_t$, $r(j\,|\,T)=\Sigma_T^{1/2}\varepsilon_j$; +and (iii) the \emph{expected-return model} --- market-capitalisation weights, the broad-market +average, reverse optimisation, and the Black--Litterman posterior with views whose uncertainty is +calibrated from a degree of belief following Idzorek. Every formula is stated with its +conventions and defaults, the numerical safeguards (covariance shrinkage, innovation +re-standardisation) are motivated with measurements on the platform's own data set, and the last +section maps the mathematics to the functions, parameters and output fields of the code. +\end{abstract} + +\tableofcontents + +% ============================================================================================ +\section{Data, conventions and notation} +% ============================================================================================ + +The universe consists of $N$ stocks with adjusted closing prices $P_i(t)$ in their listing +currency $c(i)$, observed on the \emph{union calendar} of all exchanges (a day on which at least +one exchange in the universe trades). A holiday of one exchange is filled with that exchange's +previous close (at most five days), so every trading day of the union calendar has a price for +every stock. Prices that are cleaned away (broken adjustments, spike ticks; see +\code{docs/METHODOLOGY.md}, Section~1) are treated the same way. + +FX rates $X_c(t)$ are quoted as CHF per one unit of currency $c$ (so $X_{\mathrm{CHF}}\equiv1$), +and the base currency of everything below is CHF. + +Three frequencies are used: \emph{daily} (union calendar, 252 periods per year), \emph{weekly} +(Friday closes, 52) and \emph{monthly} (month-end closes, 12); $\pi$ denotes the number of +periods per year of the chosen frequency. A \emph{horizon} is expressed as a number $h$ of +periods: at daily frequency a day is $h=1$, a week $h=5$, a month $h=21$; at weekly frequency +a week is $h=1$ and a month $h=4$. + +Throughout, $r$ denotes a log-return, $R=e^{r}-1$ the corresponding simple return, and +\emph{losses are positive numbers}. Vectors are columns; $\mathbf 1$ is the vector of ones. + +% ============================================================================================ +\section{Returns} +% ============================================================================================ + +\subsection{Log-returns and the currency conversion} + +The one-period log-return of stock $i$ in its listing currency is +\begin{equation} + r_i(t) \;=\; \ln P_i(t) - \ln P_i(t-1). + \label{eq:logret} +\end{equation} +The price in CHF is $P_i^{\mathrm{CHF}}(t)=P_i(t)\,X_{c(i)}(t)$, hence its log-return is the +\emph{sum} of the local return and the FX return, +\begin{equation} + r_i^{\mathrm{CHF}}(t) \;=\; r_i(t) + \big(\ln X_{c(i)}(t) - \ln X_{c(i)}(t-1)\big), + \label{eq:chfret} +\end{equation} +exactly and without any cross term --- the reason log-returns are the working unit of the whole +platform. (With simple returns the conversion is $(1+R_i)(1+R_X)-1$, which is neither additive +nor symmetric.) The implementation converts prices with the FX frame and takes the log-return of +the product; \eqref{eq:chfret} is checked by a unit test. + +\subsection{Aggregation over time and over assets} + +Under the i.i.d.\ assumption of the price models (Section~\ref{sec:models}) log-returns add over +time: the $h$-period return is $r^{(h)}(t)=\sum_{k=0}^{h-1} r(t-k)$. The historical risk +measures use \emph{overlapping} $h$-period sums (more observations, mildly autocorrelated); +non-overlapping blocks are available as an option. + +Over assets, log-returns do not add. The exact log-return of a portfolio whose value weights +$w_i$ are fixed at the start of the period is +\begin{equation} + r_p \;=\; \ln\Big(1+\sum_i w_i\,(e^{r_i}-1)\Big), + \label{eq:portret} +\end{equation} +because simple returns aggregate linearly. A cash residual $1-\mathbf 1^\top w$ earns zero. This +formula (\code{portfolio\_log\_returns}) is used wherever a portfolio return series is needed +--- the historical portfolio VaR and the portfolio performance statistics. + +\subsection{Why log-returns are annualised additively} + +Means of log-returns scale with time: $\E[r^{(h)}]=h\mu$, and under independence +$\Var[r^{(h)}]=h\sigma^2$. An annualised log-return is therefore $\pi\mu$, an annualised +volatility $\sqrt{\pi}\,\sigma$; the geometric (compounded) annual return is $e^{\pi\mu}-1$. Both +are reported. + +% ============================================================================================ +\section{Moving-window and exponentially weighted estimators} +\label{sec:estimators} +% ============================================================================================ + +Every statistic exists in two flavours (\code{quantcore.measurement.estimators}). + +\subsection{Moving window} + +With window length $w$ (default 200 periods) the estimates at date $t$ use the observations +$t-w+1,\dots,t$ with equal weight $1/w$: the sample mean $m_t$, the sample standard deviation +(divisor $w-1$), the sample covariance +\begin{equation} + \Sigma_t^{\mathrm{roll}} \;=\; \frac{1}{w-1}\sum_{k=0}^{w-1}(r_{t-k}-m_t)(r_{t-k}-m_t)^\top . + \label{eq:rollcov} +\end{equation} +Rows with a missing value are dropped for matrices; a series needs at least $w/2$ observations +before a value is reported. + +\subsection{Exponential weighting} + +With decay $\lambda\in(0,1)$ the observation $k$ periods back receives the weight +$\lambda^k$ (RiskMetrics \cite{RiskMetrics1996}); the weights are normalised over the available +history, +\begin{equation} + \omega_k \;=\; \frac{\lambda^k}{\sum_{j=0}^{n-1}\lambda^j}, \qquad k=0,\dots,n-1, + \label{eq:ewmw} +\end{equation} +(pandas \code{ewm(alpha=1-$\lambda$, adjust=True)}), and the estimators are the weighted +moments +\begin{equation} + m_t^{\mathrm{ewm}} = \sum_k\omega_k\, r_{t-k}, \qquad + \Sigma_t^{\mathrm{ewm}} = \sum_k \omega_k\,(r_{t-k}-m_t^{\mathrm{ewm}})(r_{t-k}-m_t^{\mathrm{ewm}})^\top, + \label{eq:ewmcov} +\end{equation} +with the diagonal of \eqref{eq:ewmcov} as the EWM variance (\code{var(bias=True)}). Both +estimators are \emph{centred}: the corresponding mean is subtracted. The effective window is +$1/(1-\lambda)$ observations (half-life $\ln 0.5/\ln\lambda$): + +\begin{center} +\small +\begin{tabular}{@{}lcccc@{}} +\toprule +$\lambda$ & 0.94 & 0.97 & 0.99 & 0.997 \\ +\midrule +effective window $1/(1-\lambda)$ & 17 & 33 & 100 & 333 \\ +half-life (periods) & 11.2 & 22.8 & 69.0 & 230.7 \\ +weight of the observation 100 periods back & 0.002 & 0.048 & 0.366 & 0.741 \\ +\bottomrule +\end{tabular} +\end{center} + +$\lambda=0.94$ is the RiskMetrics value for daily data and the default of the measurement +services (fast adaptation, noisy); $\lambda=0.99$ is the default of the optimisation covariance +(a smoother matrix that still adapts within a quarter). For matrices the history is truncated +where the weights fall below $10^{-4}$ (917 observations for $\lambda=0.99$). + +\subsection{Sequences of matrices} + +The innovation filter of Section~\ref{sec:fhs} needs $\Sigma_{t-1}$ for \emph{every} date $t$, +i.e.\ the covariance estimated from the data up to and including $t-1$. Storing $T$ matrices of +size $N\times N$ is impossible for a thousand stocks, so \code{covariance\_sequence} streams +them: the rolling estimator recomputes \eqref{eq:rollcov} for each date ($O(wN^2)$ per step), +while the exponentially weighted estimator uses the exact recursion of the centred filter with +$\alpha=1-\lambda$, +\begin{equation} + d_t = r_t - m_{t-1}, \qquad m_t = m_{t-1} + \alpha\, d_t, \qquad + \Sigma_t = \lambda\,\Sigma_{t-1} + \alpha\, d_t\,(r_t-m_t)^\top , + \label{eq:ewmrec} +\end{equation} +which costs $O(N^2)$ per step and, after a warm-up of $\max(50, 2/(1-\lambda))$ observations +initialised with \eqref{eq:ewmcov}, coincides with the normalised-weight estimate up to the +factor $1/(1-\lambda^n)\to1$ (verified by a unit test at 5\,\% for $\lambda=0.97$). + +\subsection{Shrinkage towards the diagonal} +\label{sec:shrinkage} + +A covariance estimated from $n$ effective observations is ill-conditioned when $N$ is not small +against $n$ and singular for $N>n$: the smallest eigenvalues are underestimated, the largest +overestimated (the Marchenko--Pastur effect), and anything that inverts the matrix --- the +innovation filter, mean-variance weights --- amplifies the error. The classical remedy is +linear shrinkage \cite{LedoitWolf2004} towards a structured target; the platform shrinks towards +the diagonal (the target of Schäfer and Strimmer \cite{SchaeferStrimmer2005}), which keeps the +variances and pulls the correlations towards zero: +\begin{equation} + \Sigma^{(s)} \;=\; (1-s)\,\Sigma + s\,\diag(\Sigma), \qquad s\in[0,1]. + \label{eq:shrink} +\end{equation} +The intensity can be given, or estimated from the data (\code{shrinkage="auto"}): for centred +observations $x_k$ ($k=1,\dots,n$), products $w_{kij}=x_{ki}x_{kj}$ and sample covariances +$s_{ij}=\frac{n}{n-1}\bar w_{ij}$, the intensity that minimises the expected squared Frobenius +loss of \eqref{eq:shrink} is +\begin{equation} + s^\ast \;=\; \frac{\sum_{i\ne j}\widehat{\Var}(s_{ij})}{\sum_{i\ne j}s_{ij}^2}, + \qquad + \widehat{\Var}(s_{ij}) \;=\; \frac{n}{(n-1)^3}\sum_{k=1}^n \big(w_{kij}-\bar w_{ij}\big)^2 , + \label{eq:ss} +\end{equation} +clipped to $[0,1]$. It is small for a long, strongly correlated sample and approaches one when +$N$ approaches $n$ --- exactly when the sample covariance becomes unreliable. For the EWM +estimator the intensity is computed on the last $3/(1-\lambda)$ observations with equal weights. +On the platform's data set (daily CHF returns, $\lambda=0.99$) the automatic intensity is 0.047 +for the 100 largest stocks, 0.069 for 300 and 0.114 for the full universe of 745 names. The +sequence \eqref{eq:ewmrec} uses one intensity for all its matrices. + +% ============================================================================================ +\section{Risk measures} +\label{sec:risk} +% ============================================================================================ + +\subsection{Loss, VaR and CVaR} + +For a position of value $V_0$ and a horizon log-return $r$, the loss in return space is +$\ell=-r$ and in money it is $V_0(1-e^{r})=V_0(1-e^{-\ell})$. The Value-at-Risk at level +$\beta$ is the $\beta$-quantile of the loss distribution, the Conditional Value-at-Risk (expected +shortfall) the mean of the loss beyond it, +\begin{equation} + \VaR_\beta = q_\beta(\ell), \qquad + \CVaR_\beta = \frac{1}{1-\beta}\int_\beta^1 q_\gamma(\ell)\,d\gamma + \;\;\big(=\E[\ell\mid\ell\ge\VaR_\beta]\text{ for continuous }\ell\big), + \label{eq:varcvar} +\end{equation} +see \cite{AcerbiTasche2002,MFE2015}. The platform reports both at $\beta=0.95$ and $0.99$. +Because $\ell\mapsto V_0(1-e^{-\ell})$ is increasing, quantiles commute with it, and a +return-space VaR translates into money exactly as $V_0(1-e^{-\VaR_\beta})$; the CVaR in money +is computed the same way from the return-space CVaR (which is exact for VaR and a very good +approximation for CVaR: for a weekly CVaR of 4.6\,\% the difference between the exact tail mean +of $V_0(1-e^{-\ell})$ and $V_0(1-e^{-\CVaR})$ is below 0.1\,\% of $V_0$; the scenario-based +portfolio risk of Section~\ref{sec:portfoliorisk} works in money directly and is exact). + +\subsection{Gaussian (GBM) VaR and CVaR} + +If $r\sim N(\mu,\sigma^2)$ then $\ell=-r\sim N(-\mu,\sigma^2)$, its $\beta$-quantile is +$-\mu+z_\beta\sigma$ with $z_\beta=\Phi^{-1}(\beta)$, and the tail mean follows from +$\E[Z\mid Z\ge z_\beta]=\varphi(z_\beta)/(1-\beta)$ for a standard normal $Z$: +\begin{equation} + \VaR_\beta = -\mu + z_\beta\,\sigma, \qquad + \CVaR_\beta = -\mu + \sigma\,\frac{\varphi(z_\beta)}{1-\beta}. + \label{eq:normal} +\end{equation} +The multipliers are $z_{0.95}=1.645$, $\varphi(z_{0.95})/0.05=2.063$, $z_{0.99}=2.326$, +$\varphi(z_{0.99})/0.01=2.665$. Under the i.i.d.\ assumption the horizon parameters are +$\mu_h=h\mu$ and $\sigma_h=\sqrt h\,\sigma$; $\mu$ and $\sigma$ come from either estimator of +Section~\ref{sec:estimators}. The mean is kept (a positive drift lowers the loss quantile +one-for-one); for daily data it is negligible, for a month it is not. + +\begin{example}[Reference values] + A stock with daily volatility $\sigma=1\,\%$ and zero mean, one week ($h=5$, + $\sigma_h=2.236\,\%$): $\VaR_{0.95}=3.678\,\%$, $\CVaR_{0.95}=4.612\,\%$, $\VaR_{0.99}=5.202\,\%$, + $\CVaR_{0.99}=5.960\,\%$. For a position of CHF~100\,000 the exact money figures are + $\VaR_{0.95}=3\,611$, $\CVaR_{0.95}=4\,508$, $\VaR_{0.99}=5\,069$, $\CVaR_{0.99}=5\,786$ (the + linear approximation $V_0\cdot\CVaR$ would give 4\,612 and 5\,960). The daily figures + $\VaR_{0.95}=1.645\,\%$, $\CVaR_{0.95}=2.063\,\%$, $\VaR_{0.99}=2.326\,\%$, $\CVaR_{0.99}=2.665\,\%$ + and the weekly ones are pinned by \code{tests/test\_risk.py}. +\end{example} + +\subsection{Historical VaR and CVaR} + +The empirical counterparts replace the Gaussian by the empirical distribution of the realised +$h$-period returns $r^{(h)}$ over a window of $\max(5w,750)$ periods: +\begin{equation} + \VaR_\beta^{\mathrm{hist}} = -\hat q_{1-\beta}\big(r^{(h)}\big), \qquad + \CVaR_\beta^{\mathrm{hist}} = -\frac{1}{|\mathcal T|}\sum_{t\in\mathcal T} r^{(h)}_t, \quad + \mathcal T=\big\{t: r^{(h)}_t\le \hat q_{1-\beta}\big\}, + \label{eq:hist} +\end{equation} +with $\hat q$ the empirical quantile (linear interpolation). They capture fat tails and skewness +that the Gaussian misses, at the price of noise: at $\beta=0.99$ with 750 daily observations the +tail contains eight points. + +\subsection{Discrete scenario sets} + +For $S$ equiprobable losses (a simulated scenario set, Section~\ref{sec:models}) the definition +\eqref{eq:varcvar} evaluates exactly: with $k=(1-\beta)S$ and the losses sorted decreasingly +$\ell_{(1)}\ge\ell_{(2)}\ge\dots$, +\begin{equation} + \CVaR_\beta = \frac1k\Big(\sum_{s=1}^{\lfloor k\rfloor}\ell_{(s)} + (k-\lfloor k\rfloor)\,\ell_{(\lfloor k\rfloor+1)}\Big), + \qquad \VaR_\beta = \ell_{(\lceil k\rceil)} , + \label{eq:scen} +\end{equation} +which is the value the Rockafellar--Uryasev linear programme of the optimisation returns +(function \code{scenario\_cvar}; see Remark~2.3 of the companion note). For $S=2\,000$ and +$\beta=0.99$ the tail has $k=20$ scenarios. + +\subsection{Per-asset tables and time series} + +\code{RiskService.asset\_risk} returns, for a list of stocks, a frequency, a horizon, a window +$w$ and a decay $\lambda$, one row per stock with the annualised rolling and EWM volatilities and, +for each confidence level $\beta\in\{0.95,0.99\}$, the Gaussian VaR and CVaR from the rolling +estimator, the same from the EWM estimator, and the historical VaR and CVaR --- six figures per +level, all in log-return units over the horizon (Table~\ref{tab:impl}). \code{risk\_series} gives +the same quantities as time series: Gaussian figures from the rolling / EWM volatility and mean +series, historical ones from rolling quantiles and rolling tail means of the one-period returns, +scaled by $\sqrt h$ for longer horizons (an approximation used only in the series). + +\subsection{Portfolio risk in money} +\label{sec:portfoliorisk} + +A portfolio is given in \emph{numbers of shares} $n_i$ plus cash $C$. With today's CHF prices +$P_i$ the value is $V_0=C+\sum_iP_in_i$ and the value weights are $w_i=P_in_i/V_0$ (the cash +weight $C/V_0$ is riskless). Three methods are offered (\code{portfolio\_risk}): +\begin{itemize}[nosep] + \item \textbf{normal}: $\sigma_p^2=w^\top\Sigma w$ with the rolling or EWM covariance, + $\mu_p=w^\top m$ with the window mean, then \eqref{eq:normal} with $h\mu_p$ and + $\sqrt h\sigma_p$, converted to money; + \item \textbf{historical}: the realised portfolio return series \eqref{eq:portret} over the + last $\max(5w,750)$ periods, aggregated to $h$ periods, then \eqref{eq:hist}; + \item \textbf{scenarios}: money P\&L $\Pi_s=V_0\sum_iw_i(e^{r_{s,i}}-1)$ for each scenario of a + simulated set, then \eqref{eq:scen} on the losses $-\Pi_s$ --- exact in money, and + precisely the risk the optimiser constrains. +\end{itemize} +The result (\code{PortfolioRisk}) carries the value, the weights, the horizon, the method, the +annualised portfolio volatility, and per confidence level the VaR and CVaR in return space and in +money. A unit test checks that Gaussian scenarios with the same mean and covariance reproduce the +normal figures within sampling error. + +% ============================================================================================ +\section{Performance measures} +% ============================================================================================ + +All statistics (\code{performance\_table}) are computed from one-period log-returns $r_t$, +$t=1,\dots,T$, at the chosen frequency with $\pi$ periods per year and the annual CHF risk-free +rate $r_f$ (0 by default): +\begin{align} + \text{cumulative return} &= e^{\sum_t r_t}-1, \quad + \text{ann.\ log-return} = \pi\,\bar r, \quad + \text{ann.\ return} = e^{\pi\bar r}-1, \notag\\ + \text{ann.\ volatility} &= \sqrt{\pi}\,\hat\sigma, \quad + \text{Sharpe} = \frac{\pi\bar r - r_f}{\sqrt\pi\,\hat\sigma}, \quad + \text{Calmar} = \frac{e^{\pi\bar r}-1}{-\mathrm{MDD}}, \notag\\ + \text{Sortino} &= \frac{\pi\bar r - r_f}{\sqrt{\pi\cdot\frac1T\sum_t \min(r_t,0)^2}}, + \label{eq:perf} +\end{align} +where $\hat\sigma$ is the sample standard deviation (divisor $T-1$) and the maximum drawdown +$\mathrm{MDD}=\min_t\big(V_t/\max_{s\le t}V_s-1\big)$ is taken on the value path +$V_t=\exp(\sum_{s\le t}r_s)$. Skewness and excess kurtosis of $r_t$, the hit ratio (share of +positive periods), the best and the worst period, the number of observations and the start and +end dates complete the row. + +\emph{Rolling} versions use the window mean and standard deviation of +Section~\ref{sec:estimators} for the annualised return, volatility and Sharpe ratio, plus the +cumulative return over the window and the running drawdown; \emph{EWM} versions use the +exponentially weighted mean and standard deviation. + +For a portfolio in shares the return series is either the \emph{constant-mix} series +\eqref{eq:portret} with today's value weights (the natural benchmark of a rebalanced +allocation) or the \emph{buy-and-hold} series $\ln(V_t/V_{t-1})$ with $V_t=\sum_iP_i(t)n_i$ +(the share numbers held over the whole history). Both are available; the API's +\code{/performance/portfolio} defaults to constant mix. + +% ============================================================================================ +\section{Stock-price models and scenario generation} +\label{sec:models} +% ============================================================================================ + +The optimiser and the scenario-based portfolio risk need a joint distribution of the horizon +log-returns of the candidate stocks. Two models are implemented +(\code{quantcore.models.simulation}); both assume that one-period log-returns are i.i.d.\ (no +autocorrelation), so that an $h$-period return is the sum of $h$ independent one-period draws, +both use the \emph{latest} covariance $\Sigma_T$ of a \code{CovarianceModel} (rolling window or +EWM, with shrinkage), and both take the expected returns $\mu$ from the expected-return model of +Section~\ref{sec:er} (converted to the log-drift $m$ as described in Section~\ref{sec:sampling-mm}). +The output is an $S\times N$ matrix of horizon log-returns $r_{s,i}$, optionally with scenario +probabilities, together with its provenance (model, covariance model and the shrinkage actually +applied, sampling scheme, seed, horizon, date, a fingerprint of the matrix), the \code{ScenarioSet}. + +\subsection{Geometric Brownian motion} + +Under multivariate GBM the horizon log-return is multivariate normal, +\begin{equation} + r^{(h)} \;\sim\; N\big(h m,\; h\Sigma_T\big), + \label{eq:gbm} +\end{equation} +sampled as $r_s=hm+\sqrt h\,A z_s$ with $z_s\sim N(0,I)$ and $AA^\top=\Sigma_T$ (the +eigen-root $V\Lambda^{1/2}$ with eigenvalues descending by default, so that the first coordinate +of $z_s$ drives the market factor; the symmetric and Cholesky roots give the same distribution). +The draws $z_s$ are scrambled Sobol' points passed through $\Phi^{-1}$ and moment-matched +(Section~\ref{sec:sampling-mm}), so mean and covariance of the set are exact. GBM is the natural +partner of the Gaussian risk figures of Section~\ref{sec:risk} and of mean-variance thinking; it +has no fat tails, no skewness and no tail dependence. + +\subsection{Empirical innovations (filtered historical simulation)} +\label{sec:fhs} + +The second model keeps the shape of the observed return distribution and only rescales it to +today's covariance. It is the multivariate form of filtered historical simulation +\cite{BaroneAdesi1999}. + +\paragraph{Filtering.} For every date $t$ after the warm-up of the estimator, the return vector +is standardised with the covariance known at the end of the previous day, +\begin{equation} + \varepsilon_t \;=\; \Sigma_{t-1}^{-1/2}\, r_t , + \label{eq:innov} +\end{equation} +where $\Sigma_{t-1}$ comes from the streamed sequence \eqref{eq:ewmrec} (or the rolling +recomputation) and $\Sigma^{-1/2}$ is the \emph{symmetric} inverse root $V\Lambda^{-1/2}V^\top$ +from the eigendecomposition $\Sigma=V\Lambda V^\top$ (eigenvalues floored at $10^{-10}$ of the +largest; the Cholesky root is available and used automatically above 250 stocks, where the +$O(TN^3)$ eigendecompositions become slow). If the model were exactly right, the innovations would +be uncorrelated with unit variance --- and, unlike Gaussian noise, they carry the empirical fat +tails, skewness and non-linear co-movements of the data. + +\paragraph{Resampling.} With $A=\Sigma_T^{1/2}$ (symmetric root) and log-drift $m$, the $s$-th +horizon scenario draws $h$ dates $j_1,\dots,j_h$ with replacement from the innovation history +and sets +\begin{equation} + r_s \;=\; hm + A\sum_{k=1}^{h}\tilde\varepsilon_{j_k} , + \label{eq:fhs} +\end{equation} +where $\tilde\varepsilon$ are the centred and re-standardised innovations described next. Since +the $h$ draws are independent with identity covariance, $\Cov(r_s)=hA A^\top=h\Sigma_T$, +consistent with \eqref{eq:gbm}, while the marginal shape of every asset and the joint tail +behaviour come from the data. The dates are drawn by scrambled Sobol' points in $h$ dimensions +mapped to the ranks of the innovations sorted by their market-factor component (so that every +day of the horizon covers the loss distribution evenly), alternatively uniformly (\code{"mc"}) +or balanced (every date equally often); the resampled sums are moment-matched to mean zero and +covariance $hI$ (Section~\ref{sec:sampling-mm}). + +\paragraph{Centring and re-standardisation.} The innovations are centred so that the scenario +mean equals the supplied drift exactly (the drift is the expected-return model's job, not the +sample mean's). Their variance, however, is systematically \emph{above one}: any lagged +volatility filter is too low right after a shock, so the standardised returns of the following +days are too large --- the classical reason why filtered historical simulation rescales the +residuals before resampling \cite{BaroneAdesi1999}. Measured on the platform's data set (daily +CHF returns 2016--2026, EWM $\lambda=0.99$, automatic shrinkage, symmetric root): + +\begin{center} +\small +\begin{tabular}{@{}lccc@{}} +\toprule +number of stocks $N$ & 10 & 30 & 100 \\ +\midrule +median variance of the raw innovations & 1.11 & 1.25 & 1.61 \\ +scenario volatility / model volatility without re-standardisation & 1.06 & 1.13 & 1.28 \\ +\bottomrule +\end{tabular} +\end{center} + +The growth with $N$ is the conditioning effect of Section~\ref{sec:shrinkage}: with an +effective window of 100 days and 100 stocks, $\Sigma_{t-1}$ is nearly singular and its inverse +root inflates the directions it underestimates. The implementation therefore rescales the +centred innovation sample before resampling: \code{standardize="diagonal"} (default) divides +each asset's innovations by their sample standard deviation, \code{"full"} whitens them with +the inverse root of their sample covariance (only when the history has at least $3N$ dates). +After rescaling, the per-asset scenario volatility matches the model volatility within +$\pm2\,\%$ (diagonal) or $\pm0.5\,\%$ (full), and the scenario correlations match the model's +with a mean absolute error of 0.02 (diagonal) or 0.004 (full). The diagonal option leaves every +asset's marginal distribution intact and works for any $N$, which is why it is the default. +Finally, innovations beyond $\pm8$ are winsorised as a numerical guard against the same +conditioning effect (about 0.2\,\% of the values on the data set); this is a safeguard, not a +model choice, and it is reported. + +\subsection{Sampling, moment matching, the drift convention and importance sampling} +\label{sec:sampling-mm} + +\paragraph{Why not plain Monte Carlo.} The optimiser is a sample-average approximation: it sees +the model only through the scenario set, and the sample mean of a plain Monte-Carlo set of +$S$ scenarios is off by $\sigma_i/\sqrt S$ per asset --- for $S=2\,000$ and weekly volatilities +of 3--11\,\% that is 3--13\,\% per year, the size of the expected returns themselves. Measured +on the 30 largest stocks, two seeds then give portfolios that differ by 100\,\% of wealth and a +true expected profit 10\,\% below the optimum. The remedies (all in +\code{quantcore.models.sampling}, derived and measured in the optimisation note) are: +scrambled Sobol' points instead of pseudo-random draws ($S$ is rounded up to a power of two); +\emph{moment matching}, the affine transform $\tilde z_s=C^{-1/2}(z_s-\bar z)$ of the sample +that makes its (probability-weighted) mean and covariance exactly $0$ and $I$, so that every +scenario set has mean $hm$ and covariance $h\Sigma_T$ exactly and only the shape of the tail is +sampled; and the model expected returns in the optimiser's objective instead of the scenario +mean. With these, the turnover between two seeds falls to 4--6\,\%, the flatness of the optimum. + +\paragraph{Drift convention.} The expected-return model delivers \emph{arithmetic} expected +returns $\mu$ (Section~\ref{sec:er}): reverse optimisation and Black--Litterman are statements +about simple returns. The scenarios are generated in log-returns and turned into money with +$R=e^{r}-1$, whose expectation under normality is $e^{hm+h\sigma^2/2}-1$. The log-drift is +therefore the It\^o-corrected +\begin{equation} + m_i \;=\; \frac{\ln(1+h\mu_i)}{h}-\frac{\sigma_i^2}{2}, + \qquad\text{so that}\qquad \E\big[e^{r^{(h)}_i}-1\big]=h\mu_i + \label{eq:ito} +\end{equation} +exactly (\code{drift\_convention="arithmetic"}, the default). Using $\mu_i$ itself as the +log-drift (\code{"log"}) adds $\sigma_i^2/2$ to the expected simple return: on the data set the +median bonus over the 30 largest stocks is 5.4\,\% per year against a median $\mu$ of 8.9\,\%, +and for the most volatile names it is 20--34\,\% (Micron: $\mu=16.9\,\%$ would become +$65.6\,\%$) --- a strong and unintended tilt towards volatile stocks. The scenario set reports +the expected simple return per asset (\code{expected\_simple}) that the optimiser uses. + +\paragraph{Importance sampling.} Only the $(1-\beta)S$ scenarios in the tail matter for the +CVaR. By default (\code{importance\_sampling="auto"}) the sampling distribution is therefore +tilted towards the losses of the \emph{cap-weighted portfolio of the candidate stocks}: its loss +direction is $d=-A^\top v/\|A^\top v\|$, and two blocks of a quarter of the scenarios each are +shifted along $d$ by $z_{0.95}$ and $z_{0.99}$ (GBM: $z_s\mapsto z_s+c_kd$; empirical model: the +dates are drawn with exponentially tilted probabilities +$q^k_t\propto\exp(\vartheta_k d^\top\tilde\varepsilon_t)$), so that one set covers both usual +confidence levels. The shift is a covariance statement: in return space it is +$-c_k\Sigma v/\sqrt{v^\top\Sigma v}$, so each asset moves by its covariance with the reference +portfolio --- a high-beta stock is stressed hard, a negatively correlated one moves up --- and, +an exponential tilt of a Gaussian being a mean shift, the weighted set still has exactly the +model covariance and correlations. Every scenario carries the self-normalised likelihood ratio of the mixture +proposal as its probability $p_s$, bounded by $1/(1-\sum_k\varepsilon_k)$; the weighted set is +moment-matched with these probabilities, the optimiser uses $p_s$ in its CVaR rows and the risk +figures of Section~\ref{sec:risk} use the weighted tail. At equal $S$ the tail noise falls by a +factor 2--4; at $\beta=0.99$ 2\,048 tilted scenarios are as good as 8\,192 plain ones. Because +the direction comes from the market portfolio rather than from a solution of the optimisation, +the scenario set does not depend on the risk limit, the constraints or the confidence level --- +one set serves a whole family of runs (\code{"adaptive"} tilts towards a first-pass solution +instead, which is sharper at $\beta=0.99$ but gives up that property). + +\paragraph{Which model when.} GBM is the transparent baseline; the empirical model is the +default of the optimisation because CVaR is a tail measure and the empirical tails are heavier +than Gaussian ones (the innovation kurtosis on the data set is well above zero). Both models +produce identical second moments by construction, so differences in the optimised portfolios +are entirely due to the tail shape. + +\paragraph{Cost.} The filter needs one eigendecomposition per date: for 100 stocks and 2\,500 +dates about two seconds, for 745 stocks with the Cholesky root and 1\,500 dates about six +seconds (four BLAS threads). The innovations depend only on the data and the covariance model, +so they can be cached between optimisations with different views or constraints. + +% ============================================================================================ +\section{The expected-return model} +\label{sec:er} +% ============================================================================================ + +The scenario drift $\mu$ --- and therefore the mean side of the mean--CVaR trade-off --- comes +from a Black--Litterman model (\code{quantcore.models.black\_litterman}) on \emph{weekly} CHF +log-returns over the last ten years. Weekly data reduce the asynchronous-trading noise of the +daily union calendar, and ten years give $\approx520$ observations for the long-term covariance. +Everything in this section is per week; annual figures are $52\times$ the means and +$\sqrt{52}\times$ the volatilities. + +\subsection{Market capitalisation, weights and the broad-market average} + +The market capitalisation of stock $i$ in CHF is +$M_i(t)=\text{shares outstanding}_i\times P_i(t)\times X_{c(i)}(t)$ with today's shares +outstanding held constant back in time. The \emph{cap-weighted broad market index} of the whole +universe has the log-return +\begin{equation} + r_M(t) \;=\; \ln\Big(1+\sum_i \omega_i(t-1)\,\big(e^{r_i(t)}-1\big)\Big), \qquad + \omega_i(t-1)=\frac{M_i(t-1)}{\sum_jM_j(t-1)}, + \label{eq:index} +\end{equation} +i.e.\ \eqref{eq:portret} with weights that drift with prices between rebalancings, exactly like +an index. Its average over the ten-year window, converted to an arithmetic expectation, +$\mu_M=\overline{r_M}+\tfrac12\widehat{\Var}(r_M)$, is the platform's estimate of the +expected market return ("the average of a broad market weighted by current capitalisation"); +on the data set as of 2026-09-21 the mean log-return is 11.5\,\% per year with a volatility of +16.2\,\%, i.e.\ $\mu_M=12.9\,\%$. The current weights $w_i=M_i(T)/\sum_jM_j(T)$, renormalised over the candidate stocks +of an optimisation, are the \emph{market weights} of the model. + +\subsection{Reverse optimisation: the equilibrium returns} + +A mean-variance investor with risk aversion $\delta$ solves +$\max_w\; w^\top\mu-\frac{\delta}{2}w^\top\Sigma w$, whose first-order condition is +$\mu=\delta\Sigma w$. Black and Litterman \cite{BlackLitterman1992,HeLitterman1999} read this +backwards: if the market portfolio $w$ is the optimal portfolio of the representative investor, +the excess returns that make it optimal are the \emph{implied equilibrium returns} +\begin{equation} + \pi \;=\; \delta\,\Sigma\,w, \qquad + \delta \;=\; \frac{\mu_M - r_f}{w^\top\Sigma w}, + \label{eq:pi} +\end{equation} +where $\delta$ is chosen such that the market portfolio itself earns exactly the market excess +return: $w^\top\pi=\delta\,w^\top\Sigma w=\mu_M-r_f$. Here $\Sigma$ is the ten-year weekly +covariance of the candidate stocks and $r_f$ the weekly CHF risk-free rate (0 by default). The +equilibrium returns are proportional to each stock's covariance with the market portfolio --- +its beta --- and are the neutral starting point: an investor without views holds the market. +On the data set, with the 100 largest stocks as candidates (their market portfolio has a +volatility of 19.7\,\%), $\delta\approx3.3$; the equilibrium returns range from about 2.7\,\% +(Unilever, Nestl\'e --- low beta in CHF) to about 23\,\% (Nvidia) per year. + +\begin{remark} + $\mu_M$ is the average of the \emph{broad} index of all 745 stocks, whereas $w$ and $\Sigma$ + refer to the candidate subset of an optimisation. This follows the specification (one + market-wide average) and keeps $\delta$ stable across candidate sets; the alternative of + using the candidate portfolio's own average is a one-line change. +\end{remark} + +\subsection{The Black--Litterman posterior} + +The model treats the (unknown) vector of expected excess returns $\mu$ as a random variable with +the equilibrium as prior, $\mu\sim N(\pi,\tau\Sigma)$, where the scalar $\tau$ (0.05 by default) +expresses how uncertain the equilibrium is relative to the return covariance. $K$ views are +linear statements about $\mu$ with their own uncertainty, +\begin{equation} + P\mu \;=\; Q+\varepsilon, \qquad \varepsilon\sim N(0,\Omega), \quad \Omega=\diag(\omega_1,\dots,\omega_K), + \label{eq:views} +\end{equation} +where each row $p_k$ of $P$ ($K\times N$) is a portfolio: $p_k=e_i$ for an \emph{absolute} view +("stock $i$ will return $q_k$"), $p_k=e_i-e_j$ for a \emph{relative} view ("$i$ will outperform +$j$ by $q_k$"). Combining prior and views by Bayes' rule (equivalently, generalised least squares) +gives the posterior distribution of $\mu$ with mean and covariance +\begin{equation} + \mu_{\mathrm{BL}} = \big[(\tau\Sigma)^{-1}+P^\top\Omega^{-1}P\big]^{-1}\big[(\tau\Sigma)^{-1}\pi+P^\top\Omega^{-1}Q\big], \qquad + \Sigma_\mu = \big[(\tau\Sigma)^{-1}+P^\top\Omega^{-1}P\big]^{-1}. + \label{eq:bl1} +\end{equation} +The implementation uses the algebraically identical form that needs no inverse of $\tau\Sigma$ +(a $K\times K$ system only), which is numerically preferable and also valid when $\Sigma$ is +singular: +\begin{equation} + \mu_{\mathrm{BL}} = \pi + \tau\Sigma P^\top\big(P\tau\Sigma P^\top+\Omega\big)^{-1}(Q-P\pi), \qquad + \Sigma_{\mathrm{BL}} = \Sigma + \tau\Sigma - \tau\Sigma P^\top\big(P\tau\Sigma P^\top+\Omega\big)^{-1}P\tau\Sigma . + \label{eq:bl2} +\end{equation} +The posterior mean is the prior corrected in the direction of the views by the amount by which +they disagree with the prior ($Q-P\pi$), weighted by how confident they are relative to the +prior; $\Sigma_{\mathrm{BL}}=\Sigma+\Sigma_\mu$ is the covariance of the returns themselves +(return noise plus the remaining uncertainty about the mean) and is reported for completeness. +Without views, $\mu_{\mathrm{BL}}=\pi$. The posterior mean is the drift handed to the scenario +models: per day, $\mu=\mu_{\mathrm{BL}}^{\mathrm{ann}}/252$ (total return, i.e.\ excess plus +$r_f$). + +\subsection{Idzorek's confidence: from a degree of belief to $\Omega$} + +The weak point of the model is $\Omega$: nobody can state the variance of a view. Idzorek +\cite{Idzorek2005} replaces it by a \emph{confidence} $C_k\in(0,1]$ with a portfolio meaning. +Consider one view $k$ alone. At 100\,\% confidence ($\omega_k=0$) the posterior is +$\mu_{100}=\pi+\tau\Sigma p_k^\top(p_k\tau\Sigma p_k^\top)^{-1}(q_k-p_k\pi)$ and the +unconstrained mean-variance weights move from $w$ to $w_{100}=(\delta\Sigma)^{-1}\mu_{100}$. +Idzorek defines the confidence as the fraction of that move the investor wants to make: +the target weights are $w+C_k\,(w_{100}-w)$, and $\omega_k$ is the value for which the +posterior weights $w(\omega_k)=(\delta\Sigma)^{-1}\mu_{\mathrm{BL}}(\omega_k)$ hit the target +(in the paper by a one-dimensional least-squares search). For a single view this calibration +has a closed form. From \eqref{eq:bl2} with one view, +\begin{equation} + w(\omega_k)-w \;=\; (\delta\Sigma)^{-1}\tau\Sigma p_k^\top\,\frac{q_k-p_k\pi}{p_k\tau\Sigma p_k^\top+\omega_k} + \;=\; \frac{\tau}{\delta}\,p_k^\top\,\frac{q_k-p_k\pi}{p_k\tau\Sigma p_k^\top+\omega_k}, +\end{equation} +so the tilt is always in the direction $p_k^\top$ and its size relative to the full tilt +($\omega_k=0$) is $p_k\tau\Sigma p_k^\top/(p_k\tau\Sigma p_k^\top+\omega_k)$. Setting this equal to +$C_k$ gives +\begin{equation} + \boxed{\;\omega_k \;=\; \frac{1-C_k}{C_k}\;p_k\,\tau\Sigma\,p_k^\top\;} + \label{eq:idzorek} +\end{equation} +\cite{Walters2014}: the view's variance is the prior variance of the view portfolio scaled by +the odds against the view; $C_k=1$ gives $\omega_k=0$ (the view is imposed), $C_k=0.5$ gives +$\omega_k=p_k\tau\Sigma p_k^\top$ (the He--Litterman choice, available as +\code{omega\_method="he\_litterman"}), $C_k\to0$ removes the view. The implementation uses +\eqref{eq:idzorek} for every view and then computes the joint posterior \eqref{eq:bl2}; the +numerical search of the paper is kept as \code{idzorek\_omega\_numeric} and a unit test shows +that both agree to $10^{-4}$. With several views, each $\omega_k$ is calibrated for its view +alone, as in the paper. + +\begin{example}[Four assets] + \label{ex:bl} + Annual volatilities $(20,25,15,30)\,\%$, correlations + $\rho_{12}=0.5,\rho_{13}=0.3,\rho_{14}=0.2,\rho_{23}=0.4,\rho_{24}=0.1,\rho_{34}=0.3$, market + weights $(0.4,0.3,0.2,0.1)$, market return 7\,\%, $r_f=1\,\%$, $\tau=0.05$, weekly + covariance $\Sigma=\Sigma^{\mathrm{ann}}/52$. Then $w^\top\Sigma^{\mathrm{ann}}w=0.0250$ + (market volatility 15.8\,\%), $\delta=0.06/0.0250=2.40$, and the equilibrium total returns + are $\pi+r_f=(7.36,\ 8.80,\ 4.35,\ 5.50)\,\%$ --- the market portfolio earns + $0.4\cdot7.36+0.3\cdot8.80+0.2\cdot4.35+0.1\cdot5.50=7.00\,\%$. An absolute view "asset 2 + returns 15\,\%" with confidence $C=0.4$ gives $\omega=\frac{0.6}{0.4}\,p\tau\Sigma p^\top=1.5\times + 6.01\cdot10^{-5}=9.01\cdot10^{-5}$ (weekly units; the numerical search returns the same value) + and the posterior total returns $(8.35,\ 11.28,\ 4.94,\ 5.80)\,\%$: asset 2 moves 40\,\% of the + way from 8.80\,\% to the 15\,\% it would reach at full confidence, and the correlated assets + follow. The unconstrained weights move from $0.3$ to $0.4656$ for asset 2 --- exactly + $0.3+0.4\,(0.7139-0.3)$ --- and stay unchanged for the others, as an absolute view on one + asset only tilts that asset. The numbers are pinned by \code{tests/test\_black\_litterman.py}. +\end{example} + +\subsection{Views from the AI layer: sentiment with a degree of belief} + +The platform's news pipeline delivers, per stock, an assessment "positive / negative" with a +degree of belief rather than a return forecast. \code{view\_from\_sentiment} turns +$(\text{stock } i,\ \text{sentiment } s\in[-1,1],\ \text{confidence } C)$ into the absolute view +\begin{equation} + q \;=\; \pi_i^{\mathrm{ann}} + r_f^{\mathrm{ann}} + s\cdot\gamma\cdot\sigma_i^{\mathrm{ann}}, \qquad \gamma=0.25 \text{ by default}, + \label{eq:sentiment} +\end{equation} +i.e.\ a fully positive assessment claims one quarter of the stock's annual volatility above its +equilibrium return (about $+5\,\%$ for a 20\,\%-volatility stock), and the degree of belief +becomes the Idzorek confidence $C$. The strength $\gamma$ is a modelling parameter; the view +carries the provenance of the insight that produced it (\code{source}). On the data set, a +sentiment of $+0.8$ with confidence $0.6$ on Nestl\'e states $q=6.1\,\%$ against an equilibrium +of $2.7\,\%$, calibrates $\omega=1.8\cdot10^{-5}$ and lifts the posterior to $4.7\,\%$ per +year. Explicit views (\code{assets}, \code{q}, \code{confidence}), including relative ones, are +accepted alongside. + +\subsection{Units and alternatives} + +Views state \emph{annual total} returns; internally they are converted to weekly excess returns +($q/52$ minus the weekly $r_f$ times the sum of the view weights, which vanishes for relative +views). Results are reported per week and annualised; the optimiser receives the posterior +total return per day, $\mu_{\mathrm{BL}}^{\mathrm{ann}}/252$, as the \emph{arithmetic} +expected return, converts it to the It\^o-corrected log-drift \eqref{eq:ito} of \eqref{eq:gbm} +and \eqref{eq:fhs}, and uses the expected simple return over the horizon, +$h\mu_{\mathrm{BL}}^{\mathrm{ann}}/252$, in its objective. Inputs that are log-return means +(the index average, the historical means) are converted with $+\tfrac12\sigma^2$. Two simpler +models are available for comparison: \code{expected\_returns="historical"} (the ten-year weekly +means, converted) and \code{"zero"} (no drift, pure risk minimisation). + +% ============================================================================================ +\section{Implementation map} +% ============================================================================================ + +\begin{table}[h] +\centering +\footnotesize +\begin{tabular}{@{}p{3.2cm}p{5.6cm}p{6.4cm}@{}} +\toprule +concept & code & parameters (defaults) / output \\ +\midrule +log-returns, alignment, frequencies, horizons & \code{measurement.returns}: \code{log\_returns}, \code{align\_prices}, \code{resample\_prices}, \code{horizon\_periods}, \code{horizon\_returns}, \code{portfolio\_log\_returns} & \code{freq} daily $|$ weekly $|$ monthly; \code{horizon} day $|$ week $|$ month $|$ integer \\ +CHF conversion & \code{data.fx.FXConverter}; \newline \code{MarketDataService.prices}, \code{.returns} & \code{currency="base"} (CHF) $|$ \code{"local"} \\ +rolling / EWM series & \code{estimators.rolling\_mean}, \code{rolling\_vol}, \code{ewm\_mean}, \code{ewm\_vol} & \code{window=200}, \code{lam=0.94}; annualised with the periods per year \\ +covariance matrices & \code{estimators.rolling\_cov}, \code{ewm\_cov}, \code{covariance}, \code{cov\_to\_corr}, \code{covariance\_sequence}; \newline \code{models.covariance.CovarianceModel} & \code{method} rolling $|$ ewm, \code{window}, \code{lam}, \code{shrinkage} $\in[0,1]$ or \code{"auto"} \\ +VaR / CVaR & \code{risk.normal\_var}, \code{normal\_cvar}, \code{historical\_var}, \code{historical\_cvar}, \code{scenario\_cvar}, \code{loss\_to\_money} & $\beta\in\{0.95,0.99\}$; losses positive; money via $V_0(1-e^{-\ell})$ \\ +per-asset risk table & \code{RiskService.asset\_risk} $\to$ \code{asset\_risk\_table} & columns \code{mean\_rolling\_ann}, \code{vol\_rolling\_ann}, \code{vol\_ewm\_ann}, \code{var95\_normal\_rolling}, \code{cvar95\_normal\_rolling}, \code{var95\_normal\_ewm}, \code{cvar95\_normal\_ewm}, \code{var95\_hist}, \code{cvar95\_hist} (and the same for 99), \code{horizon\_periods}, \code{n\_obs}, \code{as\_of} \\ +risk time series & \code{RiskService.risk\_series} & same names, one row per date \\ +portfolio risk & \code{RiskService.portfolio\_risk} $\to$ \code{PortfolioRisk} & \code{holdings} (shares), \code{cash}, \code{method} normal $|$ historical $|$ scenarios, \code{cov\_method}; per level \code{var}, \code{cvar}, \code{var\_money}, \code{cvar\_money} \\ +performance & \code{performance\_table}, \code{rolling\_performance}, \code{ewm\_performance}, \code{PerformanceService} & \code{risk\_free} from the settings; \code{rebalanced=True} = constant mix \\ +scenario models & \code{models.simulation.GBMSimulator}, \newline \code{EmpiricalInnovationSimulator} $\to$ \code{ScenarioSet} & \code{n\_scenarios=4096}, \code{seed=42}, \code{sampling} sobol $|$ mc $|$ antithetic $|$ lhs $|$ balanced, \code{moment\_match=True}, \code{tilt} (\code{TiltSpec}: direction, shifts, mixtures), \code{drift\_convention} arithmetic $|$ log, \code{root}, \code{standardize} diagonal $|$ full $|$ None, \code{clip=8}, \code{max\_history=2500}; output \code{probabilities}, \code{expected\_simple}, \code{hash} \\ +sampling schemes & \code{models.sampling}: \code{standard\_normal\_sample}, \code{moment\_match}, \code{pca\_root}, \code{tilt\_normal\_sample}, \code{tilted\_resample}, \code{resample\_indices}, \code{scenario\_hash} & Sobol' sizes are powers of two; likelihood ratios bounded by $1/(1-\sum_k\code{mixture}_k)$ \\ +one set per family of runs & \code{OptimizationService.scenario\_key}, \code{.market\_exposure}, \code{.clear\_scenario\_cache}; \code{ScenarioSet.with\_drift}, \code{.shocks}, \code{.shock\_hash} & the key excludes risk limit, confidence, constraints, costs, objective, cash, holdings and the expected-return model \\ +market index and weights & \code{MarketDataService}: \newline \code{cap\_weighted\_index}, \code{market\_caps}, \code{market\_weights} & shares outstanding $\times$ price $\times$ FX \\ +Black--Litterman & \code{BlackLittermanModel}, \code{View}, \code{view\_from\_sentiment}, \code{idzorek\_omega} $\to$ \code{BlackLittermanResult} & \code{tau=0.05}, \code{periods\_per\_year=52}, \code{omega\_method} idzorek $|$ he\_litterman; \code{prior\_ann}, \code{posterior\_ann}, \code{omega}, \code{delta}, \code{market\_return\_ann} \\ +pipeline in the optimiser & \code{OptimizationService}: \newline \code{expected\_returns}, \code{scenarios} & \code{bl\_years=10}, \code{expected\_returns} black\_litterman $|$ historical $|$ zero, \code{views}, \code{covariance} = \{ewm, 0.99, auto\}, \code{scenario\_model} empirical $|$ gbm $|$ gaussian\_exact, \code{importance\_sampling} auto $|$ market $|$ adaptive $|$ off, \code{row\_generation} auto $|$ on $|$ off \\ +HTTP & \code{/risk/*}, \code{/performance/*}, \code{/model/expected-returns}, \code{/model/scenarios}, \code{/market/*} & see \code{/docs} \\ +\bottomrule +\end{tabular} +\caption{Where each concept lives in \code{quantcore}.} +\label{tab:impl} +\end{table} + +\paragraph{Determinism and provenance.} Every figure is a deterministic function of the stored +data and the stated parameters; simulations take an explicit seed. Every result carries the +parameters that produced it (window, $\lambda$, shrinkage applied, model, seed, horizon, date, +views with their calibrated $\omega$), so that the advisor layer can cite them and a reviewer +can recompute them. + +% ============================================================================================ +\begin{thebibliography}{99} +\bibitem{RiskMetrics1996} + J.P.~Morgan/Reuters, \emph{RiskMetrics --- Technical Document}, 4th edition, New York, 1996. +\bibitem{LedoitWolf2004} + O.~Ledoit and M.~Wolf, \emph{A well-conditioned estimator for large-dimensional covariance + matrices}, Journal of Multivariate Analysis 88(2), 365--411, 2004. +\bibitem{SchaeferStrimmer2005} + J.~Sch\"afer and K.~Strimmer, \emph{A shrinkage approach to large-scale covariance matrix + estimation and implications for functional genomics}, Statistical Applications in Genetics and + Molecular Biology 4(1), Article 32, 2005. +\bibitem{AcerbiTasche2002} + C.~Acerbi and D.~Tasche, \emph{On the coherence of expected shortfall}, + Journal of Banking \& Finance 26(7), 1487--1503, 2002. +\bibitem{MFE2015} + A.\,J.~McNeil, R.~Frey and P.~Embrechts, \emph{Quantitative Risk Management: Concepts, + Techniques and Tools}, revised edition, Princeton University Press, 2015. +\bibitem{BaroneAdesi1999} + G.~Barone-Adesi, K.~Giannopoulos and L.~Vosper, \emph{VaR without correlations for portfolios + of derivative securities}, Journal of Futures Markets 19(5), 583--602, 1999. +\bibitem{BlackLitterman1992} + F.~Black and R.~Litterman, \emph{Global portfolio optimization}, + Financial Analysts Journal 48(5), 28--43, 1992. +\bibitem{HeLitterman1999} + G.~He and R.~Litterman, \emph{The intuition behind Black--Litterman model portfolios}, + Goldman Sachs Investment Management Research, 1999. +\bibitem{Idzorek2005} + T.\,M.~Idzorek, \emph{A step-by-step guide to the Black--Litterman model: incorporating + user-specified confidence levels}, working paper, Ibbotson Associates, 2005 (also in + S.~Satchell (ed.), \emph{Forecasting Expected Returns in the Financial Markets}, 2007). +\bibitem{Walters2014} + J.~Walters, \emph{The Black--Litterman model in detail}, working paper (SSRN), 2014. +\end{thebibliography} + +\end{document} diff --git a/docs/optimization_model.pdf b/docs/optimization_model.pdf new file mode 100644 index 0000000..d553bb6 Binary files /dev/null and b/docs/optimization_model.pdf differ diff --git a/docs/optimization_model.tex b/docs/optimization_model.tex new file mode 100644 index 0000000..2f2de47 --- /dev/null +++ b/docs/optimization_model.tex @@ -0,0 +1,1197 @@ +% !TEX encoding = UTF-8 Unicode +% --------------------------------------------------------------------------------------------- +% quantcore -- documentation of the portfolio optimisation model +% From the Rockafellar-Uryasev mean-CVaR LP in portfolio weights to the mixed-integer +% programme in whole shares with rebalancing and transaction costs, as implemented in +% quantcore/optimization/problem.py. +% Build: latexmk -pdf docs/optimization_model.tex +% --------------------------------------------------------------------------------------------- +\documentclass[11pt,a4paper]{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage[british]{babel} +\usepackage{lmodern} +\usepackage{microtype} +\usepackage[margin=2.4cm]{geometry} +\usepackage{amsmath,amssymb,amsthm,mathtools} +\usepackage{booktabs,array,longtable} +\usepackage{enumitem} +\usepackage{xcolor} +\usepackage[colorlinks=true,linkcolor=blue!50!black,citecolor=blue!50!black,urlcolor=blue!50!black]{hyperref} + +\newtheorem{theorem}{Theorem}[section] +\newtheorem{proposition}[theorem]{Proposition} +\newtheorem{lemma}[theorem]{Lemma} +\theoremstyle{definition} +\newtheorem{definition}[theorem]{Definition} +\newtheorem{example}[theorem]{Example} +\theoremstyle{remark} +\newtheorem{remark}[theorem]{Remark} + +\DeclareMathOperator{\CVaR}{CVaR} +\DeclareMathOperator{\VaR}{VaR} +\DeclareMathOperator{\E}{\mathbb{E}} +\DeclareMathOperator*{\argmin}{arg\,min} +\newcommand{\R}{\mathbb{R}} +\newcommand{\Z}{\mathbb{Z}} +\newcommand{\pos}[1]{\left[#1\right]^{+}} +\newcommand{\code}[1]{\texttt{#1}} +\newcommand{\TC}{\mathit{TC}} +\setlength{\parskip}{4pt} +\setlength{\parindent}{0pt} + +\title{Mean--CVaR Portfolio Optimisation in Whole Shares\\[1ex] +\large From the Rockafellar--Uryasev linear programme in portfolio weights to the +mixed-integer programme with rebalancing and transaction costs implemented in \code{quantcore}} +\author{AISE502 -- AI-Augmented Portfolio Intelligence Platform\\ \small Deterministic quant core, module \code{quantcore.optimization}} +\date{September 2026} + +\begin{document} +\maketitle + +\begin{abstract} +\noindent +The portfolio optimisation service of the platform selects a portfolio of stocks such that the +expected profit over a horizon is maximal while the Conditional Value-at-Risk (expected +shortfall) of the loss over that horizon stays below an absolute risk budget in CHF. Positions +are whole numbers of shares, sector and single-stock limits apply, short positions are not +allowed, and an existing portfolio can be rebalanced at proportional and fixed transaction +costs. This note states the classical scenario-based mean--CVaR linear programme of Rockafellar +and Uryasev in portfolio weights, and derives from it, step by step, the mixed-integer linear +programme (MILP) that \code{quantcore/optimization/problem.py} builds and that is solved on +NVIDIA cuOpt (with HiGHS as fallback). Every constraint of the implementation is traced back to +its origin, a worked example shows why rounding a continuous solution is not an option, a +section on scenario sets quantifies the sampling error of the approximation and derives the +variance-reduction, importance-sampling and row-generation techniques in the code together with +the exact (mean--variance-equivalent) solution of the Gaussian case, and the last section lists +the complete model with the variable and row names used in the code. +\end{abstract} + +\tableofcontents + +% ============================================================================================ +\section{Setting and notation} +% ============================================================================================ + +There are $N$ candidate stocks $i=1,\dots,N$ with current prices $P_i>0$ in the base currency +(CHF). The investor has cash $c^0\ge 0$ and, in the rebalancing case, existing holdings +$n^0_i\in\Z_{\ge 0}$ shares; the total wealth is +\begin{equation} + W \;=\; c^0 + \sum_{i=1}^N P_i\, n^0_i . + \label{eq:wealth} +\end{equation} +The uncertainty is described by $S$ equiprobable scenarios of simple returns over the horizon, +\begin{equation} + R_{s,i} \;=\; e^{\,r_{s,i}} - 1 , \qquad s = 1,\dots,S, + \label{eq:scenarios} +\end{equation} +where the log-returns $r_{s,i}$ (in CHF) are simulated by one of the stock-price models of the +platform (geometric Brownian motion or empirical innovations) with the expected returns of the +Black--Litterman model as drift; see \code{docs/METHODOLOGY.md}, Sections~6--7. Each scenario has +probability $1/S$. The confidence level of the risk measure is $\beta\in(0,1)$ (0.95 or 0.99). + +Throughout, \emph{losses are positive numbers}: a loss of $L$ means the portfolio is worth $L$ +less than at the start of the horizon. Vectors are columns, $\pos{z}=\max(z,0)$, and +$\mathbf{1}$ is the vector of ones. + +% ============================================================================================ +\section{Conditional Value-at-Risk and its linear representation} +% ============================================================================================ + +Let $L(x,Y)$ be the loss of a decision $x\in X\subseteq\R^N$ under the random outcome $Y$, with +distribution function $\Psi(x,\zeta)=\mathbb{P}\{L(x,Y)\le\zeta\}$. + +\begin{definition}[VaR and CVaR] + The \emph{Value-at-Risk} at level $\beta$ is the $\beta$-quantile of the loss, + $\VaR_\beta(x)=\min\{\zeta\in\R:\Psi(x,\zeta)\ge\beta\}$. The \emph{Conditional Value-at-Risk} + (expected shortfall) is the mean of the $\beta$-tail of the loss distribution, + \begin{equation} + \CVaR_\beta(x) \;=\; \frac{1}{1-\beta}\int_{\beta}^{1} \VaR_{\gamma}(x)\,d\gamma , + \label{eq:cvar-def} + \end{equation} + which for a continuous loss distribution equals $\E\big[L(x,Y)\,\big|\,L(x,Y)\ge\VaR_\beta(x)\big]$. + For distributions with atoms (in particular for a finite set of scenarios) the definition + \eqref{eq:cvar-def} splits the atom at the VaR appropriately \cite{RockafellarUryasev2002, + AcerbiTasche2002}. +\end{definition} + +CVaR is a coherent risk measure (monotone, sub-additive, positively homogeneous, translation +invariant), which VaR is not. Its value for optimisation purposes rests on the following +representation. + +\begin{theorem}[Rockafellar--Uryasev \cite{RockafellarUryasev2000,RockafellarUryasev2002}] + \label{thm:ru} + Define the auxiliary function + \begin{equation} + F_\beta(x,\zeta) \;=\; \zeta + \frac{1}{1-\beta}\,\E\Big[\pos{L(x,Y)-\zeta}\Big]. + \label{eq:F} + \end{equation} + Then $F_\beta(x,\cdot)$ is finite and convex, and + \begin{equation} + \CVaR_\beta(x) \;=\; \min_{\zeta\in\R} F_\beta(x,\zeta), + \qquad + \VaR_\beta(x) \;=\; \text{left endpoint of } \argmin_{\zeta} F_\beta(x,\zeta). + \label{eq:ru} + \end{equation} + If $L(x,Y)$ is convex in $x$, then $F_\beta$ is jointly convex in $(x,\zeta)$ and + \begin{equation} + \min_{x\in X}\CVaR_\beta(x) \;=\; \min_{(x,\zeta)\in X\times\R} F_\beta(x,\zeta), + \label{eq:ru-joint} + \end{equation} + with $(x^*,\zeta^*)$ minimising the right-hand side if and only if $x^*$ minimises the left-hand + side and $\zeta^*\in\argmin_\zeta F_\beta(x^*,\zeta)$. +\end{theorem} + +The theorem removes the need to know the VaR before computing the CVaR: both come out of one +minimisation over the scalar $\zeta$, and $\zeta$ can simply be added to the decision +variables. A second consequence is used for risk \emph{constraints} +\cite{KrokhmalPalmquistUryasev2002}: for any budget $\omega$, +\begin{equation} + \CVaR_\beta(x)\le\omega + \quad\Longleftrightarrow\quad + \exists\,\zeta:\; F_\beta(x,\zeta)\le\omega , + \label{eq:cvar-constraint} +\end{equation} +because the minimum over $\zeta$ is at most $\omega$ exactly when some $\zeta$ attains at most +$\omega$. + +\paragraph{Scenario approximation and linearisation.} +With $S$ equiprobable scenarios the expectation in \eqref{eq:F} becomes an average, +\begin{equation} + \tilde F_\beta(x,\zeta) \;=\; \zeta + \frac{1}{(1-\beta)S}\sum_{s=1}^{S} \pos{L_s(x)-\zeta}, + \qquad L_s(x)=L(x,Y_s). + \label{eq:F-scen} +\end{equation} +The positive parts are linearised with auxiliary variables $u_s\ge 0$: since $u_s\ge L_s(x)-\zeta$ +and $u_s\ge 0$ imply $u_s\ge\pos{L_s(x)-\zeta}$, and the objective/constraint pushes $u_s$ down, +\begin{equation} + \min_\zeta \tilde F_\beta(x,\zeta) + \;=\; + \min_{\zeta,\,u}\Big\{\zeta + \frac{1}{(1-\beta)S}\sum_{s} u_s \;:\; + u_s \ge L_s(x)-\zeta,\; u_s\ge 0,\; s=1,\dots,S\Big\}. + \label{eq:lin} +\end{equation} +Whenever $L_s(x)$ is linear in $x$, \eqref{eq:lin} is a linear programme in $(x,\zeta,u)$. + +\begin{remark}[The discrete tail and the meaning of $\zeta$] + \label{rem:discrete} + For $S$ scenarios and $k=(1-\beta)S$, the minimum of \eqref{eq:F-scen} equals the average of + the $k$ largest losses, where a fractional $k$ gives the $(\lfloor k\rfloor+1)$-th largest loss + the fractional weight $k-\lfloor k\rfloor$. This is exactly what + \code{quantcore.measurement.risk.scenario\_cvar} computes, and it is the value the LP returns. + When $k$ is an integer, $\tilde F_\beta(x,\cdot)$ is flat between the $(k{+}1)$-th and the $k$-th + largest loss, so the minimiser $\zeta$ is \emph{not unique}: any point of that interval is + optimal, and the VaR is its left endpoint. Consequently the $\zeta$ (called $\alpha$ in the + code) reported by a solver need not equal the VaR, in particular when the CVaR constraint is + not tight; the implementation therefore recomputes VaR and CVaR of the decoded portfolio from + the scenarios rather than reading them off the LP variables. +\end{remark} + +% ============================================================================================ +\section{The classical model in portfolio weights} +% ============================================================================================ + +The textbook formulation \cite{RockafellarUryasev2000,KrokhmalPalmquistUryasev2002} describes a +portfolio by weights $w\in\R^N$, the fractions of wealth invested in each stock. Its loss over +the horizon, relative to the initial wealth, is minus the portfolio return, +\begin{equation} + L_s(w) \;=\; -\sum_{i=1}^N R_{s,i}\,w_i \;=\; -R_s^{\top} w , + \label{eq:loss-w} +\end{equation} +which is linear in $w$, and the expected return is $\bar R^{\top}w$ with +$\bar R_i=\frac1S\sum_s R_{s,i}$. With a risk budget $\omega$ expressed as a fraction of wealth +(e.g.\ ``the 95\,\% CVaR over one week may not exceed 4\,\% of the portfolio value''), the +\emph{mean--CVaR linear programme} reads +\begin{subequations} +\label{eq:lp-w} +\begin{alignat}{3} + \max_{w,\zeta,u}\quad & \bar R^{\top} w \label{eq:lp-w-obj}\\ + \text{s.t.}\quad + & u_s \;\ge\; -R_s^{\top} w - \zeta, \quad u_s\ge 0, & \quad & s=1,\dots,S, \label{eq:lp-w-u}\\ + & \zeta + \frac{1}{(1-\beta)S}\sum_{s=1}^S u_s \;\le\; \omega, \label{eq:lp-w-cvar}\\ + & \mathbf{1}^{\top} w \;\le\; 1, \qquad w \;\ge\; 0, \label{eq:lp-w-budget}\\ + & w_i \;\le\; \bar w, & & i=1,\dots,N, \label{eq:lp-w-single}\\ + & \sum_{i\in G_k} w_i \;\le\; \bar w_k, & & k=1,\dots,K. \label{eq:lp-w-sector} +\end{alignat} +\end{subequations} +Here \eqref{eq:lp-w-u}--\eqref{eq:lp-w-cvar} are \eqref{eq:lin} and \eqref{eq:cvar-constraint} +combined, \eqref{eq:lp-w-budget} is the budget with a non-negative cash residual $1-\mathbf{1}^\top w$ +(earning nothing over the horizon) and the no-short-sale condition, \eqref{eq:lp-w-single} caps +each stock at the fraction $\bar w$ (10\,\% in the platform) and \eqref{eq:lp-w-sector} caps every +GICS sector $G_k$ at $\bar w_k$. + +\paragraph{Three equivalent objectives.} +Krokhmal, Palmquist and Uryasev \cite{KrokhmalPalmquistUryasev2002} show that the three +problems +\begin{equation} + \text{(a)}\;\max\,\bar R^{\top}w \text{ s.t. } \CVaR_\beta\le\omega, \quad + \text{(b)}\;\min\,\CVaR_\beta \text{ s.t. } \bar R^{\top}w\ge\rho, \quad + \text{(c)}\;\max\,\bar R^{\top}w-\lambda\CVaR_\beta + \label{eq:three} +\end{equation} +trace out the same efficient frontier as $\omega$, $\rho$ and $\lambda\ge 0$ vary. All three are +linear after the substitution \eqref{eq:lin}; the implementation offers them as the objectives +\code{max\_return}, \code{min\_cvar} and \code{utility}. The platform's primary mode is (a), +because the risk-management specification is an \emph{absolute} loss level for a period. + +\paragraph{Properties.} +Problem \eqref{eq:lp-w} has $N+1+S$ variables and $S+1+N+K+1$ constraints (the weight caps are +variable bounds). Its constraint matrix is dense in the $S$ scenario rows: $S(N+2)$ non-zeros. It +is a linear programme, so every local optimum is global, the solution is a vertex, and it can be +solved for thousands of scenarios and hundreds of assets in seconds. + +% ============================================================================================ +\section{From weights to numbers of shares} +\label{sec:shares} +% ============================================================================================ + +Weights are a convenient abstraction, but a real portfolio consists of whole shares: one cannot +buy 2.66 shares of a stock, and for a stock priced at CHF~2\,000 the difference between 2 and 3 +shares is 10\,\% of a CHF~10\,000 portfolio. The platform therefore optimises directly over the +integer numbers of shares. + +\subsection{Money instead of fractions} + +Let $n_i\in\Z_{\ge0}$ be the number of shares of stock $i$ held over the horizon. The value of +the position is $P_i n_i$, and the weight of the classical model is recovered as +\begin{equation} + w_i \;=\; \frac{P_i\,n_i}{W}. + \label{eq:w-n} +\end{equation} +Substituting \eqref{eq:w-n} into the relative loss \eqref{eq:loss-w} and multiplying by $W$ +turns it into a loss \emph{in money}: +\begin{equation} + W L_s(w) \;=\; -\sum_i R_{s,i}\,P_i\,n_i \;=\; -\sum_i g_{s,i}\,n_i, \qquad g_{s,i} := P_i R_{s,i}, + \label{eq:loss-n} +\end{equation} +where $g_{s,i}$ is the CHF gain of one share of stock $i$ in scenario $s$. Because CVaR is +positively homogeneous, $\CVaR_\beta(W L)=W\CVaR_\beta(L)$: the CVaR of the money loss is the +money amount the risk manager specifies (``the 95\,\% CVaR over a week may not exceed +CHF~40\,000''), and the relative budget $\omega$ of \eqref{eq:lp-w-cvar} becomes the absolute +budget $\Lambda=\omega W$. The expected return $\bar R^\top w$ becomes the expected profit +\begin{equation} + \E[\text{P\&L}] \;=\; \sum_i \bar g_i\,n_i, \qquad \bar g_i=\frac1S\sum_s g_{s,i}=P_i\bar R_i . + \label{eq:epnl} +\end{equation} +Both the objective and the risk constraint use the \emph{same} scenario set, so the trade-off +they describe is internally consistent (the drift of the scenarios is the expected-return model). + +\subsection{Budget, cash and the constraints} + +Multiplying each constraint of \eqref{eq:lp-w} by $W$ and substituting \eqref{eq:w-n} gives: +\begin{itemize}[nosep] + \item budget: $\sum_i P_i n_i \le W$ --- the purchases may not exceed the available cash, what + is not invested stays in cash; + \item single stock: $P_i n_i\le \bar w W$; sector: $\sum_{i\in G_k}P_i n_i\le \bar w_k W$; + \item no short sales: $n_i\ge 0$. +\end{itemize} +The single-stock cap deserves a closer look. As a linear constraint it is a row of the model, but +for an integer variable it is equivalent to the \emph{variable bound} +\begin{equation} + n_i \;\le\; \bar n_i := \Big\lfloor \frac{\bar w\,W}{P_i} \Big\rfloor , + \label{eq:nbar} +\end{equation} +which is strictly tighter than the linear row whenever $\bar w W/P_i$ is not an integer: the +rounding down is free information that a MILP solver exploits (tighter LP relaxation, smaller +branch-and-bound tree). The implementation uses \eqref{eq:nbar} with $\bar w$ replaced by +$\min(\bar w, f_{\max})$ if a maximum invested fraction $f_{\max}<1$ is given, and never leaves +the cap to a row. + +\subsection{The mean--CVaR MILP in shares} + +Putting the pieces together, the new-portfolio problem (all wealth in cash, $n^0=0$, $W=c^0$) +reads +\begin{subequations} +\label{eq:milp-n} +\begin{alignat}{3} + \max_{n,\zeta,u}\quad & \sum_{i} \bar g_i\, n_i \label{eq:milp-n-obj}\\ + \text{s.t.}\quad + & \sum_i g_{s,i}\, n_i + \zeta + u_s \;\ge\; 0, \quad u_s\ge 0, & \quad & s=1,\dots,S, \label{eq:milp-n-u}\\ + & \zeta + \frac{1}{(1-\beta)S}\sum_{s} u_s \;\le\; \Lambda, \label{eq:milp-n-cvar}\\ + & \sum_i P_i\, n_i \;\le\; c^0, \label{eq:milp-n-budget}\\ + & \sum_{i\in G_k} P_i\, n_i \;\le\; \bar w_k\, W, & & k=1,\dots,K, \label{eq:milp-n-sector}\\ + & n_i \in \{0,1,\dots,\bar n_i\}, & & i=1,\dots,N, \label{eq:milp-n-int}\\ + & \zeta\in\R . \nonumber +\end{alignat} +\end{subequations} +Row \eqref{eq:milp-n-u} is \eqref{eq:lp-w-u} written as ``$\ge 0$'' with the money loss +\eqref{eq:loss-n} inserted ($u_s\ge L_s-\zeta \Leftrightarrow \sum_i g_{s,i}n_i+\zeta+u_s\ge0$). +Apart from the integrality \eqref{eq:milp-n-int} the problem is the LP \eqref{eq:lp-w} scaled by +$W$; with the integrality it is a mixed-integer linear programme, and the LP obtained by dropping +\eqref{eq:milp-n-int} is its \emph{LP relaxation}. + +\subsection{Why the integrality cannot be handled by rounding} + +The optimum of the LP relaxation is a vertex, and its coordinates $n_i$ are generically +fractional. Rounding them is not a substitute for solving the MILP: +\begin{itemize}[nosep] + \item rounding \emph{up} (or to the nearest integer) can violate the budget, a cap, or the + CVaR constraint --- the risk budget is exceeded; + \item rounding \emph{down} is always feasible (all constraints are ``$\le$'' in $n$, and the + scenario rows are satisfied by re-optimising $\zeta,u$) but can be far from optimal, + because the ``lost'' fractions can often be re-invested in other, cheaper shares; + \item the true integer optimum can have a different \emph{structure} than the relaxed one, not + just rounded coordinates. +\end{itemize} +The gap is largest when $W/P_i$ is small, i.e.\ for expensive stocks or small portfolios; for a +CHF~1\,000\,000 portfolio and stocks below CHF~1\,000 it is negligible, but the 10\,\% cap keeps +each position to at most $\bar w W/P_i$ shares, and a few stocks in the universe trade above +CHF~10\,000 (Lindt \& Spr\"ungli partizipation certificates above CHF~100\,000). + +\begin{example}[Two stocks, eight scenarios] + \label{ex:rounding} + Prices $P=(572,\ 2208)$, cash $W=c^0=10\,000$, $\beta=0.75$ (so $k=(1-\beta)S=2$: the CVaR is + the mean of the two worst scenarios), risk budget $\Lambda=250$, caps $\bar w=0.7$, no costs. + The simple returns of the eight scenarios are + \[ + R=\begin{pmatrix} + 0.005&-0.082\\ 0.014&-0.052\\ -0.069&-0.020\\ -0.002&0.055\\ + -0.009&0.083\\ 0.048&0.064\\ 0.044&0.028\\ -0.011&0.042 + \end{pmatrix}, \qquad \bar R=(0.0025,\ 0.01475). + \] + Stock 2 has six times the expected return of stock 1 but a heavier left tail. + \begin{center} + \small + \begin{tabular}{@{}lcccccl@{}} + \toprule + portfolio & $n_1$ & $n_2$ & invested & $\E[\text{P\&L}]$ & $\CVaR_{0.75}$ & feasible \\ + \midrule + LP relaxation ($n\in\R^2_{\ge0}$) & 2.6605 & 1.7876 & 5\,468.90 & 62.02 & 250.00 & yes, CVaR binding \\ + nearest rounding & 3 & 2 & 6\,132 & 69.43 & 280.13 & \textbf{no}: $\CVaR>\Lambda$ \\ + rounding down & 2 & 1 & 3\,352 & 35.43 & 149.22 & yes, $-43\,\%$ vs.\ LP \\ + MILP optimum & 7 & 1 & 6\,212 & 42.58 & 240.74 & yes \\ + \bottomrule + \end{tabular} + \end{center} + Rounding to the nearest integers breaks the risk budget (the CVaR limit, not the cash: 6\,132 + of 10\,000 are invested). The relaxed solution buys 1.79 shares of the attractive stock~2; the + second whole share of it is too large a block of risk for the budget, so the integer optimum keeps one share of stock~2 + and fills the remaining risk budget with \emph{seven} shares of stock~1 --- a portfolio that + no rounding of $(2.66, 1.79)$ produces. The enumeration of all feasible integer portfolios + confirms $(7,1)$ as the optimum; the integrality gap $62.02-42.58$ is 31\,\% of the relaxed + value. (The numbers were computed with \code{MeanCVaRProblem} and HiGHS and are pinned by + \code{tests/test\_problem.py::test\_documentation\_example\_rounding\_fails}.) +\end{example} + +% ============================================================================================ +\section{Rebalancing an existing portfolio with transaction costs} +% ============================================================================================ + +When holdings $n^0\ne 0$ exist, the decision is no longer ``which shares to buy'' but ``which +shares to buy and which to sell'', and every trade costs money. Weights cannot express this +naturally (a weight change of $+2\,\%$ says nothing about the number of shares traded, and the +cost depends on the traded value), whereas in share numbers it is immediate. + +\subsection{Trades, linking and the cash balance} + +Introduce the integer trade variables $b_i\ge0$ (shares bought) and $s_i\ge0$ (shares sold), +linked to the final position by +\begin{equation} + n_i \;=\; n^0_i + b_i - s_i, \qquad s_i\le n^0_i \;(\text{implied by } n_i\ge0), + \label{eq:link} +\end{equation} +and the transaction cost with proportional rates $c^b, c^s$ (10 basis points by default) and an +optional fixed cost $F$ per traded stock, +\begin{equation} + \TC(b,s,t) \;=\; \sum_i \big(c^b P_i\, b_i + c^s P_i\, s_i\big) + F\sum_i t_i, + \qquad b_i + s_i \le M_i\, t_i,\; t_i\in\{0,1\}, + \label{eq:tc} +\end{equation} +where $t_i$ indicates whether stock $i$ is traded at all and $M_i=\max(\bar n_i, n^0_i)+1$ is +a valid big-M (no trade can exceed it). The fixed cost makes small trades unattractive, which +is what it should do; it is the only place where binary variables enter the cost model. + +The budget \eqref{eq:milp-n-budget} becomes a \emph{cash balance}: sales bring cash in, +purchases and costs take it out, and the cash at the end of the rebalancing may not be negative, +\begin{equation} + c^{\text{end}} \;=\; c^0 + \sum_i P_i s_i - \sum_i P_i b_i - \TC \;\ge\; 0 + \quad\Longleftrightarrow\quad + \sum_i P_i b_i - \sum_i P_i s_i + \TC \;\le\; c^0 . + \label{eq:cash} +\end{equation} + +\begin{remark}[Buying and selling the same stock] + With positive proportional costs an optimal solution never has $b_i>0$ and $s_i>0$ + simultaneously: reducing both by one share leaves $n_i$ unchanged and saves + $(c^b+c^s)P_i$ in cost and cash. With zero costs the decomposition of a net trade into $b_i$ + and $s_i$ is not unique but harmless. The decoded solution reports the net trade $b_i-s_i$. +\end{remark} + +\subsection{Transaction costs enter the loss} + +The costs are paid at the start of the horizon and are therefore a deterministic part of the +loss relative to the initial wealth $W$ of \eqref{eq:wealth}: +\begin{equation} + L_s(n,b,s,t) \;=\; \TC(b,s,t) - \sum_i g_{s,i}\, n_i . + \label{eq:loss-reb} +\end{equation} +By translation invariance, $\CVaR_\beta(\TC-G)=\TC+\CVaR_\beta(-G)$: the costs shift the +whole loss distribution and consume part of the risk budget, exactly as they reduce the expected +profit, +\begin{equation} + \E[\text{P\&L}] \;=\; \sum_i \bar g_i\, n_i - \TC(b,s,t). + \label{eq:epnl-reb} +\end{equation} +The scenario rows \eqref{eq:milp-n-u} therefore become +\begin{equation} + \sum_i g_{s,i}\, n_i - \TC(b,s,t) + \zeta + u_s \;\ge\; 0, \qquad s=1,\dots,S . + \label{eq:u-reb} +\end{equation} + +\subsection{Limits on the rebalancing itself} + +Two constraints of the specification concern the trading rather than the resulting portfolio: +a cap on the total transaction cost and a cap on the turnover (the traded value as a fraction of +wealth), +\begin{equation} + \TC(b,s,t)\;\le\;\TC_{\max}, \qquad \sum_i P_i\,(b_i+s_i)\;\le\;\tau\,W . + \label{eq:turnover} +\end{equation} +Both are optional. The single-stock and sector caps apply to the \emph{final} positions +$n=n^0+b-s$: a legacy position above the cap has to be sold down to it, and if a turnover cap +prevents that, the problem is (correctly) infeasible. + +\subsection{The new portfolio as a special case} + +With $n^0=0$ nothing can be sold, so $s=0$ and $b=n$; the linking rows \eqref{eq:link} become +trivial and are dropped, the costs become $\TC=c^b\sum_i P_i n_i+F\sum_i t_i$, and the cash +balance \eqref{eq:cash} reduces to $\sum_i P_i(1+c^b)\,n_i+F\sum_i t_i\le c^0$. This is +\eqref{eq:milp-n} with proportional costs, i.e.\ mode \code{new} of the implementation. + +\subsection{Optional cardinality and minimum position size} + +A portfolio of 30 stocks is easier to monitor than one of 120 with many tiny positions; the +integer model can express both a maximum number of positions and a minimum value per open +position with one binary $y_i$ per stock: +\begin{equation} + n_i \;\le\; \max(\bar n_i,1)\, y_i, \qquad P_i n_i \;\ge\; v_{\min}\, y_i, \qquad \sum_i y_i\le K_{\max}, + \qquad y_i\in\{0,1\}. + \label{eq:card} +\end{equation} +These constraints are optional (\code{max\_positions}, \code{min\_position\_value}); they make the +problem combinatorially harder and are usually not needed --- the CVaR constraint and the +integrality already produce concentrated portfolios. + +% ============================================================================================ +\section{Scenario sets: sampling error, variance reduction and large sets} +\label{sec:sampling} +% ============================================================================================ + +The MILP sees the return distribution only through the scenario matrix $g_{s,i}=P_iR_{s,i}$ +and the probabilities of its rows. It is a \emph{sample-average approximation} (SAA) of the +problem one would like to solve --- the one with the true distribution of the horizon returns +\cite{ShapiroDentchevaRuszczynski2009} --- and its solution inherits the sampling error of the +scenario set. This section quantifies that error on the real universe, derives the three +remedies implemented in \code{quantcore.models.sampling} and \code{quantcore.optimization}, +and closes with the Gaussian special case, in which the exact solution is available and the +mean--CVaR problem coincides with the mean--variance problem. + +\subsection{How noisy is a plain Monte-Carlo set?} +\label{sec:mc-noise} + +Let the scenarios be $S$ independent draws from the model with mean $\mu$ (per asset, over the +horizon) and covariance $\Sigma$. The sample mean of asset $i$ has standard error +$\sigma_i/\sqrt S$. With $S=2\,000$ and weekly volatilities of 3--11\,\% this is +0.07--0.25\,\% per week, i.e.\ 3--13\,\% per year --- the same order as the expected returns +themselves (5--12\,\% p.a.\ from the Black--Litterman model). The optimiser cannot tell signal +from noise: it overweights the assets whose sample mean happens to be high, the classical +``error maximisation'' of \cite{Michaud1989}. The tail is only the second problem: the CVaR at +95\,\% is estimated from the $(1-\beta)S=100$ worst scenarios, with a relative standard error of +about $0.37/\sqrt{(1-\beta)S}\approx 2\,\%$ for a Gaussian tail. + +Table~\ref{tab:noise} measures the consequences on the 30 largest stocks of the universe +(CHF 1 million, weekly horizon, $\CVaR_{0.95}\le 5\,\%$ of wealth, eight seeds per method). The +reference is the exact optimum of Section~\ref{sec:gaussian}, so ``true'' figures are exact. +The comparison is made in portfolio weights (the continuous relaxation) so that integrality +neither hides nor causes noise; the integer results are the same to within a percent. + +\begin{table}[ht] +\centering +\footnotesize +\setlength{\tabcolsep}{4pt} +\begin{tabular}{@{}lrrrrr@{}} +\toprule +$S=2\,048$, $\beta=0.95$ & plain MC & antithetic & MC + mm & Sobol' & Sobol' + mm \\ +\midrule +in-sample $\E[\text{P\&L}]$, mean $\pm$ sd & $2\,791\pm552$ & $2\,228\pm19$ & $2\,221\pm10$ & $2\,221\pm8$ & $2\,220\pm5$ \\ +true $\E[\text{P\&L}]$ of the chosen portfolio & $1\,991\pm119$ & $2\,228\pm19$ & $2\,221\pm10$ & $2\,220\pm5$ & $2\,220\pm5$ \\ +loss versus the optimum ($2\,219$) & $-228$ & $-9$ & $-2$ & $-0.6$ & $-0.4$ \\ +true CVaR over the limit, mean $\pm$ sd & $-0.5\pm5.6\,\%$ & $+1.8\pm2.8\,\%$ & $+0.6\pm1.7\,\%$ & $+0.2\pm0.7\,\%$ & $+0.15\pm0.7\,\%$ \\ +turnover between two seeds & $101\,\%$ & $12.6\,\%$ & $11.7\,\%$ & $7.2\,\%$ & $5.8\,\%$ \\ +turnover to the optimum & $75\,\%$ & $8.9\,\%$ & $8.3\,\%$ & $5.0\,\%$ & $4.1\,\%$ \\ +\bottomrule +\end{tabular} +\caption{Sampling noise of the mean--CVaR optimisation (30 stocks, eight seeds, CHF per week; +``turnover'' is $\sum_i|w_i-w_i'|$). ``mm'' = moment matching.} +\label{tab:noise} +\end{table} + +With plain Monte Carlo two seeds give essentially disjoint portfolios (turnover 101\,\%), the +in-sample expected profit is 26\,\% too high and the true expected profit 10\,\% below the +optimum. Fixing the first two moments of the set removes the illusion entirely; Sobol' points +halve the remaining tail noise. The residual 4--6\,\% turnover is the flatness of the optimum +(many portfolios within 0.02\,\% of the optimal objective), not sampling. Increasing $S$ does +not rescue plain Monte Carlo: at $S=8\,192$ its turnover between seeds is still 76\,\% +(the mean error decays like $1/\sqrt S$), while Sobol' + mm is at 4.0\,\%. + +\subsection{Quasi-Monte Carlo and moment matching} +\label{sec:qmc} + +\paragraph{Sobol' points.} Instead of pseudo-random $z_s\sim N(0,I)$ the implementation uses +scrambled Sobol' points \cite{Owen1998}: $S$ (rounded up to a power of two) points $u_s\in(0,1)^N$ +that fill the unit cube far more evenly than random points, mapped through the inverse normal +distribution function, $z_s=\Phi^{-1}(u_s)$. The scrambling is seeded, so the set is +reproducible. The first coordinates of a Sobol' sequence are the best distributed, so the draws +are coloured with the eigen-root $A=V\Lambda^{1/2}$ of $\Sigma$ with eigenvalues in descending +order: the first coordinate drives the market factor, which dominates every long-only +portfolio's loss \cite[ch.~5]{Glasserman2004}. + +\paragraph{Moment matching.} Whatever the base sample, its first two moments can be made +\emph{exact} \cite{HoylandWallace2001}. With probabilities $p_s$ (equal unless importance +sampling is used, Section~\ref{sec:is}), let $m=\sum_sp_sz_s$ and $C=\sum_sp_s(z_s-m)(z_s-m)^\top$; +then +\begin{equation} + \tilde z_s \;=\; C^{-1/2}(z_s-m), \qquad + \sum_sp_s\tilde z_s=0,\quad \sum_sp_s\tilde z_s\tilde z_s^\top=I, + \label{eq:mm} +\end{equation} +and the scenarios $r_s=h m_0+\sqrt h\,A\tilde z_s$ have mean $hm_0$ and covariance $h\Sigma$ +exactly, for any $S>N$. Only the \emph{shape} of the tail remains sampled. For the empirical +innovation model the same transform is applied to the resampled sums of innovations (target +covariance $hI$), so that both models reproduce the latest covariance exactly. + +\paragraph{The objective uses the model mean.} Once the mean is exact there is no reason to +estimate it from the scenarios at all. The implementation puts the model's expected simple +return $\mu_i$ (the Black--Litterman posterior, converted to the horizon) directly into the +objective, $\bar g_i=P_i\mu_i$, and uses the scenarios only in the CVaR rows. This also keeps +the objective exact under importance sampling, where a tail-focused sample estimates the mean +\emph{worse} than a plain one. The convention behind $\mu_i$ matters: Black--Litterman returns +are arithmetic expectations, so the log-drift of the scenarios is $\ln(1+h\mu_i)/h-\sigma_i^2/2$ +(It\^o correction) and $\E[e^{r_i}-1]=h\mu_i$; using $\mu_i$ as the log-drift would add +$\sigma_i^2/2$ to the expected simple return --- 5\,\% p.a.\ for a typical stock and 34\,\% for +the most volatile one in the universe --- and tilt the optimiser towards volatile names. + +\subsection{Importance sampling for the tail} +\label{sec:is} + +Only the scenarios in the tail of the loss matter for the CVaR: at $\beta=0.99$, 1\,\% of them. +Importance sampling draws more scenarios where they matter and corrects with likelihood ratios +\cite{GlassermanHeidelbergerShahabuddin2000}. Three things are specific to the optimisation +setting. + +\emph{The tail depends on the decision.} The loss of portfolio $v=P\circ n$ is +$L=-v^\top r=-v^\top hm_0-(\sqrt hA^\top v)^\top z$, so its tail is the half-space +$\{z:\,d^\top z\ge c\}$ with the unit direction $d=-A^\top v/\|A^\top v\|$. Because all +long-only, capped, sector-limited candidates load mainly on the market factor, one direction +serves all of them. The default takes $v$ from the \emph{cap-weighted portfolio of the candidate +stocks} (\code{importance\_sampling="market"}), which depends on the universe and the covariance +but on none of the parameters a user varies; the alternative \code{"adaptive"} takes it from a +first-pass solution of the problem itself, which is marginally sharper at $\beta=0.99$ but makes +the scenario set depend on the risk limit and the constraints (Section~\ref{sec:onesetup}). + +\emph{Bounded likelihood ratios.} A share $\varepsilon$ (default one half) of the Sobol' points +is shifted, $z_s\mapsto z_s+\theta$ with $\theta=z_\beta d$, which centres the loss of $v^{(1)}$ +at its VaR; the rest stay nominal. The proposal is the defensive mixture +$g=(1-\varepsilon)\varphi+\varepsilon\varphi(\cdot-\theta)$ \cite{Hesterberg1995}, whose +likelihood ratios +\begin{equation} + \ell_s=\frac{\varphi(z_s)}{g(z_s)}=\frac{1}{(1-\varepsilon)+\varepsilon\,e^{\theta^\top z_s-\|\theta\|^2/2}} + \;\le\;\frac{1}{1-\varepsilon} + \label{eq:lr} +\end{equation} +are bounded, so a candidate whose tail points elsewhere is never wildly mis-estimated (the +optimiser would exploit any such region). The scenario probabilities are the self-normalised +ratios $p_s=\ell_s/\sum_{s'}\ell_{s'}$, and the moment matching \eqref{eq:mm} is applied with +these weights. + +\emph{Weighted scenarios in the MILP.} Theorem~\ref{thm:ru} holds for any discrete +distribution: with probabilities $p_s$ the CVaR row \eqref{eq:milp-n-cvar} becomes +\begin{equation} + \zeta+\frac{1}{1-\beta}\sum_sp_su_s\;\le\;\Lambda, + \label{eq:cvar-weighted} +\end{equation} +and nothing else changes; the decoded VaR and CVaR are computed from the weighted sample in the +same way (the tail is the set of largest losses whose probability adds up to $1-\beta$, the last +one taken fractionally). + +For the empirical innovation model the tilt acts on the days: the resampling probabilities are +$q_t\propto\exp(\vartheta\,d^\top\varepsilon_t)$ with $\vartheta$ chosen so that the tilted mean +of $d^\top\varepsilon$ equals $z_\beta/\sqrt h$ per day (a one-dimensional root), the mixture +draws all $h$ days of a scenario either from $q$ or uniformly, and the likelihood ratio of a +scenario is $1/\big((1-\varepsilon)+\varepsilon\prod_kTq_{t_k}\big)$. + +\begin{table}[ht] +\centering +\footnotesize +\setlength{\tabcolsep}{3.2pt} +\begin{tabular}{@{}lrrrrrrrr@{}} +\toprule +& \multicolumn{2}{c}{$0.95$, $2\,048$} & \multicolumn{2}{c}{$0.95$, $8\,192$} & \multicolumn{2}{c}{$0.99$, $2\,048$} & \multicolumn{2}{c}{$0.99$, $8\,192$} \\ +\cmidrule(lr){2-3}\cmidrule(lr){4-5}\cmidrule(lr){6-7}\cmidrule(lr){8-9} +$\beta$, $S$ & S+mm & + IS & S+mm & + IS & S+mm & + IS & S+mm & + IS \\ +\midrule +scenarios beyond the VaR & 101 & 560 & 410 & 2\,249 & 19 & 512 & 80 & 2\,075 \\ +true CVaR over the limit & 0.15\,\% & 0.21\,\% & 0.08\,\% & 0.01\,\% & 2.45\,\% & 0.50\,\% & 0.75\,\% & 0.03\,\% \\ +sd of the true CVaR (\% of limit) & 0.74 & 0.30 & 0.36 & 0.15 & 1.67 & 0.40 & 0.79 & 0.20 \\ +loss versus the optimum (CHF) & 0.4 & 1.1 & 0.3 & 0.0 & 10.7 & 2.7 & 3.6 & 0.1 \\ +turnover between two seeds & 5.9\,\% & 3.6\,\% & 4.0\,\% & 1.6\,\% & 29\,\% & 11.5\,\% & 15.4\,\% & 5.7\,\% \\ +\bottomrule +\end{tabular} +\caption{Importance sampling (``+ IS'': mixture $\varepsilon=0.5$, two-stage tilt) versus +moment-matched Sobol' points (``S+mm'') at equal $S$; 30 stocks, eight seeds, exact Gaussian +reference.} +\label{tab:is} +\end{table} + +At equal $S$ the tail noise falls by a factor 2--2.5 at 95\,\% and 3--4 at 99\,\%; equivalently, +2\,048 tilted scenarios are as good as 8\,192 plain ones. The gain is largest exactly where it +is needed, at $\beta=0.99$, where a plain set of 2\,048 scenarios has 19 tail points. The +tilted LP is slower per scenario (a quarter of its rows are active instead of $1-\beta$), but the +market tilt is on by default (\code{importance\_sampling="auto"} $=$ \code{"market"}) because it +costs nothing in accuracy at 95\,\% and halves the noise at 99\,\%. Table~\ref{tab:is-market} +compares the two directions. Under the Gaussian model importance sampling is unnecessary --- the +exact solution of Section~\ref{sec:gaussian} needs no scenarios at all. + +\begin{table}[ht] +\centering +\footnotesize +\setlength{\tabcolsep}{4pt} +\begin{tabular}{@{}lrrrrrr@{}} +\toprule +& \multicolumn{3}{c}{$\beta=0.95$} & \multicolumn{3}{c}{$\beta=0.99$} \\ +\cmidrule(lr){2-4}\cmidrule(lr){5-7} +$S=2\,048$ & no tilt & market & adaptive & no tilt & market & adaptive \\ +\midrule +true CVaR over the limit & 0.14\,\% & 0.37\,\% & 0.20\,\% & 2.43\,\% & 1.37\,\% & 0.50\,\% \\ +sd of the true CVaR (\% of limit) & 0.74 & 0.72 & 0.30 & 1.69 & 0.61 & 0.42 \\ +turnover between two seeds & 5.7\,\% & 3.2\,\% & 3.6\,\% & 28.5\,\% & 17.4\,\% & 11.6\,\% \\ +turnover to the optimum & 4.1\,\% & 2.6\,\% & 2.4\,\% & 22.4\,\% & 13.0\,\% & 8.7\,\% \\ +depends on the risk parameters & no & no & \textbf{yes} & no & no & \textbf{yes} \\ +\bottomrule +\end{tabular} +\caption{Tilt direction: the cap-weighted candidate portfolio (``market'', two blocks at +$z_{0.95}$ and $z_{0.99}$) versus a first-pass solution (``adaptive'', one block at $z_\beta$); +30 stocks, eight seeds.} +\label{tab:is-market} +\end{table} + +\emph{How far does one direction carry?} The tilt enriches the tail of the \emph{reference} +portfolio; another portfolio benefits in proportion to how much its loss is correlated with the +reference's. Table~\ref{tab:alignment} measures this for portfolios from market-like to +deliberately orthogonal. Long-only portfolios under single-stock and sector caps are market-like +in practice --- the optimiser's solutions come out at an alignment of about $0.90$ --- and the +reference is the cap-weighted portfolio \emph{of the candidate stocks}, so a defensive universe +gets a defensive reference. Where the alignment is nevertheless low the tilt does not help and +costs a little precision (the nominal half of the mixture bounds the damage); the optimisation +result reports the realised alignment (\code{tilt\_alignment}) and warns below $0.3$, where +\code{importance\_sampling="adaptive"} or \code{"off"} is the better choice. + +\begin{table}[ht] +\centering +\footnotesize +\setlength{\tabcolsep}{4pt} +\begin{tabular}{@{}lrrrrrr@{}} +\toprule +& & \multicolumn{2}{c}{scenarios beyond the VaR} & \multicolumn{3}{c}{noise of the CVaR estimate} \\ +\cmidrule(lr){3-4}\cmidrule(lr){5-7} +portfolio & alignment & plain & tilted & plain & tilted & reduction \\ +\midrule +cap-weighted (the reference) & 1.00 & 41 & 789 & 0.99\,\% & 0.19\,\% & 5.3$\times$ \\ +equal weight & 0.89 & 41 & 644 & 0.79\,\% & 0.30\,\% & 2.6$\times$ \\ +mean--CVaR optimum & 0.90 & 41 & 630 & 1.39\,\% & 0.61\,\% & 2.3$\times$ \\ +five highest-beta stocks & 0.68 & 41 & 368 & 0.22\,\% & 0.34\,\% & 0.6$\times$ \\ +minimum variance & 0.69 & 41 & 350 & 1.34\,\% & 1.25\,\% & 1.1$\times$ \\ +five lowest-beta stocks & $-0.09$ & 41 & 34 & 1.66\,\% & 1.96\,\% & 0.8$\times$ \\ +three least market-like stocks & $-0.15$ & 41 & 31 & 1.93\,\% & 1.95\,\% & 1.0$\times$ \\ +\bottomrule +\end{tabular} +\caption{Reach of one tilt direction: ``alignment'' is the correlation between the portfolio's +loss and the reference's; $S=4\,096$, $\beta=0.99$, 12 seeds, 30 stocks. Noise is the standard +deviation of the CVaR estimate across seeds, relative to its mean.} +\label{tab:alignment} +\end{table} + +\subsection{One scenario set for a family of runs} +\label{sec:onesetup} + +A portfolio proposal is rarely produced once: a risk limit is varied, a constraint tightened, a +view added, two confidence levels compared. If each of those runs drew its own scenarios, the +differences between the results would mix the effect of the parameter with the sampling noise of +a new set --- precisely the noise Sections~\ref{sec:qmc} and \ref{sec:is} remove. The +implementation therefore makes the scenario set a function of the \emph{model} only. Its +\emph{scenario key} (reported with every result) contains +\begin{center} +\footnotesize +\begin{tabular}{@{}p{7.2cm}p{7.2cm}@{}} +\toprule +in the key (changing it draws new scenarios) & not in the key (the set is reused) \\ +\midrule +candidate stocks, data as-of date, frequency and horizon, covariance model, scenario model, +number of scenarios, sampling scheme, moment matching, seed, tilt mode and mixture, innovation +history and re-standardisation +& +CVaR limit (absolute or as a fraction), confidence level $\beta$, objective, risk aversion, +target return, all constraints, transaction costs, cash, existing holdings, expected-return +model and views, solver and time limit \\ +\bottomrule +\end{tabular} +\end{center} +Two properties make this possible. First, the tilt of Section~\ref{sec:is} uses the cap-weighted +candidate portfolio and blocks at both usual confidence levels, so it does not look at the +optimisation parameters. Second, the drift enters \emph{after} the random part: scenarios are +generated with zero expected return and the expected-return model is added as a per-asset +constant, $r_s\mapsto r_s+hm$, by \code{ScenarioSet.with\_drift}. Changing a view therefore +shifts every scenario by a constant and leaves the shocks --- and hence the whole risk picture --- +untouched; the result reports a fingerprint of the shocks (\code{shock\_hash}) next to the one of +the scenarios (\code{hash}). Sets are cached per key inside the service, so a family of runs also +pays the expensive innovation filter only once (measured: 14\,s for the first run, 0\,s +afterwards). + +Only \code{importance\_sampling="adaptive"} breaks the property, since its tilt comes from a +first-pass solve; the key then lists the parameters of that solve, so runs that differ in them +are not silently compared on different scenarios. + +\subsection{How many scenarios, and how many assets?} +\label{sec:howmany} + +\paragraph{The count has to grow with the candidate set.} The approximation searches an +$N$-dimensional space, so a fixed $S$ buys less precision as $N$ grows. Measured against the +exact optimum of Section~\ref{sec:gaussian} (GBM model, four seeds): + +\begin{center} +\footnotesize +\begin{tabular}{@{}rrrrrr@{}} +\toprule +$N$ & $S$ & $S/N$ & turnover between two seeds & turnover to the optimum & sd of $\E[\text{P\&L}]$ \\ +\midrule + 30 & 512 & 17 & 9.6\,\% & 7.2\,\% & 17.6 CHF \\ + 30 & 2\,048 & 68 & 3.5\,\% & 4.2\,\% & 2.8 \\ + 30 & 8\,192 & 273 & 2.6\,\% & 3.8\,\% & 1.1 \\ +100 & 512 & 5 & 26.5\,\% & 21.5\,\% & 15.5 \\ +100 & 2\,048 & 20 & 12.5\,\% & 10.0\,\% & 2.8 \\ +100 & 8\,192 & 82 & 6.2\,\% & 6.2\,\% & 2.0 \\ +300 & 512 & 1.7 & 22.7\,\% & 23.4\,\% & 18.5 \\ +300 & 2\,048 & 6.8 & 9.0\,\% & 6.8\,\% & 4.2 \\ +\bottomrule +\end{tabular} +\end{center} + +Two readings of the same table matter. The \emph{reported figures} are stable everywhere: from +2\,000 scenarios on, the expected profit of the proposal varies by less than $0.1\,\%$ between +seeds. The \emph{composition} is what moves, and at fixed $S$ it moves roughly in proportion to +$N$ (3.5\,\% at $N=30$ against 12.5\,\% at $N=100$, both at $S=2\,048$) --- with hundreds of +candidates and a 10\,\% cap there are many portfolios within a fraction of a percent of the +optimum, and the sample decides between them. About 64 scenarios per candidate keeps the +disagreement near 5\,\%; as a default that is too expensive ($N=300$ would mean 32\,768 +scenarios and minutes per solve), so the implementation draws 16 per candidate within +$[4\,096,\,16\,384]$ and says in the result when the ratio is low. Where the composition itself +has to be reproducible --- a backtest, a teaching example --- either raise $S$ or use +\code{scenario\_model="gaussian\_exact"}, which has no sampling error at all. + +\paragraph{Cost at the top end, and which solver.} A model with $N=300$ and $S=32\,768$ has 9.9\,M +non-zeros: building it takes 3.3\,s, encoding it as msgpack 0.6\,s, and the 108\,MB body transfers to +the cuOpt server at about 10\,MB/s. Transport is therefore not the constraint; proving optimality is. +Table~\ref{tab:solvers} compares the two back ends with both running on the Spark, end to end through +the service (scenario generation, build, solve, decode). + +\begin{table}[ht] +\centering +\footnotesize +\begin{tabular}{@{}lrr@{}} +\toprule +case & cuOpt (GPU) & HiGHS (CPU, 4 threads) \\ +\midrule +$N=30$, $S=8\,192$ & 15.6\,s & 14.5\,s \\ +$N=30$, $S=32\,768$ & 171.4\,s & 171.0\,s \\ +$N=100$, $S=4\,096$ & 10.0\,s & 8.9\,s \\ +$N=100$, $S=32\,768$ & 606.9\,s (time limit) & 350.7\,s (optimal) \\ +$N=300$, $S=8\,192$, row generation & 139\,s & 64\,s \\ +\bottomrule +\end{tabular} +\caption{The same models on the same machine, weekly horizon, $\beta=0.95$, 600\,s solver time limit.} +\label{tab:solvers} +\end{table} + +HiGHS is at least as fast in every case and twice as fast on the two largest. This is not a statement +about GPU optimisation in general. cuOpt's published speedups concern \emph{linear} programmes solved +with its PDLP first-order method; for mixed-integer problems its documentation states that the GPU runs +primal heuristics while \emph{branch and bound runs on the CPU}, that the MIP solver is in beta, and +that proving feasible solutions optimal ``remains under active development''. The models here have an +easy LP relaxation and a small integrality gap, so nearly all the work is the optimality proof --- the +part that is not accelerated --- whereas HiGHS warm-starts its dual simplex across branch-and-bound +nodes and, with row generation, solves a sequence of small problems in-process while cuOpt pays a fixed +cost per request. HiGHS is a demanding reference: in the Mittelmann benchmark of 7 July 2026 it solves +179 of 240 instances at a scaled shifted geometric mean of 5.44, against 1.72 for the commercial COPT +and 9.93 for SCIP. Three caveats: cuOpt's time limit covers only its internal solve, it ignores the +MIP-gap settings sent over REST and may therefore prove optimality to a tighter tolerance, and this is +one problem family on one machine rather than a benchmark. + +\subsection{Tail-row generation for large scenario sets} +\label{sec:rowgen} + +The MILP has one row and one variable $u_s$ per scenario; with $N+2$ non-zeros per row the +matrix grows linearly in $S$, but the simplex work does not: HiGHS' dual simplex on the LP +relaxation scales roughly like $S^{1.9}$ and the branch-and-bound like $S^{1.65}$, cuOpt like +$S^{1.2}$--$S^{1.6}$ (Table~\ref{tab:scaling}). Going from 2\,000 to 100\,000 scenarios costs +about 45$\times$ on cuOpt and more than 200$\times$ on HiGHS. + +\begin{table}[ht] +\centering +\small +\begin{tabular}{@{}rrrrrrr@{}} +\toprule +$S$ & non-zeros & HiGHS LP & HiGHS MILP & cuOpt MILP & payload & row generation (HiGHS) \\ +\midrule +\multicolumn{7}{@{}l}{$N=30$} \\ +2\,048 & 68\,k & 0.03\,s & 1.5\,s & 5.1\,s & 1.7\,MB & -- \\ +8\,192 & 270\,k & 0.4\,s & 14.5\,s & 10.8\,s & 6.6\,MB & 5.4\,s (2 iterations) \\ +32\,768 & 1.1\,M & 6.0\,s & 166\,s & 66.6\,s & 27\,MB & 25\,s (3 iterations) \\ +100\,000 & 3.3\,M & 56.5\,s & $>300$\,s & 220\,s & 82\,MB & 158\,s (2 iterations) \\ +\multicolumn{7}{@{}l}{$N=100$} \\ +2\,048 & 211\,k & 0.1\,s & 9.5\,s & 7.0\,s & 5\,MB & \\ +8\,192 & 844\,k & 1.2\,s & 50\,s & 11.5\,s & 21\,MB & \\ +32\,768 & 3.4\,M & 16.5\,s & 232\,s & 101\,s & 82\,MB & \\ +\bottomrule +\end{tabular} +\caption{Solve times versus the number of scenarios (MIP gap $10^{-4}$, $\beta=0.95$). The HiGHS +column was measured on a development laptop whose SciPy bundles HiGHS 1.2.0 and the cuOpt column on +the DGX Spark, so the two columns show the growth with $S$ for each solver but are not a like-for-like +comparison; Table~\ref{tab:solvers} compares them on the same machine.} +\label{tab:scaling} +\end{table} + +At the optimum only the scenarios in the tail have $u_s>0$; every other scenario row is slack. +\emph{Row generation} \cite{KunziBayMayer2006} exploits this: +\begin{enumerate}[nosep] +\item Start with a working set $\mathcal W$: the $2(1-\beta)S$ worst scenarios of a reference + portfolio (the current holdings, or an equal-weight allocation). +\item Solve the MILP with the scenario rows of $\mathcal W$ only, keeping the full-set + probabilities $p_s$ in \eqref{eq:cvar-weighted} (omitted scenarios contribute $u_s=0$) and the + full-set objective. +\item Compute the losses $L_s$ of all $S$ scenarios for the solution $(n^\ast,\zeta^\ast)$; add + every $s\notin\mathcal W$ with $L_s>\zeta^\ast$ to $\mathcal W$ and repeat; stop when there is + none. +\end{enumerate} +The restricted problem drops constraints, so it is a relaxation and its objective bounds the +full one from above; when no omitted scenario is violated the restricted solution is feasible +for the full problem with $u_s=0$ on the omitted rows, hence optimal (within the MIP gap). The +working set stays at about 11\,\% of $S$ at $\beta=0.95$ (2--3\,\% at $\beta=0.99$), two or three +iterations suffice, and the speed-up over the full MILP is 6--7$\times$ from 32\,768 scenarios +on; only the working-set rows are sent to the solver. The default +(\code{row\_generation="auto"}) uses it from 8\,192 scenarios on. + +\subsection{The Gaussian special case: mean--variance equivalence and an exact solution} +\label{sec:gaussian} + +\begin{proposition} +\label{prop:elliptic} +Let the horizon returns be elliptically distributed with mean $m$ and dispersion $C$, e.g.\ +$R\sim N(m,C)$. Then for every portfolio $\CVaR_\beta(L(w))=-m^\top w+k_\beta\,\sigma(w)$ with +$\sigma(w)=\sqrt{w^\top Cw}$ and a constant $k_\beta$ ($k_\beta=\varphi(z_\beta)/(1-\beta)$ in +the normal case, $2.063$ at 95\,\% and $2.665$ at 99\,\%). Consequently the solution of +``maximise $m^\top w$ subject to $\CVaR_\beta\le\Lambda$'' lies on the mean--variance efficient +frontier, and it is also the solution of ``maximise $m^\top w$ subject to +$\sigma(w)\le\bar\sigma$'' with the implicit bound $\bar\sigma=(\Lambda+m^\top w^\ast)/k_\beta$. +\end{proposition} + +\begin{proof} +For elliptical returns the loss of every portfolio is a location--scale transform of one and +the same symmetric distribution, so every positively homogeneous, translation-invariant risk +measure is an affine function of $\sigma(w)$ \cite{EmbrechtsMcNeilStraumann2002}. Let $w^\ast$ +be mean--CVaR optimal with mean $m^\ast$. Any feasible $w$ with $m^\top w\ge m^\ast$ and +$\sigma(w)\le\bar\sigma$ has $\CVaR_\beta\le-m^\ast+k_\beta\bar\sigma=\Lambda$, hence is +mean--CVaR feasible and cannot beat $m^\ast$: the two problems have the same optimal value, and +the same solutions up to ties. The same argument applies over the integer feasible set. +\end{proof} + +The variance bound is only known \emph{ex post} (it depends on $m^\ast$), so the mean--CVaR +problem with a Gaussian model is solved directly. In whole shares it is +\begin{equation} + \max_n\; m^\top v-\TC \quad\text{s.t.}\quad + g(n):=\TC-m^\top v+k_\beta\sqrt{v^\top Cv}\;\le\;\Lambda,\qquad v=P\circ n, + \label{eq:misocp} +\end{equation} +plus the linear constraints of Section~\ref{sec:shares}: a convex mixed-integer +second-order-cone programme (a mixed-integer QP in the utility form), which neither cuOpt nor +HiGHS solves directly. The function $g$ is convex, so it can be outer-approximated by tangent +cuts on the same MILP \cite{Kelley1960,DuranGrossmann1986}. At an exposure $v_k$ with +$\sigma_k=\sqrt{v_k^\top Cv_k}>0$ the gradient of $\sigma$ is $Cv_k/\sigma_k$, and because +$(Cv_k)^\top v_k/\sigma_k=\sigma_k$ the tangent simplifies to +\begin{equation} + g(n)\;\ge\;\TC+\Big(k_\beta\frac{Cv_k}{\sigma_k}-m\Big)^{\!\top}v . + \label{eq:cut} +\end{equation} +With an epigraph variable $t$ (\code{cvar} in the code) the MILP carries the rows +$\TC+(k_\beta Cv_k/\sigma_k-m)^\top(P\circ n)-t\le0$ for the cuts collected so far and +$t\le\Lambda$ (or $t$ in the objective for \code{min\_cvar} and \code{utility}). The loop +starts with a cut at a reference portfolio, solves the MILP, compares the true $g(n^\ast)$ with +$t^\ast$, and adds a cut at $v^\ast$ until they agree (relative tolerance $10^{-6}$). Each cut +removes the current infeasible point and the integer set is finite, so the loop terminates at +the exact optimum within the MIP gap. On the universe it takes 20--70 cuts (each a MILP with +$N$ integers and a few dozen rows, milliseconds on HiGHS) and reproduces Gurobi's direct +MISOCP solution to the cent. This is \code{scenario\_model="gaussian\_exact"}: no scenarios, +no sampling error, fully deterministic. + +\begin{remark} +The GBM scenario model is lognormal in simple returns ($R=e^r-1$ with $r$ normal), the exact +mode is normal in simple returns. At weekly volatilities of 4--5\,\% the lognormal left tail is +2--3\,\% thinner, so the scenario optimum takes a few percent more exposure; the sharp test of +the sampling machinery therefore uses Gaussian scenarios linearly, where moment-matched Sobol' +points reproduce the exact optimum within 1\,\% and plain Monte Carlo does not +(\code{tests/test\_rowgen\_gaussian.py}). For the empirical innovation model no closed form +exists --- that is the point of using it --- and the equivalence serves only as a calibration of +the sampling mechanism. +\end{remark} + +\subsection{Rank of the covariance estimate} +\label{sec:rank} + +An estimator with $T_{\mathrm{eff}}$ effective observations cannot support more than +$T_{\mathrm{eff}}$ assets: a 200-day window with $N=300$ gives a sample covariance of rank 200, +and a portfolio in its null space has \emph{zero} estimated variance --- an optimiser maximising +return under a risk cap will find it. The shrinkage of the measurement note repairs exactly this, +and its data-driven intensity rises with $N/T$ on its own: + +\begin{center} +\footnotesize +\begin{tabular}{@{}lrrrrr@{}} +\toprule +estimator, $N$ & $T_{\mathrm{eff}}$ & intensity & $\lambda_{\min}/\overline{\sigma^2}$ & condition & zero eigenvalues \\ +\midrule +rolling 200, $N=300$ & 200 & 0 & $-3.4\cdot10^{-15}$ & $4\cdot10^{298}$ & 101 \\ +rolling 200, $N=300$ & 200 & 0.080 & 0.023 & $3.0\cdot10^{3}$ & 0 \\ +rolling 200, $N=745$ & 200 & 0 & $-2.0\cdot10^{-14}$ & $6\cdot10^{298}$ & 546 \\ +rolling 200, $N=745$ & 200 & 0.128 & 0.025 & $4.3\cdot10^{3}$ & 0 \\ +EWM $\lambda=0.99$, $N=745$ & 300 & 0 & $2.2\cdot10^{-5}$ & $5.5\cdot10^{6}$ & 44 (numerically) \\ +EWM $\lambda=0.99$, $N=745$ & 300 & 0.114 & 0.023 & $4.8\cdot10^{3}$ & 0 \\ +\bottomrule +\end{tabular} +\end{center} + +With \code{shrinkage="auto"} --- the default everywhere --- the smallest eigenvalue never falls +below 2\,\% of the average variance, whatever the number of candidates. Without it the +degeneracy is real and silent: a null-space portfolio comes out at variance $-1.6\cdot10^{-20}$ +and the Sobol' draws span only 222 of 300 dimensions. \code{CovarianceModel.conditioning} +reports the figures and the optimisation warns when the estimate is singular or +ill-conditioned for the size of the candidate set. + +Two things soften the problem in this particular model, and neither should be relied on. An EWM +estimator over the full history is formally full rank (every observation keeps a positive +weight), so its degeneracy is numerical rather than exact; and no-short-selling with a 10\,\% +cap makes null directions hard to reach, since exploiting one generally requires a short leg. +Measured across $N=30\ldots745$, the optimised portfolio's out-of-sample-to-in-sample volatility +ratio exceeds the cap-weighted portfolio's by a factor of only 0.89--1.18, with or without +shrinkage --- the optimiser is not systematically mining the estimate. The exactly singular case +does bite elsewhere, though: with a 200-day window and $N\ge300$ the outer approximation of +Section~\ref{sec:gaussian} stops converging and returns a time-limited solution. + +\subsection{Reproducibility} +\label{sec:reproducibility} + +Two runs of the same request use the same scenario set, and so do two runs that differ only in +the parameters listed on the right of Section~\ref{sec:onesetup}: the base sample depends only on the +seed (scrambled Sobol' points and the PCG64 generator are deterministic across platforms), the +transformations are deterministic, and the result reports a fingerprint of the final matrix and +its probabilities (\code{scenario\_hash}). The fingerprint is exact on one platform; between +BLAS implementations (laptop versus the DGX Spark) the GBM sets agree to $10^{-11}$ and keep the +same hash, while the empirical model's innovation filter --- thousands of eigendecompositions --- +can differ at the $10^{-10}$ level, which changes the hash but none of the reported figures. +Whether the \emph{solution} is the same depends on the solver. HiGHS' branch-and-bound is deterministic: identical inputs give identical shares +(\code{reproducibility="exact"}) as long as the HiGHS build is the same --- across SciPy versions +the search may end on a different optimal vertex among ties, at the same objective. cuOpt combines GPU heuristics with a time-limited search and, +in version 26.08, ignores the MIP-gap settings sent over its REST interface; it proves +optimality within its default relative gap of $10^{-4}$, so the objective, expected profit and +CVaR of two runs agree to that gap (about 0.2 CHF per week on a CHF 1 million portfolio) while +near-tied share counts can differ by a few shares (\code{reproducibility="within\_gap"}). A +result that hit the time limit (\code{FeasibleFound}) is not reproducible at all. The +Gaussian exact mode runs on HiGHS by default and is exact in this sense. + +% ============================================================================================ +\section{The complete model as implemented} +% ============================================================================================ + +\code{MeanCVaRProblem.build()} assembles the following MILP from a \code{ProblemSpec}. The +variable vector is laid out as $x=(n,\;[b,\;s],\;\alpha,\;u,\;[y],\;[t])$; blocks in brackets +exist only in rebalancing mode ($b,s$), with cardinality/minimum-position rules ($y$) or with a +fixed cost per trade ($t$). The CVaR level $\zeta$ is called $\alpha$ in the code. + +\begin{center} +\small +\begin{tabular}{@{}llll@{}} +\toprule +block & name in the code & type, bounds & meaning \\ +\midrule +$n_i$ & \code{n[symbol]} & integer, $[0,\bar n_i]$ & final number of shares \\ +$b_i$ & \code{buy[symbol]} & integer, $[0,\bar n_i]$ & shares bought (rebalancing) \\ +$s_i$ & \code{sell[symbol]} & integer, $[0,n^0_i]$ & shares sold (rebalancing) \\ +$\alpha$ & \code{alpha} & continuous, free & the $\zeta$ of Theorem~\ref{thm:ru} \\ +$u_s$ & \code{u[s]} & continuous, $[0,\infty)$ & excess loss of scenario $s$ over $\alpha$ \\ +$t_{\mathrm{cvar}}$ & \code{cvar} & continuous, free & Gaussian CVaR epigraph (Section~\ref{sec:gaussian}) \\ +$y_i$ & \code{y[symbol]} & binary & position open (cardinality / minimum position) \\ +$t_i$ & \code{t[symbol]} & binary & stock traded (fixed cost) \\ +\bottomrule +\end{tabular} +\end{center} + +With $\bar n_i=\lfloor \min(\bar w,f_{\max})\,W/P_i\rfloor$, $g_{s,i}=P_iR_{s,i}$, +$\bar g_i=P_i\mu_i$ (the model's expected simple return over the horizon; the +probability-weighted scenario mean if no model mean is supplied), $\kappa_s=\frac{p_s}{1-\beta}$ +with $p_s=1/S$ unless the scenarios carry importance-sampling probabilities, and +$\TC=\sum_i(c^bP_ib_i+c^sP_is_i)+F\sum_it_i$ (in mode \code{new}: $c^b\sum_iP_in_i+F\sum_it_i$), +the rows are: + +\begin{center} +\footnotesize +\begin{tabular}{@{}lp{6.1cm}p{2.9cm}l@{}} +\toprule +row name & constraint & count & origin \\ +\midrule +\code{cvar\_scen\_s} & $\sum_i g_{s,i}n_i - \TC + \alpha + u_s \ge 0$ & $S$ & \eqref{eq:lin}, \eqref{eq:u-reb} \\ +\code{cvar\_limit} & $\alpha + \sum_s \kappa_s u_s \le \Lambda$ \newline (Gaussian: $t_{\mathrm{cvar}}\le\Lambda$) & 1, if a limit is given & \eqref{eq:cvar-constraint}, \eqref{eq:cvar-weighted} \\ +\code{cut\_j} & $\TC+(k_\beta Cv_j/\sigma_j-m)^{\top}(P\circ n)$ \newline $\quad-\,t_{\mathrm{cvar}}\le0$ & one per cut, \newline Gaussian mode & \eqref{eq:cut} \\ +\code{budget} & $\sum_iP_ib_i-\sum_iP_is_i+\TC\le c^0$ \newline (new: $\sum_iP_in_i+\TC\le c^0$) & 1 & \eqref{eq:cash} \\ +\code{link\_i} & $n_i-b_i+s_i=n^0_i$ & $N$, rebalancing & \eqref{eq:link} \\ +\code{invested} & $f_{\min}W\le\sum_iP_in_i\le f_{\max}W$ & $\le1$, if bounds given & invested fraction \\ +\code{sector\_k} & $\sum_{i\in G_k}P_in_i\le\bar w_kW$ & $K$, binding sectors & \eqref{eq:milp-n-sector} \\ +\code{max\_cost} & $\TC\le\TC_{\max}$ & $\le1$ & \eqref{eq:turnover} \\ +\code{turnover} & $\sum_iP_i(b_i+s_i)\le\tau W$ & $\le1$, rebalancing & \eqref{eq:turnover} \\ +\code{open\_i} & $n_i-\max(\bar n_i,1)\,y_i\le0$ & $N$, optional & \eqref{eq:card} \\ +\code{minpos\_i} & $P_in_i-v_{\min}y_i\ge0$ & $N$, optional & \eqref{eq:card} \\ +\code{cardinality} & $\sum_iy_i\le K_{\max}$ & $\le1$, optional & \eqref{eq:card} \\ +\code{trade\_i} & $b_i+s_i-M_it_i\le0$ \newline (new: $n_i-M_it_i\le0$) & $N$, fixed cost only & \eqref{eq:tc} \\ +\code{target\_pnl} & $\sum_i\bar g_in_i-\TC\ge\rho$ & 1, \code{min\_cvar} only & \eqref{eq:three}(b) \\ +\bottomrule +\end{tabular} +\end{center} + +The objective is one of +\begin{align} + \code{max\_return}:&\quad \max\; \sum_i\bar g_i n_i - \TC \quad\text{subject to \code{cvar\_limit}},\\ + \code{min\_cvar}:&\quad \min\; \alpha+\sum_s\kappa_s u_s \quad\text{subject to \code{target\_pnl}},\\ + \code{utility}:&\quad \max\; \sum_i\bar g_i n_i - \TC - \lambda\Big(\alpha+\sum_s\kappa_s u_s\Big), +\end{align} +the three forms of \eqref{eq:three} (in Gaussian mode $\alpha+\sum_s\kappa_su_s$ is replaced by +$t_{\mathrm{cvar}}$). All coefficients are in CHF; the scenario rows carry $N(+2N)+2$ non-zeros +each, so the matrix has about $S(N+2)$ non-zeros (420\,k for $N=100$, $S=4\,096$; 3\,M for the +full universe of 745 stocks with $S=4\,096$). Duplicate index pairs +that arise when a variable appears in several terms of a row (e.g.\ $n_i$ in the gain and in the +cost of mode \code{new}) are summed when the sparse matrix is assembled. + +\paragraph{The two solvers.} +The same \code{MILPModel} is accepted by both back ends. \emph{cuOpt} is NVIDIA's GPU +optimisation library, reached over HTTP as a self-hosted server (version 26.08 here) running as a +container on the DGX Spark; it is the only component that uses the GPU. \emph{HiGHS} is the +open-source LP/MILP solver of ERGO-Code at the University of Edinburgh (MIT licence, +\url{https://highs.dev}) \cite{HuangfuHall2018}; it is not installed or started separately but +ships compiled inside SciPy, so \code{scipy.optimize.milp} runs it \emph{in the calling process on +the CPU} --- in production inside the same container as the service (SciPy 1.18.1, HiGHS 1.12.0, +four threads). It serves as the fallback when cuOpt is unreachable, as the solver of the small cut +MILPs of the Gaussian mode, and as the oracle of the test suite. + +\paragraph{Solving and decoding.} +The model is sent to the cuOpt server as an LP/MILP request (CSR matrix, bounds, variable types, +a mandatory time limit; msgpack-encoded) and polled until a solution arrives; if the server is +unreachable the same \code{MILPModel} is handed to HiGHS. From 8\,192 scenarios on the +service solves by row generation (Section~\ref{sec:rowgen}), i.e.\ a sequence of such MILPs on +the working set; in Gaussian mode by the outer-approximation loop (Section~\ref{sec:gaussian}). +A status \code{Optimal} means proven optimality within the solver's relative gap; +\code{FeasibleFound} means the time limit stopped the search at an incumbent whose +\code{mip\_gap} is reported. The result states the scenario fingerprint and the level of +reproducibility (Section~\ref{sec:reproducibility}). From the solution vector, +\code{MeanCVaRProblem.decode()} rounds the integer blocks (they are integral up to solver +tolerance), rebuilds trades, costs, cash, position values and weights, and \emph{recomputes} the +expected profit (from the model means), VaR and CVaR of the decoded portfolio from the +weighted scenario set (Remark~\ref{rem:discrete}) or, in Gaussian mode, from the closed form; +these recomputed figures are what the service reports, together +with the provenance of every model input (covariance model, scenario model and seed, expected +returns and views, constraints, costs, solver statistics). + +\paragraph{What the model does not do.} +It assumes the scenario set represents the horizon distribution and that all trades execute at +today's prices; it holds the portfolio unchanged over the horizon (no intra-horizon rebalancing); +cash earns nothing over the horizon (the CHF risk-free rate is zero); it is single-period. These +are deliberate simplifications of a teaching platform, not limitations of the formulation --- +multi-period extensions replace the scenario rows by a scenario tree. + +% ============================================================================================ +\begin{thebibliography}{99} +\bibitem{RockafellarUryasev2000} + R.\,T.~Rockafellar and S.~Uryasev, + \emph{Optimization of conditional value-at-risk}, + Journal of Risk 2(3), 21--41, 2000. +\bibitem{RockafellarUryasev2002} + R.\,T.~Rockafellar and S.~Uryasev, + \emph{Conditional value-at-risk for general loss distributions}, + Journal of Banking \& Finance 26(7), 1443--1471, 2002. +\bibitem{KrokhmalPalmquistUryasev2002} + P.~Krokhmal, J.~Palmquist and S.~Uryasev, + \emph{Portfolio optimization with conditional value-at-risk objective and constraints}, + Journal of Risk 4(2), 43--68, 2002. +\bibitem{AcerbiTasche2002} + C.~Acerbi and D.~Tasche, + \emph{On the coherence of expected shortfall}, + Journal of Banking \& Finance 26(7), 1487--1503, 2002. +\bibitem{LoboFazelBoyd2007} + M.\,S.~Lobo, M.~Fazel and S.~Boyd, + \emph{Portfolio optimization with linear and fixed transaction costs}, + Annals of Operations Research 152, 341--365, 2007. +\bibitem{KellererMansiniSperanza2000} + H.~Kellerer, R.~Mansini and M.\,G.~Speranza, + \emph{Selecting portfolios with fixed costs and minimum transaction lots}, + Annals of Operations Research 99, 287--304, 2000. +\bibitem{HuangfuHall2018} + Q.~Huangfu and J.\,A.\,J.~Hall, + \emph{Parallelizing the dual revised simplex method}, + Mathematical Programming Computation 10, 119--142, 2018. +\bibitem{ShapiroDentchevaRuszczynski2009} + A.~Shapiro, D.~Dentcheva and A.~Ruszczy\'nski, + \emph{Lectures on Stochastic Programming: Modeling and Theory}, + SIAM, Philadelphia, 2009. +\bibitem{Michaud1989} + R.\,O.~Michaud, + \emph{The Markowitz optimization enigma: is `optimized' optimal?}, + Financial Analysts Journal 45(1), 31--42, 1989. +\bibitem{Owen1998} + A.\,B.~Owen, + \emph{Scrambling Sobol' and Niederreiter--Xing points}, + Journal of Complexity 14(4), 466--489, 1998. +\bibitem{Glasserman2004} + P.~Glasserman, + \emph{Monte Carlo Methods in Financial Engineering}, + Springer, New York, 2004. +\bibitem{HoylandWallace2001} + K.~H\o yland and S.\,W.~Wallace, + \emph{Generating scenario trees for multistage decision problems}, + Management Science 47(2), 295--307, 2001. +\bibitem{GlassermanHeidelbergerShahabuddin2000} + P.~Glasserman, P.~Heidelberger and P.~Shahabuddin, + \emph{Variance reduction techniques for estimating value-at-risk}, + Management Science 46(10), 1349--1364, 2000. +\bibitem{Hesterberg1995} + T.~Hesterberg, + \emph{Weighted average importance sampling and defensive mixture distributions}, + Technometrics 37(2), 185--194, 1995. +\bibitem{KunziBayMayer2006} + A.~K\"unzi-Bay and J.~Mayer, + \emph{Computational aspects of minimizing conditional value-at-risk}, + Computational Management Science 3(1), 3--27, 2006. +\bibitem{EmbrechtsMcNeilStraumann2002} + P.~Embrechts, A.\,J.~McNeil and D.~Straumann, + \emph{Correlation and dependence in risk management: properties and pitfalls}, + in: M.~Dempster (ed.), Risk Management: Value at Risk and Beyond, Cambridge University Press, + 176--223, 2002. +\bibitem{Kelley1960} + J.\,E.~Kelley, + \emph{The cutting-plane method for solving convex programs}, + Journal of the SIAM 8(4), 703--712, 1960. +\bibitem{DuranGrossmann1986} + M.\,A.~Duran and I.\,E.~Grossmann, + \emph{An outer-approximation algorithm for a class of mixed-integer nonlinear programs}, + Mathematical Programming 36, 307--339, 1986. +\bibitem{MansiniOgryczakSperanza2014} + R.~Mansini, W.~Ogryczak and M.\,G.~Speranza, + \emph{Twenty years of linear programming based portfolio optimization}, + European Journal of Operational Research 234(2), 518--535, 2014. +\end{thebibliography} + +\end{document} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..92d0fdc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "quantcore" +version = "0.1.0" +description = "Deterministic quant core of the AI-Augmented Portfolio Intelligence Platform: data, risk/performance measurement, stock-price models, Black-Litterman expected returns and mean-CVaR MILP optimisation on NVIDIA cuOpt." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "MIT" } +authors = [{ name = "Florian Herzog" }] +dependencies = [ + "numpy>=1.26", + "pandas>=2.1", + "scipy>=1.11", + "pyarrow>=14", + "yfinance>=0.2.50", + "requests>=2.31", + "lxml>=4.9", + "pydantic>=2.5", + "pydantic-settings>=2.1", + "fastapi>=0.109", + "uvicorn[standard]>=0.27", + "msgpack>=1.0", +] + +[project.optional-dependencies] +dev = ["pytest>=7.4", "pytest-cov", "httpx>=0.25", "ruff"] + +[project.scripts] +quantcore = "quantcore.cli:main" +quantcore-api = "quantcore.apicli:main" + +[tool.setuptools.packages.find] +include = ["quantcore*"] + +[tool.setuptools.package-data] +"quantcore.data" = ["static/*.csv", "static/*.json"] +"quantcore.api" = ["ui/*.html", "ui/*.css", "ui/js/*.js", "ui/js/tabs/*.js"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" +filterwarnings = ["ignore::DeprecationWarning", "ignore::FutureWarning"] + +[tool.ruff] +line-length = 110 +target-version = "py311" diff --git a/quantcore/__init__.py b/quantcore/__init__.py new file mode 100644 index 0000000..317908c --- /dev/null +++ b/quantcore/__init__.py @@ -0,0 +1,24 @@ +"""quantcore -- the deterministic quant core of the AI-Augmented Portfolio Intelligence Platform. + +The package is organised as a set of *services* with plain-Python contracts: + +* :mod:`quantcore.universe` -- ``SetupService``: builds the investment universe once + (largest 500 US + 500 European stocks, metadata, first price date, market capitalisation). +* :mod:`quantcore.data` -- ``DataDownloadService`` and ``DataStore``: Yahoo Finance + prices and FX rates, incremental updates, snapshots, resilient HTTP access. +* :mod:`quantcore.measurement` -- ``RiskService`` / ``PerformanceService``: log-returns, + moving-window and exponentially weighted volatility, correlation, VaR and CVaR, performance. +* :mod:`quantcore.models` -- covariance estimators, market index, stock-price simulation + (GBM and empirical innovations), Black-Litterman with Idzorek confidence views. +* :mod:`quantcore.optimization` -- mean-CVaR MILP in integer share numbers solved on NVIDIA + cuOpt (with a local HiGHS fallback), in "new portfolio" and "rebalancing" mode. +* :mod:`quantcore.api` -- FastAPI application exposing the services over HTTP/JSON. + +Everything in here is deterministic: same input, same output. No LLM is called anywhere in +this package -- the AI layer of the platform consumes these services through their contracts. +""" + +from quantcore.config import Settings, get_settings + +__all__ = ["Settings", "get_settings", "__version__"] +__version__ = "0.1.0" diff --git a/quantcore/api/__init__.py b/quantcore/api/__init__.py new file mode 100644 index 0000000..eb58ebd --- /dev/null +++ b/quantcore/api/__init__.py @@ -0,0 +1 @@ +"""HTTP/JSON API (FastAPI) exposing the deterministic services.""" diff --git a/quantcore/api/app.py b/quantcore/api/app.py new file mode 100644 index 0000000..4cadca8 --- /dev/null +++ b/quantcore/api/app.py @@ -0,0 +1,556 @@ +"""FastAPI application -- the HTTP/JSON surface of the quant core. + +Run with ``quantcore serve`` (or ``uvicorn quantcore.api.app:app``). Interactive docs at +``/docs``. Long-running work (setup, data update) runs as background jobs; everything else +answers synchronously. All numbers come from the deterministic services -- the API adds no +logic of its own. +""" + +from __future__ import annotations + +import json +import logging +import threading +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from fastapi import BackgroundTasks, FastAPI, HTTPException, Query +from fastapi.responses import RedirectResponse +from fastapi.staticfiles import StaticFiles + +from quantcore import __version__ +from quantcore.api import schemas as S +from quantcore.config import Settings, get_settings +from quantcore.data.download_service import DataDownloadService +from quantcore.data.market_data import MarketDataService +from quantcore.data.store import DataStore +from quantcore.measurement.performance import PerformanceService, ewm_performance, rolling_performance +from quantcore.measurement.returns import horizon_periods, periods_per_year +from quantcore.measurement.risk import RiskService +from quantcore.models.covariance import CovarianceModel +from quantcore.optimization.service import OptimizationRequest, OptimizationService +from quantcore.optimization.solvers import CuOptSolver +from quantcore.universe.setup_service import SetupService + +log = logging.getLogger(__name__) + +app = FastAPI( + title="quantcore -- Portfolio Risk Measurement & Optimisation", + version=__version__, + description=( + "Deterministic services of the AI-Augmented Portfolio Intelligence Platform: market data, " + "risk and performance measurement, stock-price models, Black-Litterman expected returns and " + "mean-CVaR MILP optimisation (NVIDIA cuOpt)." + ), +) + + +# ------------------------------------------------------------------------------------------------ +# state +# ------------------------------------------------------------------------------------------------ +class State: + def __init__(self, settings: Settings | None = None): + self.settings = settings or get_settings() + self.store = DataStore(self.settings) + self._md: MarketDataService | None = None + self.jobs: dict[str, dict] = {} + self.lock = threading.Lock() + + @property + def md(self) -> MarketDataService: + if self._md is None: + if not self.store.has_prices(): + raise HTTPException(503, "no market data yet - run the setup job (POST /jobs/setup)") + self._md = MarketDataService(self.settings, self.store) + return self._md + + def reload(self) -> None: + self._md = None + + +state = State() + + +def _records(df: pd.DataFrame, index_name: str = "symbol") -> list[dict]: + out = df.copy() + out.index.name = index_name + out = out.reset_index() + return json.loads(out.to_json(orient="records", date_format="iso", double_precision=10)) + + +def _frame(df: pd.DataFrame) -> dict: + """Wide frame -> ``{"index": [...], "columns": [...], "data": [[...], ...]}`` (NaN -> null).""" + return { + "index": [i.isoformat() if hasattr(i, "isoformat") else str(i) for i in df.index], + "columns": [str(c) for c in df.columns], + "data": json.loads(json.dumps(df.values.tolist(), default=float)) if df.size else [], + } + + +def _clean(obj: Any) -> Any: + """Recursively turn numpy / NaN / timestamps into JSON-safe values.""" + if isinstance(obj, dict): + return {str(k): _clean(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_clean(v) for v in obj] + if isinstance(obj, (np.floating, float)): + return None if not np.isfinite(obj) else float(obj) + if isinstance(obj, (np.integer,)): + return int(obj) + if isinstance(obj, np.ndarray): + return _clean(obj.tolist()) + if isinstance(obj, (pd.Timestamp, datetime)): + return obj.isoformat() + if isinstance(obj, pd.Series): + return _clean(obj.to_dict()) + return obj + + +def _http(exc: Exception) -> HTTPException: + if isinstance(exc, (KeyError, ValueError)): + return HTTPException(422, str(exc)) + if isinstance(exc, FileNotFoundError): + return HTTPException(503, str(exc)) + log.exception("request failed") + return HTTPException(500, f"{type(exc).__name__}: {exc}") + + +# ------------------------------------------------------------------------------------------------ +# health, universe, data +# ------------------------------------------------------------------------------------------------ +@app.get("/health", tags=["system"]) +def health() -> dict: + return _clean({"status": "ok", "version": __version__, "store": state.store.status(), + "cuopt": CuOptSolver(state.settings).health(), "base_currency": state.settings.base_currency}) + + +@app.get("/universe", tags=["data"]) +def universe(region: str | None = None, sector: str | None = None, country: str | None = None) -> list[dict]: + try: + md = state.md + symbols = md.symbols(region, sector, country) + return _records(md.universe.loc[symbols]) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.get("/universe/{symbol}", tags=["data"]) +def universe_symbol(symbol: str) -> dict: + md = state.md + if symbol not in md.universe.index: + raise HTTPException(404, f"{symbol} not in the universe") + return _clean({"symbol": symbol, **md.universe.loc[symbol].to_dict()}) + + +@app.get("/prices", tags=["data"]) +def prices(symbols: str = Query(..., description="comma-separated symbols"), start: str | None = None, + end: str | None = None, currency: str = "base", freq: str = "daily") -> dict: + try: + px = state.md.prices(symbols.split(","), start, end, currency, freq) + return {"currency": currency, "freq": freq, **_frame(px)} + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.get("/returns", tags=["data"]) +def returns(symbols: str = Query(...), start: str | None = None, end: str | None = None, currency: str = "base", + freq: str = "daily") -> dict: + try: + r = state.md.returns(symbols.split(","), start, end, currency, freq) + return {"currency": currency, "freq": freq, "kind": "log_returns", **_frame(r)} + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.get("/data/symbols", tags=["data"]) +def data_symbols(region: str | None = None, sector: str | None = None, country: str | None = None, + search: str | None = None, limit: int = Query(200, ge=1, le=2000), + sort: str = Query("market_cap", pattern="^(market_cap|symbol|name|sector)$")) -> dict: + """The universe as one table (metadata, current market cap and price) plus the filter facets. + + One call feeds a stock picker: the matching rows and the counts per region, sector and country. + """ + try: + md = state.md + u = md.universe + caps = md.market_caps().reindex(u.index) + px = md.latest_prices(u.index.tolist(), currency="base").reindex(u.index) + table = pd.DataFrame({ + "name": u["name"], "region": u["region"], "country": u["country"], "sector": u["gics_sector"], + "industry": u["yahoo_industry"], "currency": u["listing_currency"], "exchange": u["exchange"], + "market_cap": caps, "market_weight": caps / caps.sum(), "price": px, + "price_local": u["last_price"], "first_price_date": u["first_price_date"].astype(str), + "price_coverage": u["price_coverage"], "index_weight": u["index_weight"], + }) + facets = {"regions": table["region"].value_counts().to_dict(), + "sectors": table["sector"].fillna("Unknown").value_counts().to_dict(), + "countries": table["country"].value_counts().to_dict()} + m = table + if region: + m = m[m["region"].str.lower() == region.lower()] + if sector: + m = m[m["sector"].fillna("Unknown").str.lower() == sector.lower()] + if country: + m = m[m["country"].str.lower() == country.lower()] + if search: + q = search.lower() + m = m[[q in str(i).lower() or q in str(n).lower() for i, n in zip(m.index, m["name"])]] + m = m.sort_values("market_cap", ascending=False) if sort == "market_cap" else m.sort_values(sort) + return _clean({"as_of": md.last_date(), "base_currency": state.settings.base_currency, + "n_universe": int(len(table)), "n_matched": int(len(m)), "facets": facets, + "symbols": _records(m.head(limit))}) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.get("/market/index", tags=["data"]) +def market_index(start: str | None = None, end: str | None = None, freq: str = "weekly") -> dict: + try: + r = state.md.cap_weighted_index(start=start, end=end, freq=freq) + ppy = periods_per_year(freq) + return _clean({"freq": freq, "mean_ann": float(r.mean() * ppy), "vol_ann": float(r.std() * np.sqrt(ppy)), + "n": int(len(r)), **_frame(r.to_frame())}) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.get("/market/weights", tags=["data"]) +def market_weights(max_assets: int | None = None) -> dict: + try: + w = state.md.market_weights().sort_values(ascending=False) + if max_assets: + w = w.iloc[:max_assets] + return _clean({"as_of": state.md.last_date(), "weights": w.round(8).to_dict()}) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +# ------------------------------------------------------------------------------------------------ +# risk +# ------------------------------------------------------------------------------------------------ +@app.post("/risk/assets", tags=["risk"]) +def risk_assets(req: S.AssetRiskRequest) -> list[dict]: + try: + table = RiskService(state.md).asset_risk(req.symbols, req.freq, req.currency, req.window, req.lam, + tuple(req.confidence), req.horizon, req.start, req.end, + space=req.space, drift=req.drift) + return _records(table) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/risk/series", tags=["risk"]) +def risk_series(req: S.RiskSeriesRequest) -> dict: + try: + df = RiskService(state.md).risk_series(req.symbol, req.freq, req.currency, req.window, req.lam, + tuple(req.confidence), req.horizon, req.start, req.end) + return {"symbol": req.symbol, **_frame(df.dropna(how="all"))} + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/risk/covariance", tags=["risk"]) +def risk_covariance(req: S.CovarianceRequest) -> dict: + try: + cov = RiskService(state.md).covariance(req.symbols, req.freq, req.currency, req.method, req.window, req.lam, + req.shrinkage, req.start, req.end) + return {"method": req.method, "freq": req.freq, **_frame(cov)} + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/risk/correlation", tags=["risk"]) +def risk_correlation(req: S.CovarianceRequest) -> dict: + try: + corr = RiskService(state.md).correlation(req.symbols, req.freq, req.currency, req.method, req.window, req.lam, + req.start, req.end) + return {"method": req.method, "freq": req.freq, **_frame(corr)} + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/risk/portfolio", tags=["risk"]) +def risk_portfolio(req: S.PortfolioRiskRequest) -> dict: + try: + pr = RiskService(state.md).portfolio_risk(req.holdings, req.cash, req.freq, req.currency, req.horizon, + tuple(req.confidence), req.method, req.cov_method, req.window, + req.lam, req.shrinkage, req.start, req.end) + return _clean(pr.as_dict()) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +# ------------------------------------------------------------------------------------------------ +# performance +# ------------------------------------------------------------------------------------------------ +def _perf() -> PerformanceService: + return PerformanceService(state.md, state.settings.risk_free_rate) + + +@app.post("/performance/assets", tags=["performance"]) +def performance_assets(req: S.PerformanceRequest) -> list[dict]: + try: + return _records(_perf().asset_performance(req.symbols, req.freq, req.currency, req.start, req.end)) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/performance/rolling", tags=["performance"]) +def performance_rolling(req: S.RollingPerformanceRequest) -> dict: + try: + r = state.md.returns([req.symbol], req.start, req.end, req.currency, req.freq)[req.symbol].dropna() + rf = state.settings.risk_free_rate + df = ewm_performance(r, req.lam, req.freq, rf) if req.lam else rolling_performance(r, req.window, req.freq, rf) + return {"symbol": req.symbol, **_frame(df.dropna(how="all"))} + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/performance/portfolio", tags=["performance"]) +def performance_portfolio(req: S.PortfolioPerformanceRequest) -> dict: + try: + row = _perf().portfolio_performance(req.holdings, req.freq, req.currency, req.start, req.end, req.rebalanced) + return _clean(row.to_dict()) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +# ------------------------------------------------------------------------------------------------ +# models +# ------------------------------------------------------------------------------------------------ +@app.post("/model/expected-returns", tags=["models"]) +def expected_returns(req: S.ExpectedReturnsRequest) -> dict: + try: + md = state.md + svc = OptimizationService(state.settings, md) + oreq = OptimizationRequest(symbols=req.symbols, max_assets=req.max_assets, views=[v.as_dict() for v in req.views], + bl_tau=req.tau, bl_years=req.years, bl_omega_method=req.omega_method, cvar_limit=1.0) + symbols = svc.candidate_symbols(oreq) + # the very model the optimiser uses (arithmetic equilibrium return, same tau, same views) + mu_ann, desc, result = svc.expected_returns(oreq, symbols) + start = (md.last_date() - pd.DateOffset(years=req.years)).date().isoformat() + weekly = md.returns(symbols, start=start, currency="base", freq="weekly").dropna() + out = result.as_dict() + out["volatility_ann"] = (weekly.std() * np.sqrt(52)).round(6).to_dict() + out["expected_returns_ann"] = mu_ann.round(6).to_dict() + out["model"] = desc + out["omega_method"] = req.omega_method + return _clean(out) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/model/covariance", tags=["models"]) +def model_covariance(req: S.ModelCovarianceRequest) -> dict: + """Covariance and correlation of the estimator the scenario generator and the optimiser use. + + Unlike ``/risk/covariance`` (a measurement with explicit parameters) this takes the same + ``covariance`` model dict as ``/optimize``, scales it to the horizon and reports how well + conditioned the estimate is for the number of assets. + """ + try: + md = state.md + model = CovarianceModel.from_dict(req.covariance) + r = md.returns(req.symbols, start=req.start, end=req.end, currency="base", freq=req.freq).dropna() + cov = model.estimate(r) + h = horizon_periods(req.horizon, req.freq) + ppy = periods_per_year(req.freq) + sd = np.sqrt(np.diag(cov.values)) + corr = pd.DataFrame(cov.values / np.outer(sd, sd), index=cov.index, columns=cov.columns) + vals = np.linalg.eigvalsh(cov.values)[::-1] + off = corr.values[np.triu_indices(len(sd), 1)] + return _clean({ + "model": model.describe(), "shrinkage_applied": model.intensity(r), + "conditioning": model.conditioning(r), "observations": int(len(r)), + "freq": req.freq, "horizon_periods": h, "as_of": r.index[-1], + "volatility_ann": dict(zip(cov.index, sd * np.sqrt(ppy))), + "volatility_horizon": dict(zip(cov.index, sd * np.sqrt(h))), + "eigenvalues": (vals / vals.sum()).tolist(), + "correlation_summary": {"mean": float(off.mean()), "min": float(off.min()), "max": float(off.max())}, + "covariance": _frame(cov * h), "correlation": _frame(corr), + }) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +@app.post("/model/scenarios", tags=["models"]) +def scenarios(req: S.ScenarioRequest) -> dict: + try: + md = state.md + svc = OptimizationService(state.settings, md) + oreq = OptimizationRequest(symbols=req.symbols, expected_returns=req.expected_returns, cvar_limit=1.0, + horizon=req.horizon, freq=req.freq, covariance=req.covariance, seed=req.seed, + n_scenarios=req.n_scenarios, scenario_model=req.model, + innovation_history=req.innovation_history, sampling=req.sampling, + moment_match=req.moment_match, drift_convention=req.drift_convention, + importance_sampling=req.importance_sampling, bl_tau=req.bl_tau, + bl_years=req.bl_years, bl_omega_method=req.bl_omega_method, + views=[v.as_dict() for v in req.views]) + symbols = svc.candidate_symbols(oreq) + mu_ann, desc, _ = svc.expected_returns(oreq, symbols) + scen = svc.scenarios(oreq, symbols, mu_ann) + simple = scen.simple_returns() + out = {"description": scen.describe(), "expected_returns": desc, "scenario_key": scen.extra.get("key"), + "expected_returns_ann": _clean(mu_ann.reindex(symbols).to_dict()), + "summary": _records(scen.summary()), "horizon_periods": horizon_periods(req.horizon, req.freq)} + if req.focus: + out["asset_histograms"] = {} + for sym in req.focus: + if sym not in symbols: + continue + col = simple[:, symbols.index(sym)] + mass, edges = np.histogram(col, bins=req.bins, weights=scen.weights) + out["asset_histograms"][sym] = {"edges": edges.tolist(), "mass": mass.tolist()} + exposure, value = _exposure(req, symbols) + if exposure is not None: + out["portfolio"] = { + "value": value, "cash": req.cash, "invested": float(exposure.sum()), + "exposure": {s: float(v) for s, v in zip(symbols, exposure) if v != 0}, + **scen.distribution(exposure, bins=req.bins, confidence=tuple(req.confidence)), + } + return _clean(out) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +def _exposure(req: S.ScenarioRequest, symbols: list[str]) -> tuple[np.ndarray | None, float]: + """Money exposure per candidate from ``holdings`` (shares) or ``weights`` (of ``value``).""" + if req.holdings: + px = state.md.latest_prices(symbols, currency="base").reindex(symbols) + v = np.array([float(req.holdings.get(s, 0.0)) for s in symbols]) * px.values + return v, float(v.sum() + req.cash) + if req.weights: + v = np.array([float(req.weights.get(s, 0.0)) for s in symbols]) * req.value + return v, float(req.value) + return None, 0.0 + + +# ------------------------------------------------------------------------------------------------ +# optimisation +# ------------------------------------------------------------------------------------------------ +@app.post("/optimize", tags=["optimisation"]) +def optimize(req: S.OptimizeRequest) -> dict: + try: + payload = req.model_dump() + payload["views"] = [v.as_dict() for v in req.views] + oreq = OptimizationRequest(**payload) + result = OptimizationService(state.settings, state.md).optimize(oreq) + return _clean(result.as_dict()) + except Exception as exc: # noqa: BLE001 + raise _http(exc) from exc + + +# ------------------------------------------------------------------------------------------------ +# jobs: setup and data update (background), snapshots +# ------------------------------------------------------------------------------------------------ +def _run_job(job_id: str, fn) -> None: + job = state.jobs[job_id] + job["status"], job["started_at"] = "running", datetime.now(timezone.utc).isoformat(timespec="seconds") + try: + job["result"] = _clean(fn()) + job["status"] = "done" + state.reload() + except Exception as exc: # noqa: BLE001 + log.exception("job %s failed", job_id) + job["status"], job["error"] = "failed", f"{type(exc).__name__}: {exc}" + job["finished_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + try: + state.settings.jobs_dir.mkdir(parents=True, exist_ok=True) + (state.settings.jobs_dir / f"{job_id}.json").write_text(json.dumps(job, indent=2, default=str)) + except OSError: + pass + + +def _submit(kind: str, fn, background: BackgroundTasks) -> dict: + with state.lock: + running = [j for j in state.jobs.values() if j["status"] in ("queued", "running") and j["kind"] in ("setup", "update")] + if running and kind in ("setup", "update"): + raise HTTPException(409, f"a {running[0]['kind']} job is already running ({running[0]['id']})") + job_id = uuid.uuid4().hex[:12] + state.jobs[job_id] = {"id": job_id, "kind": kind, "status": "queued", "started_at": None, "finished_at": None, + "result": None, "error": None} + background.add_task(_run_job, job_id, fn) + return state.jobs[job_id] + + +@app.post("/jobs/setup", tags=["jobs"], status_code=202) +def job_setup(background: BackgroundTasks, offline: bool = False, max_per_region: int | None = None) -> dict: + return _submit("setup", lambda: SetupService(state.settings, state.store).run(allow_web=not offline, + max_per_region=max_per_region).as_dict(), + background) + + +@app.post("/jobs/update", tags=["jobs"], status_code=202) +def job_update(background: BackgroundTasks, full: bool = False) -> dict: + return _submit("update", lambda: DataDownloadService(state.settings, state.store).update(full=full).as_dict(), background) + + +@app.get("/jobs", tags=["jobs"]) +def jobs() -> list[dict]: + return list(state.jobs.values()) + + +@app.get("/jobs/{job_id}", tags=["jobs"]) +def job(job_id: str) -> dict: + if job_id not in state.jobs: + raise HTTPException(404, "unknown job") + return state.jobs[job_id] + + +@app.post("/snapshots", tags=["jobs"], status_code=201) +def create_snapshot(req: S.SnapshotRequest) -> dict: + try: + return {"path": str(state.store.create_snapshot(req.name))} + except FileExistsError as exc: + raise HTTPException(409, f"snapshot exists: {exc}") from exc + + +@app.get("/snapshots", tags=["jobs"]) +def list_snapshots() -> list[str]: + return state.store.list_snapshots() + + +# ------------------------------------------------------------------------------------------------ +# the test bench (single-page UI served from this app) +# ------------------------------------------------------------------------------------------------ +UI_DIR = Path(__file__).with_name("ui") + + +class _NoCacheStatic(StaticFiles): + """Static files that must be revalidated on every request. + + The bench is edited while it is open, and without a ``Cache-Control`` header a browser applies + heuristic caching to the ES modules: it keeps serving yesterday's ``optimize.js`` and the reader + sees a fixed bug that is still there. ``no-cache`` does not mean "do not store" -- the ETag still + produces a 304 for an unchanged file, so revalidation costs a round trip and no body. + """ + + async def get_response(self, path: str, scope): # noqa: ANN001 (Starlette's own signature) + response = await super().get_response(path, scope) + response.headers["Cache-Control"] = "no-cache, must-revalidate" + return response + + +def mount_test_bench(application: FastAPI, settings: Settings) -> bool: + """Serve the static test bench at ``/ui/`` and redirect ``/`` to it; ``True`` when mounted. + + A deployment-time switch (``QUANTCORE_ENABLE_UI=false``, read once when this module is + imported): the bench is a client of this API, not part of it, so an installation whose users + build their own front end can ship the same image without it. ``/docs`` is unaffected. + """ + if not settings.enable_ui or not UI_DIR.is_dir(): + return False + application.mount("/ui", _NoCacheStatic(directory=UI_DIR, html=True), name="ui") + + @application.get("/", include_in_schema=False) + def _root() -> RedirectResponse: + return RedirectResponse("/ui/") + + return True + + +UI_ENABLED = mount_test_bench(app, state.settings) diff --git a/quantcore/api/schemas.py b/quantcore/api/schemas.py new file mode 100644 index 0000000..8dd3a05 --- /dev/null +++ b/quantcore/api/schemas.py @@ -0,0 +1,233 @@ +"""Pydantic request/response models of the HTTP API (the *contracts* of the services).""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class SymbolsRequest(BaseModel): + symbols: list[str] = Field(..., min_length=1, description="Yahoo symbols from the universe") + freq: str = Field("daily", pattern="^(daily|weekly|monthly)$") + currency: str = Field("base", pattern="^(base|local)$") + start: str | None = None + end: str | None = None + + +class AssetRiskRequest(SymbolsRequest): + window: int = Field(200, ge=10, description="moving-window length in periods") + lam: float = Field(0.94, gt=0, lt=1, description="EWM decay") + confidence: list[float] = Field(default=[0.95, 0.99]) + horizon: str | int = Field("day", description="'day' | 'week' | 'month' | integer periods") + drift: str = Field("estimated", pattern="^(estimated|zero)$", + description="'estimated': the window's mean enters the parametric VaR/CVaR; 'zero': " + "volatility only (the usual market-risk convention, and the only sane one " + "for long horizons, where the mean is the noisiest input)") + space: str = Field("log", pattern="^(log|value)$", + description="'log': VaR/CVaR as log-returns (the modelling space); 'value': as a " + "fraction of the position's value, 1 - exp(-x), which cannot reach 100 %") + + +class RiskSeriesRequest(BaseModel): + symbol: str + freq: str = Field("daily", pattern="^(daily|weekly|monthly)$") + currency: str = Field("base", pattern="^(base|local)$") + window: int = Field(200, ge=10) + lam: float = Field(0.94, gt=0, lt=1) + confidence: list[float] = Field(default=[0.95, 0.99]) + horizon: str | int = "day" + start: str | None = None + end: str | None = None + + +class CovarianceRequest(SymbolsRequest): + method: str = Field("rolling", pattern="^(rolling|ewm)$") + window: int = Field(200, ge=10) + lam: float = Field(0.94, gt=0, lt=1) + shrinkage: float = Field(0.0, ge=0, le=1) + + +class PortfolioRiskRequest(BaseModel): + holdings: dict[str, float] = Field(..., description="number of shares per symbol") + cash: float = 0.0 + freq: str = Field("daily", pattern="^(daily|weekly|monthly)$") + currency: str = Field("base", pattern="^(base|local)$") + horizon: str | int = "week" + confidence: list[float] = Field(default=[0.95, 0.99]) + method: str = Field("normal", pattern="^(normal|historical)$") + cov_method: str = Field("rolling", pattern="^(rolling|ewm)$") + window: int = Field(200, ge=10) + lam: float = Field(0.94, gt=0, lt=1) + shrinkage: float = Field(0.0, ge=0, le=1) + start: str | None = None + end: str | None = None + + +class PerformanceRequest(SymbolsRequest): + pass + + +class RollingPerformanceRequest(BaseModel): + symbol: str + window: int = Field(252, ge=10) + lam: float | None = Field(None, gt=0, lt=1, description="if given: exponentially weighted instead of rolling") + freq: str = Field("daily", pattern="^(daily|weekly|monthly)$") + currency: str = Field("base", pattern="^(base|local)$") + start: str | None = None + end: str | None = None + + +class PortfolioPerformanceRequest(BaseModel): + holdings: dict[str, float] + freq: str = Field("daily", pattern="^(daily|weekly|monthly)$") + currency: str = Field("base", pattern="^(base|local)$") + start: str | None = None + end: str | None = None + rebalanced: bool = True + + +class ViewSpec(BaseModel): + """Either an explicit view (``assets`` + ``q``) or a sentiment view (``symbol`` + ``sentiment``).""" + + assets: dict[str, float] | None = None + q: float | None = Field(None, description="annual expected return of the view portfolio") + symbol: str | None = None + sentiment: float | None = Field(None, ge=-1, le=1) + strength: float = Field(0.25, ge=0) + confidence: float = Field(0.5, gt=0, le=1) + label: str = "" + source: str = "" + + def as_dict(self) -> dict: + d = self.model_dump(exclude_none=True) + return d + + +class ExpectedReturnsRequest(BaseModel): + symbols: list[str] | None = None + max_assets: int = 100 + views: list[ViewSpec] = Field(default_factory=list) + tau: float = Field(0.05, gt=0) + years: int = Field(10, ge=1) + omega_method: str = Field("idzorek", pattern="^(idzorek|he_litterman)$") + + +class ModelCovarianceRequest(BaseModel): + """Covariance as the *models* see it (the estimator of the scenario generator / optimiser).""" + + symbols: list[str] = Field(..., min_length=1) + freq: str = Field("daily", pattern="^(daily|weekly)$") + covariance: dict = Field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) + horizon: str | int = "week" + start: str | None = None + end: str | None = None + + +class ScenarioRequest(BaseModel): + symbols: list[str] = Field(..., min_length=1) + model: str = Field("empirical", pattern="^(gbm|empirical)$") + n_scenarios: int = Field(4096, ge=100, le=200_000, description="rounded up to a power of two for Sobol' sampling") + horizon: str | int = "week" + freq: str = Field("daily", pattern="^(daily|weekly)$") + covariance: dict = Field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) + seed: int = 42 + sampling: str = Field("sobol", pattern="^(sobol|mc|antithetic|lhs|balanced)$") + moment_match: bool = Field(True, description="exact scenario mean and covariance") + drift_convention: str = Field("arithmetic", pattern="^(arithmetic|log)$") + expected_returns: str = Field("zero", pattern="^(zero|historical|black_litterman)$") + innovation_history: int | None = 2500 + importance_sampling: str = Field("auto", pattern="^(auto|market|adaptive|off)$") + views: list["ViewSpec"] = Field(default_factory=list) + bl_tau: float = Field(0.05, gt=0) + bl_years: int = Field(10, ge=1) + bl_omega_method: str = Field("idzorek", pattern="^(idzorek|he_litterman)$") + # the portfolio whose P&L distribution is reported (shares, or weights of `value`) + holdings: dict[str, float] = Field(default_factory=dict) + weights: dict[str, float] = Field(default_factory=dict) + value: float = Field(1_000_000.0, gt=0, description="portfolio value when `weights` are given") + cash: float = Field(0.0, ge=0) + confidence: list[float] = Field(default=[0.95, 0.99]) + bins: int = Field(60, ge=5, le=400) + focus: list[str] = Field(default_factory=list, description="assets to return a histogram for") + + +class ConstraintsSpec(BaseModel): + max_weight: float = Field(0.10, gt=0, le=1) + max_sector_weight: float | dict[str, float] = 0.25 + max_positions: int | None = Field(None, ge=1) + min_position_value: float = Field(0.0, ge=0) + max_turnover: float | None = Field(None, ge=0) + max_transaction_cost: float | None = Field(None, ge=0) + min_invested: float = Field(0.0, ge=0, le=1) + max_invested: float = Field(1.0, gt=0, le=1) + + +class CostsSpec(BaseModel): + proportional_buy: float = Field(0.001, ge=0) + proportional_sell: float = Field(0.001, ge=0) + fixed_per_trade: float = Field(0.0, ge=0) + + +class OptimizeRequest(BaseModel): + mode: str = Field("new", pattern="^(new|rebalance)$") + cash: float = Field(1_000_000.0, ge=0) + holdings: dict[str, int] = Field(default_factory=dict) + symbols: list[str] | None = None + max_assets: int = Field(100, ge=2) + horizon: str | int = "week" + freq: str = Field("daily", pattern="^(daily|weekly)$") + confidence: float = Field(0.95, gt=0.5, lt=1) + cvar_limit: float | None = Field(None, gt=0, description="max expected shortfall in CHF over the horizon") + cvar_limit_fraction: float | None = Field(None, gt=0, lt=1) + objective: str = Field("max_return", pattern="^(max_return|min_cvar|utility)$") + risk_aversion: float = Field(1.0, ge=0) + target_return: float | None = None + scenario_model: str = Field("empirical", pattern="^(gbm|empirical|gaussian_exact)$", + description="'gaussian_exact': no scenarios, outer approximation of the normal model") + n_scenarios: int | None = Field(None, ge=100, le=200_000, + description="default: 16 per candidate, within [4096, 16384]; rounded up to a " + "power of two for Sobol' sampling. About 64 per candidate is " + "needed to pin the composition; the result warns below 8") + seed: int = 42 + sampling: str = Field("sobol", pattern="^(sobol|mc|antithetic|lhs|balanced)$") + moment_match: bool = Field(True, description="exact scenario mean and covariance") + importance_sampling: str = Field("auto", pattern="^(auto|market|adaptive|off)$", + description="'market': tilt towards the tail of the cap-weighted candidate " + "portfolio (independent of the risk parameters, so a family of runs " + "shares one scenario set); 'adaptive': tilt towards a first-pass " + "solution; 'auto': 'market' when the tail would be thin") + is_mixture: float = Field(0.25, gt=0, lt=0.5, description="share of scenarios per tilted block") + drift_convention: str = Field("arithmetic", pattern="^(arithmetic|log)$") + risk_drift: str = Field("model", pattern="^(model|zero)$", + description="'model': the CVaR is measured on the scenarios including the expected " + "return; 'zero': on the centred scenarios, so a high return forecast " + "cannot relax the risk limit (the usual market-risk convention)") + row_generation: str = Field("auto", pattern="^(auto|on|off)$", + description="solve on the tail rows only (large scenario sets)") + covariance: dict = Field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) + expected_returns: str = Field("black_litterman", pattern="^(black_litterman|historical|zero)$") + views: list[ViewSpec] = Field(default_factory=list) + bl_tau: float = Field(0.05, gt=0) + bl_years: int = Field(10, ge=1) + bl_omega_method: str = Field("idzorek", pattern="^(idzorek|he_litterman)$") + constraints: ConstraintsSpec = Field(default_factory=ConstraintsSpec) + costs: CostsSpec = Field(default_factory=CostsSpec) + solver: str = Field("auto", pattern="^(auto|cuopt|highs)$") + time_limit: float | None = Field(None, gt=0) + innovation_history: int | None = 2500 + innovation_standardize: str | None = Field("diagonal", pattern="^(diagonal|full)$") + + +class JobStatus(BaseModel): + id: str + kind: str + status: str # queued | running | done | failed + started_at: str | None = None + finished_at: str | None = None + result: Any | None = None + error: str | None = None + + +class SnapshotRequest(BaseModel): + name: str | None = None diff --git a/quantcore/api/ui/index.html b/quantcore/api/ui/index.html new file mode 100644 index 0000000..b0580b2 --- /dev/null +++ b/quantcore/api/ui/index.html @@ -0,0 +1,27 @@ + + + + + +quantcore test bench + + + + + +
+
+
quantcoretest bench
+
checking …
+ +
+ +
+ +
+
+
+
+ + + diff --git a/quantcore/api/ui/js/charts.js b/quantcore/api/ui/js/charts.js new file mode 100644 index 0000000..232fd9d --- /dev/null +++ b/quantcore/api/ui/js/charts.js @@ -0,0 +1,508 @@ +/* Small SVG chart library: line, bar, histogram, heatmap, scatter. + No dependencies (the container may have no internet). Colours come from the CSS custom + properties in styles.css, so the theme switch is a re-render, never a second palette. + Every chart keeps its config on the node and re-draws on resize and on a theme change. */ + +const NS = "http://www.w3.org/2000/svg"; +const PLOT = { left: 58, right: 16, top: 10, bottom: 26 }; + +export function svgEl(name, attrs = {}) { + const e = document.createElementNS(NS, name); + for (const [k, v] of Object.entries(attrs)) if (v !== null && v !== undefined) e.setAttribute(k, String(v)); + return e; +} + +export function cssVar(name) { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim() || "#888"; +} + +export function seriesColor(i) { return cssVar(`--series-${(i % 8) + 1}`); } + +/* ------------------------------------------------------------------ scales and ticks */ +const lin = (d0, d1, r0, r1) => (v) => (d1 === d0 ? (r0 + r1) / 2 : r0 + ((v - d0) / (d1 - d0)) * (r1 - r0)); + +export function niceTicks(lo, hi, count = 5) { + if (!isFinite(lo) || !isFinite(hi)) return [0]; + if (lo === hi) return [lo]; + const span = (hi - lo) / count; + const mag = Math.pow(10, Math.floor(Math.log10(Math.abs(span)))); + const step = [1, 2, 2.5, 5, 10].map((m) => m * mag).find((s) => s >= span) || 10 * mag; + const out = []; + for (let t = Math.ceil(lo / step) * step; t <= hi + step * 1e-6; t += step) out.push(Math.abs(t) < step * 1e-9 ? 0 : t); + return out; +} + +function padDomain([lo, hi], pad = 0.06, includeZero = false) { + if (includeZero) { lo = Math.min(lo, 0); hi = Math.max(hi, 0); } + if (lo === hi) { const d = Math.abs(lo) || 1; return [lo - d * 0.1, hi + d * 0.1]; } + const p = (hi - lo) * pad; + return [lo - p, hi + p]; +} + +const extent = (vals) => vals.reduce(([a, b], v) => [Math.min(a, v), Math.max(b, v)], [Infinity, -Infinity]); + +/* ------------------------------------------------------------------ tooltip (singleton) */ +function tipNode() { + let t = document.getElementById("tooltip"); + if (!t) { t = document.createElement("div"); t.id = "tooltip"; document.body.appendChild(t); } + return t; +} + +export function showTip(ev, title, rows) { + const t = tipNode(); + t.replaceChildren(); + if (title) { const h = document.createElement("div"); h.className = "tt-title"; h.textContent = title; t.appendChild(h); } + for (const r of rows) { + const row = document.createElement("div"); row.className = "tt-row"; + if (r.color) { const k = document.createElement("span"); k.className = "key"; k.style.background = r.color; row.appendChild(k); } + const nm = document.createElement("span"); nm.className = "nm"; nm.textContent = r.name ?? ""; row.appendChild(nm); + const v = document.createElement("span"); v.className = "v"; v.textContent = r.value ?? ""; row.appendChild(v); + t.appendChild(row); + } + t.style.opacity = "1"; + const pad = 14, w = t.offsetWidth, h = t.offsetHeight; + let x = ev.clientX + pad, y = ev.clientY + pad; + if (x + w > window.innerWidth - 8) x = ev.clientX - w - pad; + if (y + h > window.innerHeight - 8) y = ev.clientY - h - pad; + t.style.left = `${Math.max(4, x)}px`; t.style.top = `${Math.max(4, y)}px`; +} + +export function hideTip() { const t = document.getElementById("tooltip"); if (t) t.style.opacity = "0"; } + +/* ------------------------------------------------------------------ chart frame */ +function frame(node, cfg) { + const height = cfg.height || 220; + const margin = { ...PLOT, ...(cfg.margin || {}) }; + const width = Math.max(260, node.clientWidth || node.parentElement?.clientWidth || 640); + const w = width - margin.left - margin.right; + const h = height - margin.top - margin.bottom; + const svg = svgEl("svg", { viewBox: `0 0 ${width} ${height}`, width, height, role: "img" }); + const g = svgEl("g", { transform: `translate(${margin.left},${margin.top})` }); + svg.appendChild(g); + return { svg, g, w, h, width, height, margin, grid: cssVar("--border"), ink: cssVar("--text-secondary"), + muted: cssVar("--text-muted"), surface: cssVar("--surface-1") }; +} + +function yAxis(f, y, ticks, format) { + for (const t of ticks) { + const yy = y(t); + f.g.appendChild(svgEl("line", { x1: 0, x2: f.w, y1: yy, y2: yy, stroke: f.grid, "stroke-width": 1 })); + const lab = svgEl("text", { x: -8, y: yy + 3.5, "text-anchor": "end", fill: f.muted, "font-size": 10.5 }); + lab.textContent = format(t); + f.g.appendChild(lab); + } +} + +function xLabels(f, entries) { // entries: [{x, text}] + for (const e of entries) { + const t = svgEl("text", { x: e.x, y: f.h + 15, "text-anchor": "middle", fill: f.muted, "font-size": 10.5 }); + t.textContent = e.text; + f.g.appendChild(t); + } +} + +function legendFor(node, items) { + const box = document.createElement("div"); + box.className = "legend"; + for (const it of items) { + const s = document.createElement("span"); s.className = "item"; + const k = document.createElement("span"); k.className = `key${it.rect ? " rect" : ""}`; k.style.background = it.color; + const n = document.createElement("span"); n.textContent = it.name; + s.append(k, n); box.appendChild(s); + } + node.appendChild(box); +} + +/** Render `fn` into `node` now, and again on resize / theme change. */ +function mount(node, fn) { + node.__render = fn; + fn(); + if (!node.__observed) { + node.__observed = true; + const ro = new ResizeObserver(() => { if (node.isConnected) requestAnimationFrame(() => node.__render?.()); }); + ro.observe(node); + } +} + +export function redrawAll(root = document) { + root.querySelectorAll(".chart").forEach((n) => n.__render?.()); +} + +/* ------------------------------------------------------------------ line chart */ +/** series: [{name, points: [[x, y], ...], color?}] ; xType 'time' | 'linear' */ +export function lineChart(node, cfg) { + mount(node, () => { + node.replaceChildren(); + const series = (cfg.series || []).filter((s) => s.points?.length); + if (!series.length) { node.appendChild(emptyBox("no data")); return; } + const f = frame(node, cfg); + const xs = series.flatMap((s) => s.points.map((p) => p[0])); + const ys = series.flatMap((s) => s.points.map((p) => p[1])).filter(Number.isFinite); + const [x0, x1] = extent(xs); + const [y0, y1] = padDomain(extent(ys), 0.08, cfg.includeZero); + const x = lin(x0, x1, 0, f.w), y = lin(y0, y1, f.h, 0); + const yFmt = cfg.yFormat || ((v) => v.toFixed(2)); + yAxis(f, y, niceTicks(y0, y1, cfg.yTicks || 5), yFmt); + if (cfg.zeroLine && y0 < 0 && y1 > 0) { + f.g.appendChild(svgEl("line", { x1: 0, x2: f.w, y1: y(0), y2: y(0), stroke: cssVar("--border-strong"), "stroke-width": 1 })); + } + // x labels: ~6 evenly spaced actual data positions + const xFmt = cfg.xFormat || ((v) => String(Math.round(v))); + const uniq = [...new Set(xs)].sort((a, b) => a - b); + const n = Math.min(6, uniq.length); + const picks = Array.from({ length: n }, (_, i) => uniq[Math.round((i * (uniq.length - 1)) / Math.max(n - 1, 1))]); + xLabels(f, picks.map((v) => ({ x: x(v), text: xFmt(v) }))); + + series.forEach((s, i) => { + const color = s.color || seriesColor(i); + const d = s.points.filter((p) => Number.isFinite(p[1])) + .map((p, j) => `${j ? "L" : "M"}${x(p[0]).toFixed(2)},${y(p[1]).toFixed(2)}`).join(" "); + if (cfg.area) { + const pts = s.points.filter((p) => Number.isFinite(p[1])); + const base = y(Math.max(y0, Math.min(0, y1))); + const area = `${d} L${x(pts[pts.length - 1][0]).toFixed(2)},${base} L${x(pts[0][0]).toFixed(2)},${base} Z`; + f.g.appendChild(svgEl("path", { d: area, fill: color, "fill-opacity": 0.1, stroke: "none" })); + } + f.g.appendChild(svgEl("path", { d, fill: "none", stroke: color, "stroke-width": 2, + "stroke-linejoin": "round", "stroke-linecap": "round" })); + }); + + // crosshair + one tooltip listing every series + const hair = svgEl("line", { y1: 0, y2: f.h, stroke: cssVar("--border-strong"), "stroke-width": 1, opacity: 0 }); + f.g.appendChild(hair); + const dots = series.map((s, i) => { + const c = svgEl("circle", { r: 4, fill: s.color || seriesColor(i), stroke: f.surface, "stroke-width": 2, opacity: 0 }); + f.g.appendChild(c); return c; + }); + const overlay = svgEl("rect", { x: 0, y: 0, width: f.w, height: f.h, fill: "transparent" }); + f.g.appendChild(overlay); + const onMove = (ev) => { + const box = f.svg.getBoundingClientRect(); + const px = ((ev.clientX - box.left) / box.width) * f.width - f.margin.left; + const target = x0 + (px / f.w) * (x1 - x0); + let best = uniq[0]; + for (const v of uniq) if (Math.abs(v - target) < Math.abs(best - target)) best = v; + hair.setAttribute("x1", x(best)); hair.setAttribute("x2", x(best)); hair.setAttribute("opacity", 1); + const rows = []; + series.forEach((s, i) => { + const p = s.points.find((q) => q[0] === best); + dots[i].setAttribute("opacity", p && Number.isFinite(p[1]) ? 1 : 0); + if (p && Number.isFinite(p[1])) { + dots[i].setAttribute("cx", x(p[0])); dots[i].setAttribute("cy", y(p[1])); + rows.push({ color: s.color || seriesColor(i), name: s.name, value: yFmt(p[1]) }); + } + }); + showTip(ev, xFmt(best), rows); + }; + overlay.addEventListener("pointermove", onMove); + overlay.addEventListener("pointerleave", () => { + hair.setAttribute("opacity", 0); dots.forEach((d) => d.setAttribute("opacity", 0)); hideTip(); + }); + node.appendChild(f.svg); + if (series.length > 1) legendFor(node, series.map((s, i) => ({ name: s.name, color: s.color || seriesColor(i) }))); + }); +} + +/* ------------------------------------------------------------------ bar chart */ +/** items: [{label, values: [v, ...], meta?}] ; groups: [name, ...] (one entry = single series) */ +export function barChart(node, cfg) { + mount(node, () => { + node.replaceChildren(); + const items = cfg.items || []; + if (!items.length) { node.appendChild(emptyBox(cfg.emptyText || "nothing to show")); return; } + const groups = cfg.groups || [""]; + const horizontal = cfg.horizontal !== false; + const fmt = cfg.valueFormat || ((v) => v.toFixed(2)); + const all = items.flatMap((it) => it.values).filter(Number.isFinite); + const refs = (cfg.references || []).map((r) => r.value); + const dom = padDomain(extent([...all, ...refs]), 0.04, true); + const colorOf = (gi, v) => cfg.diverging ? (v < 0 ? cssVar("--div-neg") : cssVar("--div-pos")) + : (cfg.colors?.[gi] || seriesColor(gi)); + + if (horizontal) { + const rowH = Math.min(cfg.rowHeight || 26, 34); + const height = items.length * rowH + 34; + const f = frame(node, { ...cfg, height, margin: { ...PLOT, left: cfg.labelWidth || 92, bottom: 22, top: 6 } }); + const x = lin(dom[0], dom[1], 0, f.w); + const band = f.h / items.length; + const thick = Math.min(24, Math.max(4, (band - 8) / groups.length - (groups.length > 1 ? 2 : 0))); + for (const t of niceTicks(dom[0], dom[1], 5)) { + f.g.appendChild(svgEl("line", { x1: x(t), x2: x(t), y1: 0, y2: f.h, stroke: f.grid, "stroke-width": 1 })); + const lab = svgEl("text", { x: x(t), y: f.h + 14, "text-anchor": "middle", fill: f.muted, "font-size": 10.5 }); + lab.textContent = fmt(t); f.g.appendChild(lab); + } + items.forEach((it, i) => { + const y0 = i * band + (band - thick * groups.length - (groups.length - 1) * 2) / 2; + const lab = svgEl("text", { x: -8, y: i * band + band / 2 + 3.5, "text-anchor": "end", + fill: cssVar("--text-secondary"), "font-size": 11 }); + lab.textContent = it.label; f.g.appendChild(lab); + it.values.forEach((v, gi) => { + if (!Number.isFinite(v)) return; + const yy = y0 + gi * (thick + 2); + const zero = x(Math.max(dom[0], Math.min(0, dom[1]))); + const xv = x(v); + const r = svgEl("path", { d: roundedBar(Math.min(zero, xv), yy, Math.abs(xv - zero), thick, v >= 0 ? "right" : "left"), + fill: colorOf(gi, v) }); + r.style.cursor = "default"; + r.addEventListener("pointermove", (ev) => showTip(ev, it.label, + groups.map((g, k) => ({ color: colorOf(k, it.values[k]), name: g || (cfg.valueName || "value"), + value: Number.isFinite(it.values[k]) ? fmt(it.values[k]) : "-" })))); + r.addEventListener("pointerleave", hideTip); + f.g.appendChild(r); + if (cfg.valueLabels !== false && items.length <= 24 && groups.length === 1) { + const text = fmt(v); + const outside = v >= 0 ? xv + 5 : xv - 5; + const t = svgEl("text", { x: outside, y: yy + thick / 2 + 3.5, "text-anchor": v >= 0 ? "start" : "end", + fill: f.muted, "font-size": 10 }); + t.textContent = text; + if (outside + (v >= 0 ? text.length * 5.6 : -text.length * 5.6) <= f.w + 4 && + outside + (v >= 0 ? 0 : -text.length * 5.6) >= -4) f.g.appendChild(t); + } + }); + }); + for (const r of cfg.references || []) drawRef(f, x(r.value), r, true); + node.appendChild(f.svg); + } else { + const f = frame(node, { ...cfg, height: cfg.height || 220, margin: { ...PLOT, bottom: cfg.bottom || 46 } }); + const y = lin(dom[0], dom[1], f.h, 0); + const band = f.w / items.length; + const thick = Math.min(24, Math.max(3, (band - 6) / groups.length - (groups.length > 1 ? 2 : 0))); + yAxis(f, y, niceTicks(dom[0], dom[1], 5), fmt); + items.forEach((it, i) => { + const x0 = i * band + (band - thick * groups.length - (groups.length - 1) * 2) / 2; + it.values.forEach((v, gi) => { + if (!Number.isFinite(v)) return; + const zero = y(Math.max(dom[0], Math.min(0, dom[1]))); + const yv = y(v); + const r = svgEl("path", { d: roundedBar(x0 + gi * (thick + 2), Math.min(zero, yv), thick, Math.abs(yv - zero), + v >= 0 ? "up" : "down", true), fill: colorOf(gi, v) }); + r.addEventListener("pointermove", (ev) => showTip(ev, it.label, + groups.map((g, k) => ({ color: colorOf(k, it.values[k]), name: g || (cfg.valueName || "value"), + value: Number.isFinite(it.values[k]) ? fmt(it.values[k]) : "-" })))); + r.addEventListener("pointerleave", hideTip); + f.g.appendChild(r); + }); + if (items.length <= 30) { + const t = svgEl("text", { x: i * band + band / 2, y: f.h + 14, "text-anchor": "end", fill: f.muted, + "font-size": 10, transform: `rotate(-35 ${i * band + band / 2} ${f.h + 14})` }); + t.textContent = it.label; f.g.appendChild(t); + } + }); + node.appendChild(f.svg); + } + if (groups.length > 1) legendFor(node, groups.map((g, i) => ({ name: g, color: cfg.colors?.[i] || seriesColor(i), rect: true }))); + }); +} + +function roundedBar(x, y, w, h, dir, vertical = false) { + const r = Math.min(4, vertical ? w / 2 : h / 2, vertical ? h : w); + if (w <= 0.5 || h <= 0.5) return `M${x},${y}h${Math.max(w, 0.5)}v${Math.max(h, 0.5)}h${-Math.max(w, 0.5)}Z`; + if (!vertical) { + return dir === "right" + ? `M${x},${y} h${w - r} a${r},${r} 0 0 1 ${r},${r} v${h - 2 * r} a${r},${r} 0 0 1 ${-r},${r} h${-(w - r)} Z` + : `M${x + r},${y} h${w - r} v${h} h${-(w - r)} a${r},${r} 0 0 1 ${-r},${-r} v${-(h - 2 * r)} a${r},${r} 0 0 1 ${r},${-r} Z`; + } + return dir === "up" + ? `M${x},${y + h} v${-(h - r)} a${r},${r} 0 0 1 ${r},${-r} h${w - 2 * r} a${r},${r} 0 0 1 ${r},${r} v${h - r} Z` + : `M${x},${y} h${w} v${h - r} a${r},${r} 0 0 1 ${-r},${r} h${-(w - 2 * r)} a${r},${r} 0 0 1 ${-r},${-r} Z`; +} + +function drawRef(f, pos, ref, vertical) { + const line = vertical + ? svgEl("line", { x1: pos, x2: pos, y1: 0, y2: f.h, stroke: ref.color || cssVar("--text-muted"), "stroke-width": 1 }) + : svgEl("line", { x1: 0, x2: f.w, y1: pos, y2: pos, stroke: ref.color || cssVar("--text-muted"), "stroke-width": 1 }); + f.g.appendChild(line); + if (ref.label) { + const near = pos > f.w - Math.max(56, ref.label.length * 5.4); // keep the label inside the plot + const t = svgEl("text", { x: vertical ? (near ? pos - 4 : pos + 4) : f.w, y: vertical ? f.h - 4 : pos - 4, + "text-anchor": vertical ? (near ? "end" : "start") : "end", + fill: ref.color || cssVar("--text-muted"), "font-size": 10, + // a surface-coloured halo keeps it legible where it crosses a mark + stroke: f.surface, "stroke-width": 3, "paint-order": "stroke fill" }); + t.textContent = ref.label; f.g.appendChild(t); + } +} + +/* ------------------------------------------------------------------ histogram */ +/** cfg: {edges, mass, markers: [{value, label, color}], tailFrom, valueFormat} */ +export function histogram(node, cfg) { + mount(node, () => { + node.replaceChildren(); + const { edges, mass } = cfg; + if (!edges?.length || !mass?.length) { node.appendChild(emptyBox("no distribution")); return; } + const f = frame(node, { ...cfg, height: cfg.height || 240, margin: { ...PLOT, left: 52, bottom: 28 } }); + const x = lin(edges[0], edges[edges.length - 1], 0, f.w); + const ymax = Math.max(...mass); + const y = lin(0, ymax, f.h, 0); + const fmt = cfg.valueFormat || ((v) => v.toFixed(2)); + const pct = (v) => `${(v * 100).toFixed(1)} %`; + for (const t of niceTicks(0, ymax, 4)) { + f.g.appendChild(svgEl("line", { x1: 0, x2: f.w, y1: y(t), y2: y(t), stroke: f.grid, "stroke-width": 1 })); + const lab = svgEl("text", { x: -8, y: y(t) + 3.5, "text-anchor": "end", fill: f.muted, "font-size": 10.5 }); + lab.textContent = pct(t); f.g.appendChild(lab); + } + const main = cssVar("--series-1"), tail = cssVar("--series-2"); + const tailFrom = cfg.tailFrom; + mass.forEach((m, i) => { + const x0 = x(edges[i]), x1 = x(edges[i + 1]); + const gap = Math.min(2, (x1 - x0) * 0.3); + const w = Math.max(x1 - x0 - gap, 0.6); + const isTail = tailFrom !== undefined && edges[i + 1] <= tailFrom; + const h = f.h - y(m); + const bar = svgEl("path", { d: roundedBar(x0 + gap / 2, y(m), w, h, "up", true), fill: isTail ? tail : main }); + bar.addEventListener("pointermove", (ev) => showTip(ev, `${fmt(edges[i])} … ${fmt(edges[i + 1])}`, + [{ color: isTail ? tail : main, name: "probability", value: pct(m) }])); + bar.addEventListener("pointerleave", hideTip); + f.g.appendChild(bar); + }); + for (const t of niceTicks(edges[0], edges[edges.length - 1], 6)) { + const lab = svgEl("text", { x: x(t), y: f.h + 15, "text-anchor": "middle", fill: f.muted, "font-size": 10.5 }); + lab.textContent = fmt(t); f.g.appendChild(lab); + } + (cfg.markers || []).forEach((m, i) => { + const px = x(m.value); + f.g.appendChild(svgEl("line", { x1: px, x2: px, y1: 0, y2: f.h, stroke: m.color || cssVar("--text-primary"), + "stroke-width": 1 })); + const t = svgEl("text", { x: px + 3, y: 10 + i * 12, "text-anchor": px > f.w - 60 ? "end" : "start", + fill: m.color || cssVar("--text-secondary"), "font-size": 10 }); + t.setAttribute("x", px > f.w - 60 ? px - 3 : px + 3); + t.textContent = m.label; f.g.appendChild(t); + }); + node.appendChild(f.svg); + const items = [{ name: cfg.mainLabel || "outcomes", color: main, rect: true }]; + if (tailFrom !== undefined) items.push({ name: cfg.tailLabel || "beyond VaR", color: tail, rect: true }); + legendFor(node, items); + }); +} + +/* ------------------------------------------------------------------ heatmap */ +/** cfg: {labels, matrix, diverging (default true), domain, format} */ +export function heatmap(node, cfg) { + mount(node, () => { + node.replaceChildren(); + const labels = cfg.labels || [], m = cfg.matrix || []; + if (!labels.length) { node.appendChild(emptyBox("no matrix")); return; } + const n = labels.length; + const width = Math.max(260, node.clientWidth || 600); + const pad = Math.min(96, Math.max(44, width * 0.14)); + const cell = Math.max(6, Math.min(34, (width - pad - 66) / n)); + const size = cell * n; + const f = { svg: svgEl("svg", { viewBox: `0 0 ${width} ${size + pad + 30}`, width, height: size + pad + 30 }), + muted: cssVar("--text-muted") }; + const g = svgEl("g", { transform: `translate(${pad},${pad})` }); + f.svg.appendChild(g); + const dom = cfg.domain || (cfg.diverging === false ? [0, Math.max(...m.flat())] : [-1, 1]); + const fmt = cfg.format || ((v) => v.toFixed(2)); + const show = n <= 26; + for (let i = 0; i < n; i++) { + for (let j = 0; j < n; j++) { + const v = m[i]?.[j]; + if (!Number.isFinite(v)) continue; + const c = colorFor(v, dom, cfg.diverging !== false); + const r = svgEl("rect", { x: j * cell, y: i * cell, width: Math.max(cell - 1, 1), height: Math.max(cell - 1, 1), + fill: c, rx: Math.min(2, cell / 6) }); + r.addEventListener("pointermove", (ev) => showTip(ev, `${labels[i]} · ${labels[j]}`, + [{ color: c, name: cfg.valueName || "correlation", value: fmt(v) }])); + r.addEventListener("pointerleave", hideTip); + g.appendChild(r); + } + if (show) { + const ty = svgEl("text", { x: -6, y: i * cell + cell / 2 + 3, "text-anchor": "end", fill: f.muted, "font-size": 9.5 }); + ty.textContent = labels[i]; g.appendChild(ty); + const tx = svgEl("text", { x: i * cell + cell / 2, y: -6, "text-anchor": "start", fill: f.muted, "font-size": 9.5, + transform: `rotate(-60 ${i * cell + cell / 2} -6)` }); + tx.textContent = labels[i]; g.appendChild(tx); + } + } + // scale legend + const lw = Math.min(180, size), lx = pad, ly = pad + size + 18; + const grad = svgEl("linearGradient", { id: `hm-${Math.random().toString(36).slice(2)}` }); + for (let k = 0; k <= 10; k++) { + grad.appendChild(svgEl("stop", { offset: `${k * 10}%`, + "stop-color": colorFor(dom[0] + (k / 10) * (dom[1] - dom[0]), dom, cfg.diverging !== false) })); + } + const defs = svgEl("defs"); defs.appendChild(grad); f.svg.appendChild(defs); + f.svg.appendChild(svgEl("rect", { x: lx, y: ly, width: lw, height: 8, rx: 2, fill: `url(#${grad.id})` })); + [dom[0], (dom[0] + dom[1]) / 2, dom[1]].forEach((v, k) => { + const t = svgEl("text", { x: lx + (k * lw) / 2, y: ly + 20, "text-anchor": k === 0 ? "start" : k === 2 ? "end" : "middle", + fill: f.muted, "font-size": 10 }); + t.textContent = fmt(v); f.svg.appendChild(t); + }); + node.appendChild(f.svg); + }); +} + +function hex2rgb(h) { + const s = h.replace("#", "").trim(); + const v = s.length === 3 ? s.split("").map((c) => c + c).join("") : s; + return [parseInt(v.slice(0, 2), 16), parseInt(v.slice(2, 4), 16), parseInt(v.slice(4, 6), 16)]; +} +const mix = (a, b, t) => `rgb(${a.map((v, i) => Math.round(v + (b[i] - v) * t)).join(",")})`; + +export function colorFor(v, [lo, hi], diverging = true) { + if (diverging) { + const neg = hex2rgb(cssVar("--div-neg")), mid = hex2rgb(cssVar("--div-mid")), pos = hex2rgb(cssVar("--div-pos")); + const t = Math.max(-1, Math.min(1, v / Math.max(Math.abs(lo), Math.abs(hi) || 1))); + return t < 0 ? mix(mid, neg, -t) : mix(mid, pos, t); + } + const t = Math.max(0, Math.min(1, (v - lo) / ((hi - lo) || 1))); + const stops = ["--seq-100", "--seq-250", "--seq-400", "--seq-550", "--seq-700"].map((s) => hex2rgb(cssVar(s))); + const k = Math.min(stops.length - 2, Math.floor(t * (stops.length - 1))); + return mix(stops[k], stops[k + 1], t * (stops.length - 1) - k); +} + +/* ------------------------------------------------------------------ scatter */ +/** cfg: {points: [{x, y, label, group}], groups, xFormat, yFormat, xLabel, yLabel} */ +export function scatterChart(node, cfg) { + mount(node, () => { + node.replaceChildren(); + const pts = (cfg.points || []).filter((p) => Number.isFinite(p.x) && Number.isFinite(p.y)); + if (!pts.length) { node.appendChild(emptyBox("no points")); return; } + const f = frame(node, { ...cfg, height: cfg.height || 260, margin: { ...PLOT, bottom: 40, left: 62 } }); + const [x0, x1] = padDomain(extent(pts.map((p) => p.x)), 0.08); + const [y0, y1] = padDomain(extent(pts.map((p) => p.y)), 0.08); + const x = lin(x0, x1, 0, f.w), y = lin(y0, y1, f.h, 0); + const xFmt = cfg.xFormat || ((v) => v.toFixed(2)), yFmt = cfg.yFormat || ((v) => v.toFixed(2)); + yAxis(f, y, niceTicks(y0, y1, 5), yFmt); + for (const t of niceTicks(x0, x1, 5)) { + f.g.appendChild(svgEl("line", { x1: x(t), x2: x(t), y1: 0, y2: f.h, stroke: f.grid, "stroke-width": 1 })); + const lab = svgEl("text", { x: x(t), y: f.h + 15, "text-anchor": "middle", fill: f.muted, "font-size": 10.5 }); + lab.textContent = xFmt(t); f.g.appendChild(lab); + } + if (cfg.xLabel) { + const t = svgEl("text", { x: f.w / 2, y: f.h + 33, "text-anchor": "middle", fill: f.muted, "font-size": 10.5 }); + t.textContent = cfg.xLabel; f.g.appendChild(t); + } + if (cfg.yLabel) { + const t = svgEl("text", { x: -f.margin.left + 12, y: f.h / 2, "text-anchor": "middle", fill: f.muted, "font-size": 10.5, + transform: `rotate(-90 ${-f.margin.left + 12} ${f.h / 2})` }); + t.textContent = cfg.yLabel; f.g.appendChild(t); + } + const groups = cfg.groups || [...new Set(pts.map((p) => p.group ?? ""))]; + pts.forEach((p) => { + const gi = Math.max(0, groups.indexOf(p.group ?? "")); + const c = p.color || seriesColor(gi); + const dot = svgEl("circle", { cx: x(p.x), cy: y(p.y), r: 5, fill: c, stroke: f.surface, "stroke-width": 2 }); + const hit = svgEl("circle", { cx: x(p.x), cy: y(p.y), r: 12, fill: "transparent" }); + hit.addEventListener("pointermove", (ev) => showTip(ev, p.label, [ + { color: c, name: cfg.xLabel || "x", value: xFmt(p.x) }, { color: c, name: cfg.yLabel || "y", value: yFmt(p.y) }, + ])); + hit.addEventListener("pointerleave", hideTip); + f.g.append(dot, hit); + }); + if (cfg.labelPoints !== false && pts.length <= 14) { + pts.forEach((p) => { + const t = svgEl("text", { x: x(p.x) + 8, y: y(p.y) - 7, fill: f.muted, "font-size": 9.5 }); + t.textContent = p.label; f.g.appendChild(t); + }); + } + node.appendChild(f.svg); + if (groups.length > 1) legendFor(node, groups.map((g, i) => ({ name: g, color: seriesColor(i) }))); + }); +} + +function emptyBox(text) { + const d = document.createElement("div"); + d.className = "empty"; d.textContent = text; + return d; +} diff --git a/quantcore/api/ui/js/core.js b/quantcore/api/ui/js/core.js new file mode 100644 index 0000000..9414aec --- /dev/null +++ b/quantcore/api/ui/js/core.js @@ -0,0 +1,393 @@ +/* Shared state, API client, formatting and DOM helpers for the test bench. */ + +/* ------------------------------------------------------------------ API */ +const BASE = location.pathname.replace(/\/ui\/?.*$/, ""); + +async function request(path, { method = "GET", body, signal } = {}) { + const res = await fetch(BASE + path, { + method, + headers: body ? { "Content-Type": "application/json" } : undefined, + body: body ? JSON.stringify(body) : undefined, + signal, + }); + const text = await res.text(); + let data = null; + try { data = text ? JSON.parse(text) : null; } catch { data = text; } + if (!res.ok) { + const detail = data && typeof data === "object" ? data.detail ?? JSON.stringify(data) : String(data || res.statusText); + throw new Error(typeof detail === "string" ? detail : JSON.stringify(detail)); + } + return data; +} + +export const api = { + get: (path, params) => request(path + (params ? "?" + new URLSearchParams( + Object.entries(params).filter(([, v]) => v !== null && v !== undefined && v !== "")).toString() : "")), + post: (path, body, opts = {}) => request(path, { method: "POST", body, ...opts }), +}; + +/* ------------------------------------------------------------------ state */ +const KEY = "quantcore-bench-v2"; + +export const DEFAULT_STATE = { + ui: { tab: "stocks", theme: "auto", migrations: [] }, + universe: { region: "", sector: "", country: "", search: "", top: 30 }, + selection: [], + portfolio: { mode: "new", cash: 1000000, holdings: {} }, + model: { + freq: "daily", horizon: "week", scenario_model: "gbm", n_scenarios: 4096, seed: 42, + sampling: "sobol", moment_match: true, importance_sampling: "auto", drift_convention: "arithmetic", + innovation_history: 2500, + covariance: { method: "ewm", lam: 0.99, window: 200, shrinkage: "auto" }, + expected_returns: "black_litterman", bl_tau: 0.05, bl_years: 10, bl_omega_method: "idzorek", + }, + views: [], + constraints: { + max_weight: 0.1, max_sector_weight: 0.25, sector_overrides: {}, max_positions: null, + min_position_value: 0, max_turnover: null, max_transaction_cost: null, min_invested: 0, max_invested: 1, + }, + costs: { proportional_buy: 0.001, proportional_sell: 0.001, fixed_per_trade: 0 }, + opt: { + objective: "max_return", confidence: 0.95, cvar_mode: "fraction", cvar_limit_fraction: 0.05, + cvar_limit: null, risk_aversion: 1, target_return: null, solver: "auto", time_limit: 300, + row_generation: "auto", auto_scenarios: true, risk_drift: "model", + }, + data: { start: "", end: "", freq: "daily" }, +}; + +function deepMerge(base, add) { + if (Array.isArray(base) || add === null || typeof add !== "object" || typeof base !== "object") return add ?? base; + const out = { ...base }; + for (const [k, v] of Object.entries(add)) out[k] = k in base ? deepMerge(base[k], v) : v; + return out; +} + +function load() { + try { + const raw = localStorage.getItem(KEY); + return raw ? migrate(deepMerge(structuredClone(DEFAULT_STATE), JSON.parse(raw))) : structuredClone(DEFAULT_STATE); + } catch { return structuredClone(DEFAULT_STATE); } +} + +/** Fixes for settings stored by an earlier version (each applied once). */ +function migrate(s) { + s.ui.migrations = s.ui.migrations || []; + if (!s.ui.migrations.includes("time-limit")) { + // a stored `null` meant "no limit", and without one HiGHS can run for hours on a large problem + if (s.opt.time_limit === null) s.opt.time_limit = DEFAULT_STATE.opt.time_limit; + s.ui.migrations.push("time-limit"); + } + return s; +} + +export const state = load(); + +const listeners = new Set(); +export function onChange(fn) { listeners.add(fn); return () => listeners.delete(fn); } + +/** Persist and notify. `topic` says what changed ('selection', 'model', ...); `source` names the + tab that changed it, so that tab can update itself without being re-rendered from outside. */ +export function changed(topic = "state", source = null) { + try { localStorage.setItem(KEY, JSON.stringify(state)); } catch { /* private mode: keep going */ } + for (const fn of listeners) fn(topic, source); +} + +export function resetState() { + try { localStorage.removeItem(KEY); } catch { /* ignore */ } + location.reload(); +} + +/* ------------------------------------------------------------------ requests built from the state */ +export function covarianceSpec() { + const c = state.model.covariance; + return { method: c.method, lam: Number(c.lam), window: Number(c.window), + shrinkage: c.shrinkage === "auto" ? "auto" : Number(c.shrinkage) }; +} + +export function viewPayload() { + return state.views.map((v) => (v.kind === "sentiment" + ? { symbol: v.symbol, sentiment: Number(v.sentiment), confidence: Number(v.confidence), + strength: Number(v.strength), label: v.label || "", source: v.source || "" } + : { assets: v.assets, q: Number(v.q), confidence: Number(v.confidence), label: v.label || "", source: v.source || "" })); +} + +export function constraintPayload() { + const c = state.constraints; + const overrides = Object.entries(c.sector_overrides || {}).filter(([, v]) => v !== "" && v !== null); + return { + max_weight: Number(c.max_weight), + max_sector_weight: overrides.length + ? { "*": Number(c.max_sector_weight), ...Object.fromEntries(overrides.map(([k, v]) => [k, Number(v)])) } + : Number(c.max_sector_weight), + max_positions: c.max_positions ? Number(c.max_positions) : null, + min_position_value: Number(c.min_position_value) || 0, + max_turnover: c.max_turnover === null || c.max_turnover === "" ? null : Number(c.max_turnover), + max_transaction_cost: c.max_transaction_cost === null || c.max_transaction_cost === "" ? null : Number(c.max_transaction_cost), + min_invested: Number(c.min_invested) || 0, + max_invested: Number(c.max_invested) || 1, + }; +} + +export function scenarioPayload(extra = {}) { + const m = state.model; + return { + symbols: state.selection, model: m.scenario_model === "gaussian_exact" ? "gbm" : m.scenario_model, + n_scenarios: Number(m.n_scenarios), horizon: m.horizon, freq: m.freq, covariance: covarianceSpec(), + seed: Number(m.seed), sampling: m.sampling, moment_match: !!m.moment_match, + drift_convention: m.drift_convention, expected_returns: m.expected_returns, + innovation_history: m.innovation_history ? Number(m.innovation_history) : null, + importance_sampling: m.importance_sampling, views: viewPayload(), + bl_tau: Number(m.bl_tau), bl_years: Number(m.bl_years), + bl_omega_method: m.bl_omega_method || "idzorek", ...extra, + }; +} + +export function optimizePayload(extra = {}) { + const m = state.model, o = state.opt, p = state.portfolio; + const body = { + mode: p.mode, cash: Number(p.cash), holdings: p.mode === "rebalance" ? intHoldings(p.holdings) : {}, + symbols: state.selection.length ? state.selection : null, max_assets: Math.max(state.selection.length || 30, 2), + horizon: m.horizon, freq: m.freq, confidence: Number(o.confidence), + objective: o.objective, risk_aversion: Number(o.risk_aversion), + scenario_model: m.scenario_model, seed: Number(m.seed), sampling: m.sampling, + moment_match: !!m.moment_match, importance_sampling: m.importance_sampling, + drift_convention: m.drift_convention, row_generation: o.row_generation, risk_drift: o.risk_drift, + covariance: covarianceSpec(), expected_returns: m.expected_returns, views: viewPayload(), + bl_tau: Number(m.bl_tau), bl_years: Number(m.bl_years), bl_omega_method: m.bl_omega_method || "idzorek", + constraints: constraintPayload(), costs: { + proportional_buy: Number(state.costs.proportional_buy), proportional_sell: Number(state.costs.proportional_sell), + fixed_per_trade: Number(state.costs.fixed_per_trade), + }, + solver: o.solver, time_limit: o.time_limit ? Number(o.time_limit) : null, + innovation_history: m.innovation_history ? Number(m.innovation_history) : null, + }; + if (!o.auto_scenarios && m.scenario_model !== "gaussian_exact") body.n_scenarios = Number(m.n_scenarios); + if (o.objective === "max_return") { + if (o.cvar_mode === "money" && o.cvar_limit) body.cvar_limit = Number(o.cvar_limit); + else body.cvar_limit_fraction = Number(o.cvar_limit_fraction); + } else if (o.objective === "min_cvar") { + body.target_return = o.target_return === null || o.target_return === "" ? null : Number(o.target_return); + } + return { ...body, ...extra }; // `extra` wins: a sweep overrides exactly the field it varies +} + +export function intHoldings(h) { + return Object.fromEntries(Object.entries(h || {}).map(([k, v]) => [k, Math.round(Number(v) || 0)]).filter(([, v]) => v > 0)); +} + +/* ------------------------------------------------------------------ formatting */ +const nf = (min, max) => new Intl.NumberFormat("de-CH", { minimumFractionDigits: min, maximumFractionDigits: max }); + +export const fmt = { + money: (v, d = 0) => (Number.isFinite(v) ? nf(d, d).format(v) : "–"), + money2: (v) => (Number.isFinite(v) ? nf(2, 2).format(v) : "–"), + signed: (v, d = 0) => (Number.isFinite(v) ? (v > 0 ? "+" : "") + nf(d, d).format(v) : "–"), + compact: (v) => { + if (!Number.isFinite(v)) return "–"; + const a = Math.abs(v); + if (a >= 1e12) return `${nf(2, 2).format(v / 1e12)} T`; + if (a >= 1e9) return `${nf(1, 1).format(v / 1e9)} bn`; + if (a >= 1e6) return `${nf(1, 1).format(v / 1e6)} m`; + if (a >= 1e3) return `${nf(1, 1).format(v / 1e3)} k`; + return nf(0, 1).format(v); + }, + pct: (v, d = 1) => (Number.isFinite(v) ? `${nf(d, d).format(v * 100)} %` : "–"), + pctSigned: (v, d = 1) => (Number.isFinite(v) ? `${v > 0 ? "+" : ""}${nf(d, d).format(v * 100)} %` : "–"), + bp: (v) => (Number.isFinite(v) ? `${nf(0, 1).format(v * 10000)} bp` : "–"), + num: (v, d = 2) => (Number.isFinite(v) ? nf(d, d).format(v) : "–"), + int: (v) => (Number.isFinite(v) ? nf(0, 0).format(Math.round(v)) : "–"), + date: (v) => (typeof v === "string" ? v.slice(0, 10) : new Date(v).toISOString().slice(0, 10)), + secs: (v) => (Number.isFinite(v) ? `${nf(1, 1).format(v)} s` : "–"), +}; + +export const parseDate = (s) => Date.parse(String(s).slice(0, 10) + "T00:00:00Z"); + +/* ------------------------------------------------------------------ DOM helpers */ +export function el(tag, props = {}, ...children) { + const e = document.createElement(tag); + for (const [k, v] of Object.entries(props || {})) { + if (v === null || v === undefined) continue; + if (k === "class") e.className = v; + else if (k === "text") e.textContent = v; + else if (k === "html") e.innerHTML = v; // only for constants in this file + else if (k === "style") Object.assign(e.style, v); + else if (k.startsWith("on") && typeof v === "function") e.addEventListener(k.slice(2), v); + else if (k === "value") e.value = v; + else if (typeof v === "boolean") { if (v) e.setAttribute(k, ""); } + else e.setAttribute(k, String(v)); + } + for (const c of children.flat()) { + if (c === null || c === undefined || c === false) continue; + e.append(c instanceof Node ? c : document.createTextNode(String(c))); + } + return e; +} + +/** `replaceChildren` that drops null/false entries (so `cond ? node : null` is safe). */ +export function setChildren(node, ...kids) { + node.replaceChildren(...kids.flat().filter((k) => k !== null && k !== undefined && k !== false)); +} + +export function card(title, sub, ...content) { + const head = el("header", {}, title ? el("h2", { text: title }) : null, sub ? el("span", { class: "sub", text: sub }) : null); + return el("section", { class: "card" }, title || sub ? head : null, ...content); +} + +export function field(label, control, hint) { + return el("div", { class: "field" }, el("label", { text: label }), control, + hint ? el("span", { class: "hint", text: hint }) : null); +} + +export function input(type, value, onInput, attrs = {}) { + const e = el("input", { type, value: value ?? "", ...attrs }); + e.addEventListener("change", () => onInput(type === "number" ? (e.value === "" ? null : Number(e.value)) : e.value)); + return e; +} + +export function select(options, value, onInput, attrs = {}) { + const e = el("select", attrs); + for (const o of options) { + const [v, label] = Array.isArray(o) ? o : [o, o]; + e.appendChild(el("option", { value: v, text: label, selected: String(v) === String(value) })); + } + e.value = value; + e.addEventListener("change", () => onInput(e.value)); + return e; +} + +export function checkbox(label, value, onInput) { + const box = el("input", { type: "checkbox" }); + box.checked = !!value; + box.addEventListener("change", () => onInput(box.checked)); + return el("label", { class: "check" }, box, el("span", { text: label })); +} + +export function segmented(options, value, onInput) { + const wrap = el("div", { class: "seg" }); + for (const o of options) { + const [v, label] = Array.isArray(o) ? o : [o, o]; + const b = el("button", { type: "button", text: label, "aria-pressed": String(v) === String(value) }); + b.addEventListener("click", () => onInput(v)); + wrap.appendChild(b); + } + return wrap; +} + +export function button(label, onClick, cls = "btn", title = "") { + return el("button", { class: cls, type: "button", text: label, onclick: onClick, title }); +} + +/** Sortable table. columns: [{key, label, num, fmt, cls, title}] */ +export function dataTable(columns, rows, opts = {}) { + const wrap = el("div", { class: "table-wrap" }); + const table = el("table", { class: "data" }); + const thead = el("thead"), tbody = el("tbody"); + let sortKey = opts.sortKey ?? null, sortDir = opts.sortDir ?? -1; + + const headRow = el("tr"); + for (const c of columns) { + const th = el("th", { class: `${c.num ? "num" : ""}${opts.sortable === false ? "" : " sortable"}`, title: c.title || "" }); + th.append(document.createTextNode(c.label)); + if (opts.sortable !== false) { + th.appendChild(el("span", { class: "arrow", text: sortKey === c.key ? (sortDir < 0 ? " ▼" : " ▲") : "" })); + th.addEventListener("click", () => { + if (sortKey === c.key) sortDir = -sortDir; else { sortKey = c.key; sortDir = c.num ? -1 : 1; } + render(); + }); + } + headRow.appendChild(th); + } + thead.appendChild(headRow); + + function render() { + const data = [...rows]; + if (sortKey) { + data.sort((a, b) => { + const x = a[sortKey], y = b[sortKey]; + if (x === y) return 0; + if (x === null || x === undefined) return 1; + if (y === null || y === undefined) return -1; + return (typeof x === "number" && typeof y === "number" ? x - y : String(x).localeCompare(String(y))) * sortDir; + }); + } + tbody.replaceChildren(); + for (const r of data.slice(0, opts.limit || 5000)) { + const tr = el("tr", opts.rowClass ? { class: opts.rowClass(r) || "" } : {}); + if (opts.onRowClick) { tr.style.cursor = "pointer"; tr.addEventListener("click", () => opts.onRowClick(r)); } + for (const c of columns) { + const td = el("td", { class: [c.num ? "num" : "", c.cls ? (typeof c.cls === "function" ? c.cls(r) : c.cls) : ""].join(" ").trim() }); + const v = c.render ? c.render(r) : (c.fmt ? c.fmt(r[c.key], r) : r[c.key]); + if (v instanceof Node) td.appendChild(v); + else td.textContent = v === null || v === undefined ? "–" : String(v); + tr.appendChild(td); + } + tbody.appendChild(tr); + } + headRow.querySelectorAll(".arrow").forEach((a, i) => { + a.textContent = sortKey === columns[i].key ? (sortDir < 0 ? " ▼" : " ▲") : ""; + }); + } + render(); + table.append(thead, tbody); + wrap.appendChild(table); + return wrap; +} + +export function statTiles(items) { + return el("div", { class: "stats" }, ...items.map((s) => el("div", { class: "stat" }, + el("div", { class: "label", text: s.label }), + el("div", { class: "value", text: s.value, title: s.title || "" }), + s.delta ? el("div", { class: `delta ${s.deltaClass || "muted"}`, text: s.delta }) : null))); +} + +export function kv(pairs) { + const dl = el("dl", { class: "kv" }); + for (const [k, v] of pairs) { + if (v === null || v === undefined) continue; + dl.append(el("dt", { text: k }), el("dd", { text: String(v) })); + } + return dl; +} + +/** The panel currently in the document for `id` -- a re-render elsewhere may have replaced the + node a pending request captured, and writing a result into a detached node loses it. */ +export function live(id, fallback) { + return document.getElementById(id) || fallback; +} + +export function chartBox(height) { + return el("div", { class: "chart", style: height ? { minHeight: `${height}px` } : {} }); +} + +export function warnBox(messages) { + if (!messages?.length) return null; + return el("div", {}, ...messages.map((m) => el("div", { class: "warn-box", text: m }))); +} + +export function errorBox(err) { + return el("div", { class: "error-box", text: String(err?.message || err) }); +} + +export function empty(text) { return el("div", { class: "empty", text }); } + +/** Run an async task, showing a spinner in `btn` and errors in `target`. */ +export async function withBusy(btn, target, fn) { + const label = btn?.textContent; + if (btn) { btn.disabled = true; btn.replaceChildren(el("span", { class: "spinner" }), document.createTextNode(" " + label)); } + target?.classList.add("busy"); + try { + return await fn(); + } catch (err) { + target?.replaceChildren(errorBox(err)); + throw err; + } finally { + if (btn) { btn.disabled = false; btn.textContent = label; } + target?.classList.remove("busy"); + } +} + +export const HORIZONS = [["day", "1 day"], ["week", "1 week"], ["month", "1 month"], ["quarter", "1 quarter"], ["year", "1 year"]]; +export const SECTOR_ORDER = [ + "Information Technology", "Financials", "Health Care", "Consumer Discretionary", "Industrials", + "Communication Services", "Consumer Staples", "Energy", "Materials", "Utilities", "Real Estate", "Unknown", +]; diff --git a/quantcore/api/ui/js/main.js b/quantcore/api/ui/js/main.js new file mode 100644 index 0000000..3bcbe45 --- /dev/null +++ b/quantcore/api/ui/js/main.js @@ -0,0 +1,85 @@ +/* Entry point: the tab frame, the theme and the wiring between the sidebar and the tabs. */ + +import { changed, el, onChange, state } from "./core.js"; +import { redrawAll } from "./charts.js"; +import { buildSidebar, refreshHealth } from "./sidebar.js"; +import { universe } from "./universe.js"; +import * as stocks from "./tabs/stocks.js"; +import * as risk from "./tabs/risk.js"; +import * as views from "./tabs/views.js"; +import * as constraints from "./tabs/constraints.js"; +import * as optimize from "./tabs/optimize.js"; + +const TABS = [ + { id: "stocks", label: "Stocks & data", mod: stocks, deps: ["selection", "universe", "data", "model"] }, + { id: "risk", label: "Risk & scenarios", mod: risk, deps: ["selection", "model", "portfolio", "views"] }, + { id: "views", label: "Expected returns", mod: views, deps: ["selection", "model", "views"] }, + { id: "constraints", label: "Constraints", mod: constraints, deps: ["selection", "constraints", "costs", "portfolio"] }, + { id: "optimize", label: "Optimisation", mod: optimize, deps: ["selection", "model", "portfolio", "opt", "constraints", "costs", "views"] }, +]; + +const tabBar = document.getElementById("tabs"); +const main = document.getElementById("main"); + +function activate(id, force = false) { + if (!force && state.ui.tab === id && main.dataset.tab === id) return; + state.ui.tab = id; + changed("ui"); + main.dataset.tab = id; + for (const b of tabBar.children) b.setAttribute("aria-selected", String(b.dataset.tab === id)); + const tab = TABS.find((t) => t.id === id) || TABS[0]; + main.replaceChildren(el("div", { class: "tabpanel" })); + const panel = main.firstChild; + Promise.resolve(tab.mod.render(panel)).catch((err) => { + panel.replaceChildren(el("div", { class: "error-box", text: String(err?.message || err) })); + }); +} + +for (const t of TABS) { + const b = el("button", { type: "button", role: "tab", text: t.label, "data-tab": t.id, "aria-selected": "false" }); + b.addEventListener("click", () => activate(t.id, true)); + tabBar.appendChild(b); +} + +// a tab can send the reader to another tab ("edit these constraints") +document.addEventListener("quantcore:tab", (ev) => { + if (TABS.some((t) => t.id === ev.detail)) activate(ev.detail, true); +}); + +/* ------------------------------------------------------------------ theme */ +const themeBtn = document.getElementById("theme"); +function applyTheme() { + const t = state.ui.theme; + if (t === "auto") document.documentElement.removeAttribute("data-theme"); + else document.documentElement.setAttribute("data-theme", t); + themeBtn.textContent = t === "auto" ? "Theme: system" : t === "dark" ? "Theme: dark" : "Theme: light"; + requestAnimationFrame(() => redrawAll()); +} +themeBtn.addEventListener("click", () => { + state.ui.theme = { auto: "light", light: "dark", dark: "auto" }[state.ui.theme] || "auto"; + changed("ui"); + applyTheme(); +}); +applyTheme(); +matchMedia("(prefers-color-scheme: dark)").addEventListener("change", () => { if (state.ui.theme === "auto") applyTheme(); }); + +/* ------------------------------------------------------------------ start */ +buildSidebar(document.getElementById("sidebar")); +refreshHealth(document.getElementById("status")); +activate(state.ui.tab || "stocks", true); + +// Every tab reads names, sectors and prices synchronously through `info()`, but only the Stocks tab +// used to fetch the universe -- so a reload that restored any other tab (the tab is part of the +// stored state) drew blank names and a price of 0, and the holdings editor showed 0 CHF positions. +// Paint first, then redraw once the reference data is in. +universe().then(() => activate(state.ui.tab, true)).catch(() => { /* the tab shows its own error */ }); + +let pending = null; +onChange((topic, source) => { + if (topic === "ui") return; + if (source && source === state.ui.tab) return; // the tab already updated itself + const tab = TABS.find((t) => t.id === state.ui.tab); + if (!tab?.deps.includes(topic)) return; + clearTimeout(pending); + pending = setTimeout(() => activate(state.ui.tab, true), 60); +}); diff --git a/quantcore/api/ui/js/runs.js b/quantcore/api/ui/js/runs.js new file mode 100644 index 0000000..750e80a --- /dev/null +++ b/quantcore/api/ui/js/runs.js @@ -0,0 +1,47 @@ +/* Optimisation runs of this session, kept in memory so several tabs can refer to them. + * + * One of them is *selected*: the run the bench is showing and the one every "use this result" action + * works on. A new run selects itself; clicking a row in the history selects that one instead. Runs + * are addressed by `id`, never by their position -- `addRun` unshifts, so an index captured while + * rendering would point at a different run as soon as the next run arrives. + */ + +const runs = []; +let selectedId = null; +let nextId = 1; + +export function addRun(entry) { + const run = { ...entry, id: nextId++, at: new Date() }; + runs.unshift(run); + if (runs.length > 12) runs.pop(); + selectedId = run.id; + return run; +} + +export function allRuns() { return runs; } + +/* There is deliberately no `lastRun()`: "the newest run" and "the run the user is looking at" are + * different things once the history has more than one entry, and reaching for the newest was exactly + * the bug -- the holdings adopted were not the ones on screen. Use `selectedRun()`. */ + +/** The run on screen: what "this result" means in every tab. Falls back to the newest. */ +export function selectedRun() { + return runs.find((r) => r.id === selectedId) || runs[0] || null; +} + +export function selectRun(id) { + if (runs.some((r) => r.id === id)) selectedId = id; + return selectedRun(); +} + +export function isSelected(id) { + const sel = selectedRun(); + return !!sel && sel.id === id; +} + +export function clearRuns() { runs.length = 0; selectedId = null; } + +/** Shares of a result, as a {symbol: shares} map. */ +export function sharesOf(result) { + return Object.fromEntries((result?.positions || []).filter((p) => p.shares > 0).map((p) => [p.symbol, p.shares])); +} diff --git a/quantcore/api/ui/js/sidebar.js b/quantcore/api/ui/js/sidebar.js new file mode 100644 index 0000000..cca1d05 --- /dev/null +++ b/quantcore/api/ui/js/sidebar.js @@ -0,0 +1,179 @@ +/* The sidebar: the state every tab shares -- candidate stocks, the portfolio and the models. */ + +import { + api, button, changed, checkbox, el, field, fmt, input, intHoldings, onChange, resetState, + segmented, select, state, HORIZONS, +} from "./core.js"; +import { info, topSymbols, universe } from "./universe.js"; + +const SCENARIO_MODELS = [ + ["gbm", "GBM (normal)"], + ["empirical", "empirical innovations"], + ["gaussian_exact", "exact Gaussian (no scenarios)"], +]; + +const SHOWN = ["selection", "universe", "portfolio", "model", "state"]; // topics the sidebar displays +let root = null; + +export function buildSidebar(node) { + root = node; + draw(node); + // Only re-draw for changes made elsewhere: rebuilding the panel while someone is editing it + // would replace the control under the pointer (a button clicked right after typing is lost). + onChange((topic, source) => { + if (source !== "sidebar" && SHOWN.includes(topic)) draw(root); + }); +} + +/** Re-draw the sidebar itself (for edits that change which controls it shows). */ +function refresh() { if (root) draw(root); } + +const notify = (topic) => changed(topic, "sidebar"); + +function section(title, count, open, ...content) { + return el("details", { class: "section", open }, + el("summary", {}, document.createTextNode(title), el("span", { class: "count", text: count ?? "" })), + el("div", {}, ...content)); +} + +function draw(node) { + node.replaceChildren( + candidatesSection(), + portfolioSection(), + modelSection(), + el("div", { class: "btnbar", style: { marginTop: "12px" } }, + button("Reset all settings", () => { if (confirm("Reset every setting of the test bench?")) resetState(); }, "btn small")), + ); +} + +/* ------------------------------------------------------------------ candidates */ +function candidatesSection() { + const u = state.universe; + const body = el("div"); + const chips = el("div", { class: "chips", style: { marginTop: "8px" } }); + + const redraw = () => { + chips.replaceChildren(); + if (!state.selection.length) { + chips.appendChild(el("span", { class: "hint", text: "no stocks selected - pick them in the Stocks tab or fill from the market" })); + } + for (const s of state.selection.slice(0, 60)) { + chips.appendChild(el("span", { class: "chip" }, document.createTextNode(s), + button("x", () => { + state.selection = state.selection.filter((x) => x !== s); + notify("selection"); redraw(); + }, ""))); + } + if (state.selection.length > 60) chips.appendChild(el("span", { class: "hint", text: `+ ${state.selection.length - 60} more` })); + }; + redraw(); + + const fill = async () => { + const { rows } = await universe(); + state.selection = topSymbols(rows, { region: u.region, sector: u.sector }, Number(u.top)); + notify("selection"); + refresh(); + }; + + body.append( + el("div", { class: "row" }, + field("Region", select([["", "world"], ["US", "US"], ["Europe", "Europe"]], u.region, + (v) => { u.region = v; notify("universe"); })), + field("Largest", input("number", u.top, (v) => { u.top = v || 30; notify("universe"); }, { min: 2, max: 500, step: 1 }))), + el("div", { class: "btnbar" }, + button("Fill from market caps", fill, "btn small primary"), + button("Clear", () => { state.selection = []; notify("selection"); refresh(); }, "btn small")), + chips, + ); + return section("Candidate stocks", `${state.selection.length}`, true, body); +} + +/* ------------------------------------------------------------------ portfolio */ +function portfolioSection() { + const p = state.portfolio; + const holdings = intHoldings(p.holdings); + const nHold = Object.keys(holdings).length; + const value = Object.entries(holdings).reduce((a, [s, n]) => a + (info(s)?.price || 0) * n, 0); + const body = el("div", + {}, + field("Mode", segmented([["new", "New portfolio"], ["rebalance", "Rebalance"]], p.mode, (v) => { + p.mode = v; notify("portfolio"); refresh(); + })), + field(p.mode === "new" ? "Cash to invest (CHF)" : "Free cash (CHF)", + input("number", p.cash, (v) => { p.cash = v ?? 0; notify("portfolio"); }, { min: 0, step: 10000 })), + el("div", { class: "btnbar", style: { marginBottom: "6px" } }, + ...[["1 m", 1e6], ["10 m", 1e7], ["50 m", 5e7]].map(([label, amount]) => + button(label, () => { p.cash = amount; notify("portfolio"); refresh(); }, "btn small"))), + p.mode === "rebalance" + ? el("div", { class: "note" }, + `${nHold} holdings, ${fmt.compact(value)} CHF at the last close. Edit them in the Optimisation tab.`) + : el("div", { class: "note", text: "A new portfolio starts from cash; holdings are ignored." }), + ); + return section("Portfolio", p.mode === "rebalance" ? `${nHold} positions` : fmt.compact(p.cash), true, body); +} + +/* ------------------------------------------------------------------ models */ +function modelSection() { + const m = state.model; + const set = (k, v) => { m[k] = v; notify("model"); }; + const setShape = (k, v) => { m[k] = v; notify("model"); refresh(); }; + const cov = m.covariance; + const exact = m.scenario_model === "gaussian_exact"; + + const covControls = cov.method === "ewm" + ? field("EWM decay λ", input("number", cov.lam, (v) => { cov.lam = v; notify("model"); }, { min: 0.5, max: 0.9999, step: 0.005 }), + `≈ ${Math.round(3 / (1 - Number(cov.lam)))} effective observations`) + : field("Window (periods)", input("number", cov.window, (v) => { cov.window = v; notify("model"); }, { min: 10, step: 10 })); + + return section("Models", m.scenario_model, true, el("div", {}, + el("div", { class: "row" }, + field("Frequency", select([["daily", "daily"], ["weekly", "weekly"]], m.freq, (v) => set("freq", v))), + field("Horizon", select(HORIZONS, m.horizon, (v) => set("horizon", v)))), + field("Stock-price model", select(SCENARIO_MODELS, m.scenario_model, (v) => setShape("scenario_model", v))), + exact ? el("div", { class: "note", text: "Exact Gaussian: mean-CVaR solved as the equivalent mean-variance problem by outer approximation - no sampling, no scenario noise." }) : null, + !exact ? el("div", { class: "row" }, + field("Scenarios", input("number", m.n_scenarios, (v) => set("n_scenarios", v), { min: 256, step: 1024 }), + "rounded up to a power of two"), + field("Seed", input("number", m.seed, (v) => set("seed", v), { step: 1 }))) : null, + !exact ? el("div", { class: "row" }, + field("Sampling", select([["sobol", "Sobol'"], ["mc", "plain MC"], ["antithetic", "antithetic"], ["lhs", "Latin hypercube"], ["balanced", "balanced (empirical)"]], m.sampling, (v) => set("sampling", v))), + field("Tail tilt", select([["auto", "auto"], ["market", "market"], ["adaptive", "adaptive"], ["off", "off"]], m.importance_sampling, (v) => set("importance_sampling", v)))) : null, + !exact ? checkbox("Moment matching (exact mean and covariance)", m.moment_match, (v) => set("moment_match", v)) : null, + el("div", { class: "row", style: { marginTop: "8px" } }, + field("Covariance", select([["ewm", "EWM"], ["rolling", "rolling"]], cov.method, (v) => { cov.method = v; notify("model"); refresh(); })), + covControls), + field("Shrinkage", select([["auto", "auto (Schäfer-Strimmer)"], [0, "none"], [0.1, "0.10"], [0.25, "0.25"], [0.5, "0.50"]], + cov.shrinkage, (v) => { cov.shrinkage = v === "auto" ? "auto" : Number(v); notify("model"); })), + field("Expected returns", select([["black_litterman", "Black-Litterman"], ["historical", "historical mean"], ["zero", "zero drift"]], + m.expected_returns, (v) => setShape("expected_returns", v))), + m.expected_returns === "black_litterman" + ? el("div", { class: "row" }, + field("τ", input("number", m.bl_tau, (v) => set("bl_tau", v), { min: 0.001, step: 0.01 })), + field("History (years)", input("number", m.bl_years, (v) => set("bl_years", v), { min: 1, max: 20, step: 1 }))) + : null, + field("Drift convention", select([["arithmetic", "arithmetic (Itô-corrected)"], ["log", "log"]], m.drift_convention, (v) => set("drift_convention", v))), + )); +} + +/* ------------------------------------------------------------------ health pill */ +export async function refreshHealth(node) { + try { + const h = await api.get("/health"); + const store = h.store || {}; + const cuopt = h.cuopt || {}; + node.replaceChildren( + pill("data", `${store.n_universe ?? "?"} stocks · ${fmt.date(store.last_price_date || "")}`, "ok"), + pill("cuOpt", cuopt.reachable ? (cuopt.version || "up") : "unreachable", cuopt.reachable ? "ok" : "warn"), + pill("HiGHS", "in-process", "ok"), + pill("base", h.base_currency || "CHF", "ok"), + el("span", { class: "pill" }, document.createTextNode(`v${h.version}`)), + ); + } catch (err) { + node.replaceChildren(pill("api", String(err.message).slice(0, 60), "bad")); + } +} + +function pill(label, value, kind) { + return el("span", { class: "pill" }, el("span", { class: `dot ${kind}` }), + document.createTextNode(label + " "), el("b", { text: value })); +} diff --git a/quantcore/api/ui/js/tabs/constraints.js b/quantcore/api/ui/js/tabs/constraints.js new file mode 100644 index 0000000..258516c --- /dev/null +++ b/quantcore/api/ui/js/tabs/constraints.js @@ -0,0 +1,173 @@ +/* Tab 4 -- constraints and costs, with a feasibility check against the candidate set. */ + +import { + card, changed, chartBox, dataTable, el, empty, field, fmt, input, intHoldings, setChildren, state, + statTiles, +} from "../core.js"; +import { barChart } from "../charts.js"; +import { info } from "../universe.js"; + +export function render(node) { + node.replaceChildren(); + if (!state.selection.length) { node.appendChild(empty("select candidate stocks in the Stocks tab first")); return; } + const derived = el("div", { id: "constraint-derived" }); + const refresh = () => { + const live = document.getElementById("constraint-derived") || derived; + setChildren(live, feasibilityCard(), sectorCard()); + }; + refresh(); + node.append(limitsCard(refresh), costsCard(refresh), derived); +} + +function wealth() { + const h = intHoldings(state.portfolio.holdings); + const held = state.portfolio.mode === "rebalance" + ? Object.entries(h).reduce((a, [s, n]) => a + (info(s)?.price || 0) * n, 0) : 0; + return Number(state.portfolio.cash) + held; +} + +/* ------------------------------------------------------------------ limits */ +function limitsCard(refresh) { + const c = state.constraints; + const set = (k) => (v) => { c[k] = v; changed("constraints", "constraints"); refresh(); }; + const W = wealth(); + const sectors = [...new Set(state.selection.map((s) => info(s)?.sector || "Unknown"))].sort(); + + const overrides = el("div"); + const drawOverrides = () => { + overrides.replaceChildren(el("div", { class: "row" }, ...sectors.map((sec) => + field(sec, input("number", c.sector_overrides?.[sec] ?? "", (v) => { + c.sector_overrides = { ...(c.sector_overrides || {}) }; + if (v === null || v === "") delete c.sector_overrides[sec]; else c.sector_overrides[sec] = v; + changed("constraints", "constraints"); refresh(); + }, { min: 0, max: 1, step: 0.05, placeholder: String(c.max_sector_weight) }))))); + }; + drawOverrides(); + + return card("Portfolio constraints", `fractions of wealth (${fmt.compact(W)} CHF)`, el("div", {}, + el("div", { class: "row" }, + field("Max weight per stock", input("number", c.max_weight, set("max_weight"), { min: 0.001, max: 1, step: 0.01 }), + `fraction of ${fmt.compact(W)} CHF per name`), + field("Max weight per sector", input("number", c.max_sector_weight, set("max_sector_weight"), { min: 0.01, max: 1, step: 0.05 }), + "applies to every GICS sector unless overridden below"), + field("Max positions", input("number", c.max_positions ?? "", set("max_positions"), { min: 1, step: 1, placeholder: "no limit" }), + "cardinality limit (adds binaries)"), + field("Min position value", input("number", c.min_position_value, set("min_position_value"), { min: 0, step: 1000 }), + "CHF; a position is either 0 or at least this")), + el("div", { class: "row" }, + field("Min invested", input("number", c.min_invested, set("min_invested"), { min: 0, max: 1, step: 0.05 }), + "fraction of wealth that must be in stocks"), + field("Max invested", input("number", c.max_invested, set("max_invested"), { min: 0.01, max: 1, step: 0.05 }), + "the rest stays in cash"), + field("Max turnover", input("number", c.max_turnover ?? "", set("max_turnover"), { min: 0, step: 0.05, placeholder: "no limit" }), + "buys + sells as a fraction of wealth"), + field("Max transaction cost", input("number", c.max_transaction_cost ?? "", set("max_transaction_cost"), + { min: 0, step: 500, placeholder: "no limit" }), "CHF")), + el("h3", { text: "Sector overrides", style: { marginTop: "10px" } }), + el("div", { class: "hint", text: "empty = the general sector cap" }), + overrides)); +} + +/* ------------------------------------------------------------------ costs */ +function costsCard(refresh) { + const k = state.costs; + const set = (key) => (v) => { k[key] = v ?? 0; changed("costs", "constraints"); refresh(); }; + return card("Transaction costs", "charged on the traded value; they enter the objective and the cash balance", + el("div", { class: "row" }, + field("Buy", input("number", k.proportional_buy, set("proportional_buy"), { min: 0, step: 0.0005 }), fmt.bp(k.proportional_buy)), + field("Sell", input("number", k.proportional_sell, set("proportional_sell"), { min: 0, step: 0.0005 }), fmt.bp(k.proportional_sell)), + field("Fixed per traded stock", input("number", k.fixed_per_trade, set("fixed_per_trade"), { min: 0, step: 5 }), + k.fixed_per_trade > 0 ? "CHF - adds one binary per stock" : "CHF (0 = no binaries)"))); +} + +/* ------------------------------------------------------------------ feasibility */ +function feasibilityCard() { + const c = state.constraints, W = wealth(); + const sel = state.selection; + const bySector = {}; + for (const s of sel) (bySector[info(s)?.sector || "Unknown"] ||= []).push(s); + const cap = (sec) => Number(c.sector_overrides?.[sec] ?? c.max_sector_weight); + + const reachable = sel.filter((s) => (info(s)?.price || 0) <= c.max_weight * W); + const headroomBySector = Object.entries(bySector) + .map(([sec, list]) => Math.min(cap(sec), list.length * Number(c.max_weight))); + const headroom = headroomBySector.reduce((a, b) => a + b, 0); + const cardHeadroom = c.max_positions ? Number(c.max_positions) * Number(c.max_weight) : Infinity; + + const checks = [ + { ok: sel.length * c.max_weight >= c.min_invested, + label: "Enough names for the minimum investment", + detail: `${sel.length} candidates × ${fmt.pct(c.max_weight, 0)} = ${fmt.pct(sel.length * c.max_weight, 0)} of wealth available, ${fmt.pct(c.min_invested, 0)} required` }, + { ok: headroom >= c.min_invested, + label: "Sector caps leave room for the minimum investment", + detail: `the sector caps allow at most ${fmt.pct(headroom, 0)} of wealth in total` }, + { ok: cardHeadroom >= c.min_invested, + label: "Cardinality limit leaves room", + detail: c.max_positions ? `${c.max_positions} positions × ${fmt.pct(c.max_weight, 0)} = ${fmt.pct(cardHeadroom, 0)}` : "no cardinality limit" }, + { ok: !c.min_position_value || c.min_position_value <= c.max_weight * W, + label: "Minimum position fits under the weight cap", + detail: `minimum ${fmt.compact(c.min_position_value)} CHF vs cap ${fmt.compact(c.max_weight * W)} CHF` }, + { ok: c.min_invested <= c.max_invested, + label: "Investment band is consistent", + detail: `${fmt.pct(c.min_invested, 0)} … ${fmt.pct(c.max_invested, 0)}` }, + { ok: reachable.length === sel.length, + label: "Every candidate is affordable as one share", + detail: reachable.length === sel.length ? "one share of each candidate fits under the weight cap" + : `${sel.length - reachable.length} stocks cost more per share than ${fmt.compact(c.max_weight * W)} CHF and can never be held` }, + ]; + + const failed = checks.filter((c2) => !c2.ok); + return card("Feasibility", failed.length ? `${failed.length} issue${failed.length > 1 ? "s" : ""} - the solver would return infeasible or ignore these names` + : "the constraints and the candidate set are consistent", + el("div", {}, + statTiles([ + { label: "Candidates", value: fmt.int(sel.length), delta: `${Object.keys(bySector).length} sectors` }, + { label: "Wealth", value: `${fmt.compact(W)} CHF` }, + { label: "Cap per name", value: fmt.compact(c.max_weight * W), delta: fmt.pct(c.max_weight, 1) }, + { label: "Max reachable", value: fmt.pct(Math.min(headroom, cardHeadroom, sel.length * c.max_weight, c.max_invested), 0), + delta: "of wealth under all caps" }, + { label: "Min positions needed", value: fmt.int(Math.ceil(c.min_invested / c.max_weight)), + delta: `to reach ${fmt.pct(c.min_invested, 0)}` }, + ]), + el("div", { style: { marginTop: "10px" } }, dataTable([ + { key: "ok", label: "", render: (r) => el("span", { class: r.ok ? "pos" : "neg", text: r.ok ? "✓" : "✗" }) }, + { key: "label", label: "Check" }, + { key: "detail", label: "Detail", cls: "muted" }, + ], checks, { sortable: false })))); +} + +/* ------------------------------------------------------------------ sector composition */ +function sectorCard() { + const c = state.constraints; + const sel = state.selection; + const bySector = {}; + for (const s of sel) { + const sec = info(s)?.sector || "Unknown"; + (bySector[sec] ||= { n: 0, cap: 0 }); + bySector[sec].n += 1; + bySector[sec].cap += info(s)?.market_cap || 0; + } + const totalCap = Object.values(bySector).reduce((a, b) => a + b.cap, 0) || 1; + const items = Object.entries(bySector).sort((a, b) => b[1].cap - a[1].cap).map(([sec, v]) => ({ + label: sec.length > 22 ? sec.slice(0, 21) + "…" : sec, values: [v.cap / totalCap], + })); + const box = chartBox(); + const out = card("Sectors of the candidate set", "cap-weighted share of the selection against the cap that will be enforced", + el("div", {}, box, + dataTable([ + { key: "sector", label: "Sector" }, + { key: "n", label: "Stocks", num: true, fmt: (v) => fmt.int(v) }, + { key: "share", label: "Cap-weighted share", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: "limit", label: "Limit", num: true, fmt: (v) => fmt.pct(v, 0) }, + { key: "room", label: "Max under the caps", num: true, fmt: (v) => fmt.pct(v, 0), + title: "the smaller of the sector cap and (number of stocks × the per-stock cap)" }, + ], Object.entries(bySector).map(([sec, v]) => ({ + sector: sec, n: v.n, share: v.cap / totalCap, + limit: Number(c.sector_overrides?.[sec] ?? c.max_sector_weight), + room: Math.min(Number(c.sector_overrides?.[sec] ?? c.max_sector_weight), v.n * Number(c.max_weight)), + })), { sortKey: "share", sortDir: -1 }))); + barChart(box, { items, valueFormat: (v) => fmt.pct(v, 0), horizontal: true, rowHeight: 24, labelWidth: 150, + valueName: "cap-weighted share", + references: [{ value: Number(c.max_sector_weight), label: `sector cap ${fmt.pct(c.max_sector_weight, 0)}` }] }); + return out; +} diff --git a/quantcore/api/ui/js/tabs/optimize.js b/quantcore/api/ui/js/tabs/optimize.js new file mode 100644 index 0000000..c43971a --- /dev/null +++ b/quantcore/api/ui/js/tabs/optimize.js @@ -0,0 +1,511 @@ +/* Tab 5 -- the optimisation itself: inputs, the proposal, and before/after for a rebalancing. */ + +import { + api, button, card, changed, chartBox, constraintPayload, dataTable, el, empty, field, fmt, input, + intHoldings, kv, live, optimizePayload, segmented, select, setChildren, state, statTiles, warnBox, + withBusy, +} from "../core.js"; +import { barChart, scatterChart } from "../charts.js"; +import { info } from "../universe.js"; +import { addRun, allRuns, clearRuns, isSelected, selectRun, selectedRun, sharesOf } from "../runs.js"; + +/** The result on screen. It is the *selected* run, so every "use this result" action and the + * history table agree on which one that is -- see js/runs.js. */ +const shown = () => selectedRun()?.result || null; + +export function render(node) { + node.replaceChildren(); + if (!state.selection.length) { node.appendChild(empty("select candidate stocks in the Stocks tab first")); return; } + const rerender = () => render(node); + const out = el("div", { id: "opt-result" }); + node.append(inputsCard(out, rerender), out, historyCard(rerender)); + const current = shown(); + if (current) drawResult(out, current); + else out.appendChild(empty("run the optimisation to see the proposal")); +} + +/* ------------------------------------------------------------------ inputs */ +function inputsCard(out, rerender) { + const o = state.opt, p = state.portfolio; + // A plain value edit must not rebuild the card: the change event fires on blur, so rebuilding + // here would replace the button the user is clicking and swallow the click. + const set = (k) => (v) => { o[k] = v; changed("opt", "optimize"); }; + const setShape = (k) => (v) => { o[k] = v; changed("opt", "optimize"); rerender(); }; + let runBtn, sweepBtn; + + const run = async (extra = {}, label = null) => { + const panel = live("opt-result", out); + await withBusy(runBtn, panel, async () => { + const body = optimizePayload(extra); + const t0 = performance.now(); + const abort = new AbortController(); + const stop = showProgress(panel, body, "", () => abort.abort()); + let result; + try { + result = await api.post("/optimize", body, { signal: abort.signal }); + } catch (err) { + if (err.name === "AbortError") { cancelled(live("opt-result", panel)); return; } + throw err; + } finally { stop(); } + addRun({ label: label || describeRun(body), request: body, result, wall: (performance.now() - t0) / 1000 }); + drawResult(live("opt-result", panel), result); + rerender(); + }).catch(() => {}); + }; + + // The sweep traces the frontier along the parameter the *selected* objective is steered by. It + // used to force objective='max_return' and vary the risk limit whatever was selected, so a sweep + // started under "minimise CVaR" silently ran five constrained max-return problems and every one + // of them came back sitting on its CVaR limit. + const sweepPlan = () => { + if (o.objective === "min_cvar") { + const base = Number(o.target_return); + if (!Number.isFinite(base) || base <= 0) return null; + return { + what: "the target return", + steps: [0.25, 0.5, 1, 1.5, 2].map((k) => ({ + extra: { target_return: round4(base * k) }, + label: `E[r] ≥ ${fmt.pct(base * k, 3)}`, + })), + }; + } + if (o.objective === "utility") { + const base = Number(o.risk_aversion) || 1; + return { + what: "the risk aversion λ", + steps: [0.25, 0.5, 1, 2, 4].map((k) => ({ + extra: { risk_aversion: round4(base * k) }, + label: `λ = ${round4(base * k)}`, + })), + }; + } + return { + what: "the risk limit", + steps: [0.01, 0.02, 0.03, 0.05, 0.08].map((f) => ({ + extra: { cvar_limit_fraction: f, cvar_limit: null }, + label: `CVaR ≤ ${fmt.pct(f, 0)}`, + })), + }; + }; + + const sweep = async () => { + const plan = sweepPlan(); + if (!plan) { alert("set a target return first - the sweep varies it around the value you entered"); return; } + await withBusy(sweepBtn, live("opt-result", out), async () => { + for (const step of plan.steps) { + const body = optimizePayload(step.extra); + const t0 = performance.now(); + const abort = new AbortController(); + const stop = showProgress(live("opt-result", out), body, step.label, () => abort.abort()); + let result; + try { + result = await api.post("/optimize", body, { signal: abort.signal }); + } catch (err) { + if (err.name === "AbortError") { cancelled(live("opt-result", out)); break; } + throw err; + } finally { stop(); } + addRun({ label: step.label, request: body, result, wall: (performance.now() - t0) / 1000 }); + drawResult(live("opt-result", out), result); + } + rerender(); + }).catch(() => {}); + }; + + runBtn = button("Run optimisation", () => run(), "btn primary"); + const sweepWhat = o.objective === "min_cvar" ? "target return" + : o.objective === "utility" ? "\u03bb" : "risk limit"; + sweepBtn = button(`Sweep the ${sweepWhat} (5 runs)`, sweep, "btn", + `five runs varying ${sweepWhat} around the current value, keeping the objective`); + + // the hint under the limit is updated in place (see `set` above) + const limitHint = el("span", { class: "hint", text: limitHintText() }); + const limitField = (label, control) => el("div", { class: "field" }, el("label", { text: label }), control, limitHint); + const onLimit = (k) => (v) => { set(k)(v); limitHint.textContent = limitHintText(); }; + const limitControls = o.objective === "max_return" + ? el("div", { class: "row" }, + field("CVaR limit", segmented([["fraction", "% of wealth"], ["money", "CHF"]], o.cvar_mode, setShape("cvar_mode"))), + o.cvar_mode === "fraction" + // entered in percent, because the control above says "% of wealth" + ? limitField("Limit (% of wealth)", input("number", round4((o.cvar_limit_fraction || 0) * 100), + (v) => onLimit("cvar_limit_fraction")((v ?? 0) / 100), + { min: 0.1, max: 99, step: 0.5 })) + : limitField("Limit (CHF)", input("number", o.cvar_limit, onLimit("cvar_limit"), { min: 1, step: 1000 }))) + : o.objective === "min_cvar" + ? field("Target return over the horizon", input("number", o.target_return ?? "", set("target_return"), { step: 0.001, placeholder: "none" }), + "fraction of wealth, e.g. 0.002") + : field("Risk aversion λ", input("number", o.risk_aversion, set("risk_aversion"), { min: 0, step: 0.01 }), + "maximise E[P&L] − λ · CVaR; both are money, so λ is the francs of expected return you " + + "give up per franc of CVaR — over a week the interesting range is around 0.01–0.05, " + + "and a large λ keeps everything in cash"); + + return card("Optimisation", "mean-CVaR in integer shares; the model panel on the left decides how the scenarios are made", + el("div", {}, + el("div", { class: "row" }, + field("Mode", segmented([["new", "New portfolio"], ["rebalance", "Rebalance"]], p.mode, (v) => { + p.mode = v; changed("portfolio", "optimize"); rerender(); })), + field("Objective", select([["max_return", "maximise expected return"], ["min_cvar", "minimise CVaR"], + ["utility", "expected return − λ · CVaR"]], o.objective, setShape("objective"))), + field("Confidence β", select([[0.9, "90 %"], [0.95, "95 %"], [0.99, "99 %"]], o.confidence, (v) => set("confidence")(Number(v)))), + field("Solver", select([["auto", "auto (cuOpt, then HiGHS)"], ["cuopt", "cuOpt (GPU)"], ["highs", "HiGHS (CPU)"]], o.solver, set("solver"))), + field("Time limit (s)", input("number", o.time_limit ?? "", set("time_limit"), { min: 1, step: 10, placeholder: "no limit" }), + o.time_limit ? "budget for the whole run; the solver returns its best solution at that point" + : "empty: HiGHS runs until proven optimal - a large problem can take hours")), + limitControls, + el("div", { class: "row" }, + field("Risk measured on", select([["model", "scenarios incl. the return forecast"], + ["zero", "centred scenarios (no drift)"]], o.risk_drift, set("risk_drift")), + o.risk_drift === "model" + ? "over a long horizon a large forecast can push the whole tail into profit" + : "the risk figure does not depend on the return forecast"), + field("Tail rows", select([["auto", "auto"], ["on", "always"], ["off", "never"]], o.row_generation, set("row_generation")), + "solve on the worst scenarios only, then verify"), + el("div", { class: "field" }, + el("label", { text: "Scenario count" }), + segmented([[true, "automatic"], [false, `fixed (${fmt.int(state.model.n_scenarios)})`]], o.auto_scenarios, + (v) => setShape("auto_scenarios")(v === "true" || v === true))), + el("div", { class: "field auto" }, el("label", { text: " " }), el("div", { class: "btnbar" }, runBtn, sweepBtn))), + constraintStrip(), + p.mode === "rebalance" ? holdingsEditor(rerender) : null)); +} + +/** The constraints and costs this run will send -- edited in the Constraints tab, shown here so it + is never a guess which limits are in force. */ +function constraintStrip() { + const c = constraintPayload(), k = state.costs; + const sector = typeof c.max_sector_weight === "object" + ? `${fmt.pct(c.max_sector_weight["*"], 0)} per sector (${Object.keys(c.max_sector_weight).length - 1} overridden)` + : `${fmt.pct(c.max_sector_weight, 0)} per sector`; + const parts = [ + `${fmt.pct(c.max_weight, 0)} per stock`, sector, + c.max_positions ? `at most ${c.max_positions} positions` : "no cardinality limit", + c.min_position_value ? `min position ${fmt.compact(c.min_position_value)} CHF` : null, + `invested ${fmt.pct(c.min_invested, 0)} – ${fmt.pct(c.max_invested, 0)}`, + c.max_turnover !== null ? `turnover ≤ ${fmt.pct(c.max_turnover, 0)}` : null, + c.max_transaction_cost !== null ? `cost ≤ ${fmt.compact(c.max_transaction_cost)} CHF` : null, + `costs ${fmt.bp(k.proportional_buy)} buy / ${fmt.bp(k.proportional_sell)} sell`, + Number(k.fixed_per_trade) ? `+ ${fmt.money(k.fixed_per_trade)} CHF per traded stock` : null, + ].filter(Boolean); + return el("div", { class: "note", style: { marginTop: "10px" } }, + el("b", { text: "Constraints in force: " }), document.createTextNode(parts.join(" · ") + " "), + button("edit", () => document.dispatchEvent(new CustomEvent("quantcore:tab", { detail: "constraints" })), "btn small")); +} + +/** A live "solving …" panel with a cancel button; returns the function that stops the timer. */ +function showProgress(out, body, extra = "", onCancel = null) { + const started = performance.now(); + const secs = el("b", { text: "0 s" }); + const line = el("div", { class: "empty" }, el("span", { class: "spinner" }), + document.createTextNode(" solving … "), secs, + onCancel ? document.createTextNode(" ") : null, + onCancel ? button("Cancel", onCancel, "btn small") : null); + const limit = body.time_limit ? `, time limit ${fmt.int(body.time_limit)} s` : ", no time limit"; + const detail = el("div", { class: "hint", style: { marginTop: "6px" } }, + `${body.scenario_model} · ${body.n_scenarios ? fmt.int(body.n_scenarios) + " scenarios" : "automatic scenario count"} · ` + + `${body.symbols?.length ?? body.max_assets} candidates · solver ${body.solver}${limit}${extra ? " · " + extra : ""}`); + // the service solves one problem at a time (the MILP holds the interpreter), so say so + const queued = el("div", { class: "hint" }, + "The service answers one request at a time while a solve runs - the other tabs wait until this finishes."); + out.replaceChildren(line, detail, queued); + const t = setInterval(() => { secs.textContent = `${((performance.now() - started) / 1000).toFixed(0)} s`; }, 500); + return () => clearInterval(t); +} + +function cancelled(panel) { + setChildren(panel, el("div", { class: "warn-box", text: + "Run cancelled. The server finishes the solve it already started (it stops at the time limit); " + + "the next run is unaffected." })); +} + +const round4 = (v) => Math.round(v * 1e4) / 1e4; // 0.045 * 100 must read as 4.5, not 4.499999 + +function limitHintText() { + const o = state.opt; + return o.cvar_mode === "fraction" + ? `the expected shortfall may not exceed ${fmt.compact((o.cvar_limit_fraction || 0) * wealth())} CHF ` + + `over one ${state.model.horizon}` + : `${fmt.pct((o.cvar_limit || 0) / wealth(), 1)} of wealth over one ${state.model.horizon}`; +} + +function wealth() { + const h = intHoldings(state.portfolio.holdings); + const held = state.portfolio.mode === "rebalance" + ? Object.entries(h).reduce((a, [s, n]) => a + (info(s)?.price || 0) * n, 0) : 0; + return Number(state.portfolio.cash) + held; +} + +function describeRun(body) { + const risk = body.objective === "min_cvar" + ? `min CVaR, E[r] \u2265 ${fmt.pct(body.target_return ?? 0, 3)}` + : body.objective === "utility" ? `E[P&L] \u2212 ${round4(body.risk_aversion)} \u00b7 CVaR` + : body.cvar_limit_fraction ? `CVaR \u2264 ${fmt.pct(body.cvar_limit_fraction, 1)}` + : body.cvar_limit ? `CVaR \u2264 ${fmt.compact(body.cvar_limit)}` : body.objective; + return `${body.mode} · ${body.scenario_model} · ${risk} · β ${fmt.pct(body.confidence, 0)}`; +} + +/** Cash is money: two decimals, not the full float that a valuation produces. */ +function round2(x) { return Math.round(x * 100) / 100; } + +/* ------------------------------------------------------------------ holdings */ +function holdingsEditor(rerender) { + const p = state.portfolio; + const holdings = intHoldings(p.holdings); + const rows = Object.entries(holdings).map(([symbol, shares]) => ({ + symbol, name: info(symbol)?.name || symbol, shares, price: info(symbol)?.price, + value: (info(symbol)?.price || 0) * shares, + })); + const total = rows.reduce((a, r) => a + r.value, 0); + + const addSel = select(["", ...state.selection.filter((s) => !(s in holdings))].map((s) => [s, s || "add a stock …"]), "", + (v) => { if (!v) return; p.holdings = { ...holdings, [v]: 1 }; changed("portfolio", "optimize"); rerender(); }); + + return el("div", { style: { marginTop: "12px" } }, + el("h3", { text: "Current holdings" }), + el("div", { class: "hint", text: `${rows.length} positions, ${fmt.compact(total)} CHF at the last close, plus ${fmt.compact(p.cash)} CHF cash` }), + el("div", { class: "btnbar", style: { margin: "7px 0" } }, + button("Use the shown result", () => { + // the run on screen, not the newest one: with several runs in the history those differ, and + // the weights that get adopted have to be the ones the user is looking at + const r = selectedRun(); + if (!r) { alert("no optimisation has run yet"); return; } + p.holdings = sharesOf(r.result); + // the proposal spent the cash: carry over what it left, or the portfolio would be the new + // positions *plus* the money already invested in them and every weight would come out low + p.cash = round2(Math.max(0, Number(r.result.cash_end ?? p.cash))); + p.mode = "rebalance"; changed("portfolio", "optimize"); rerender(); + }, "btn small", selectedRun() ? `takes the ${selectedRun().result.n_positions} positions of "${selectedRun().label}"` : "run an optimisation first"), + selectedRun() ? el("span", { class: "hint", text: `shown: ${selectedRun().label}` }) : null, + button("Equal weight from cash", () => { + const cash = Number(p.cash); + const each = cash / Math.max(state.selection.length, 1); + p.holdings = Object.fromEntries(state.selection.map((s) => [s, Math.floor(each / (info(s)?.price || 1))]) + .filter(([, n]) => n > 0)); + // likewise: the cash has been turned into shares, only the rounding remainder is left + const spent = Object.entries(p.holdings).reduce((a, [s, n]) => a + (info(s)?.price || 0) * n, 0); + p.cash = round2(Math.max(0, cash - spent)); + changed("portfolio", "optimize"); rerender(); + }, "btn small", "buys equal amounts of the selected stocks and leaves the rounding remainder as cash"), + button("Clear", () => { p.holdings = {}; changed("portfolio", "optimize"); rerender(); }, "btn small"), + addSel), + rows.length ? dataTable([ + { key: "symbol", label: "Symbol", cls: "sym" }, + { key: "name", label: "Name", cls: "name" }, + { key: "price", label: "Price", num: true, fmt: (v) => fmt.money2(v) }, + { key: "shares", label: "Shares", num: true, render: (r) => { + const box = el("input", { type: "number", value: r.shares, min: 0, step: 1, style: { width: "92px", textAlign: "right" } }); + box.addEventListener("change", () => { + const next = { ...intHoldings(p.holdings) }; + const n = Math.max(0, Math.round(Number(box.value) || 0)); + if (n === 0) delete next[r.symbol]; else next[r.symbol] = n; + p.holdings = next; changed("portfolio", "optimize"); + }); + return box; + } }, + { key: "value", label: "Value (CHF)", num: true, fmt: (v) => fmt.money(v) }, + { key: "w", label: "Weight", num: true, render: (r) => document.createTextNode(fmt.pct(r.value / (total + Number(p.cash)), 1)) }, + { key: "del", label: "", render: (r) => button("remove", () => { + const next = { ...intHoldings(p.holdings) }; delete next[r.symbol]; + p.holdings = next; changed("portfolio", "optimize"); rerender(); + }, "btn small") }, + ], rows, { sortKey: "value", sortDir: -1 }) : el("div", { class: "note", text: "no holdings - add stocks or start from a previous result" })); +} + +/* ------------------------------------------------------------------ result */ +function drawResult(out, r) { + const before = r.before; + const ok = r.status === "Optimal" || r.status === "FeasibleFound"; + const tiles = [ + { label: "Status", value: r.status, delta: `${r.solver} · ${fmt.secs(r.timings?.solve_s)}`, + deltaClass: ok ? "muted" : "neg" }, + { label: "Expected P&L", value: fmt.money(r.expected_pnl), delta: `${fmt.pctSigned(r.expected_return, 2)} of wealth`, + deltaClass: r.expected_pnl >= 0 ? "pos" : "neg" }, + { label: `CVaR ${fmt.pct(r.confidence, 0)}`, value: fmt.money(r.cvar), + delta: r.cvar_limit ? `limit ${fmt.money(r.cvar_limit)}` : "no limit", + deltaClass: r.cvar_limit && r.cvar > r.cvar_limit * 1.001 ? "neg" : "muted" }, + { label: `VaR ${fmt.pct(r.confidence, 0)}`, value: fmt.money(r.var) }, + { label: "Invested", value: fmt.money(r.invested), delta: `${fmt.pct(r.invested / r.wealth, 1)} of ${fmt.compact(r.wealth)}` }, + { label: "Positions", value: fmt.int(r.n_positions), delta: `of ${fmt.int(r.n_candidates)} candidates` }, + { label: "Turnover", value: fmt.pct(r.turnover, 1), delta: `cost ${fmt.money(r.total_cost)}` }, + { label: "Reproducible", value: r.reproducibility, delta: r.scenario_hash ? `set ${r.scenario_hash.slice(0, 8)}` : "no scenarios", + deltaClass: r.reproducibility === "exact" ? "pos" : "muted" }, + ]; + + const weightChart = chartBox(), sectorChart = chartBox(); + const positions = r.positions || []; + const parts = [warnBox(r.warnings), statTiles(tiles)]; + + if (before) parts.push(beforeAfterCard(r, before)); + + parts.push(el("div", { class: "grid two", style: { marginTop: "12px" } }, + card("Portfolio weights", before ? "before and after the rebalancing" : "share of wealth per stock", weightChart), + card("Sector exposure", "against the limit that was enforced", sectorChart))); + + parts.push(card("Positions", before ? "shares before and after, with the trades" : "the proposed portfolio", + positionsTable(positions, !!before, r.wealth))); + + parts.push(provenanceCard(r)); + setChildren(out, ...parts); + + const held = positions.filter((p) => p.shares > 0 || p.shares_before > 0) + .sort((a, b) => b.value - a.value).slice(0, 25); + barChart(weightChart, { + items: held.map((p) => ({ label: p.symbol, values: before ? [p.shares_before * p.price / r.wealth, p.weight] : [p.weight] })), + groups: before ? ["before", "after"] : [""], + valueFormat: (v) => fmt.pct(v, 1), horizontal: true, rowHeight: before ? 24 : 20, labelWidth: 84, + references: [{ value: state.constraints.max_weight, label: "cap" }], valueName: "weight", + }); + const sectors = Object.entries(r.sector_exposure || {}).sort((a, b) => b[1] - a[1]); + const cap = Number(state.constraints.max_sector_weight); + barChart(sectorChart, { + items: sectors.map(([sec, v]) => ({ label: sec.length > 20 ? sec.slice(0, 19) + "…" : sec, + values: before ? [before.sector_exposure?.[sec] || 0, v] : [v] })), + groups: before ? ["before", "after"] : [""], valueName: "exposure", + valueFormat: (v) => fmt.pct(v, 0), horizontal: true, rowHeight: before ? 28 : 22, labelWidth: 140, + references: [{ value: cap, label: `sector cap ${fmt.pct(cap, 0)}` }], + }); +} + +function beforeAfterCard(r, before) { + const rows = [ + { k: "Invested (CHF)", a: before.invested, b: r.invested, f: fmt.money }, + { k: "Cash (CHF)", a: before.cash, b: r.cash_end, f: fmt.money }, + { k: "Positions", a: before.n_positions, b: r.n_positions, f: fmt.int }, + { k: "Expected P&L (CHF)", a: before.expected_pnl, b: r.expected_pnl, f: fmt.money }, + { k: "Expected return of wealth", a: before.expected_return, b: r.expected_return, f: (v) => fmt.pctSigned(v, 3) }, + { k: `VaR ${fmt.pct(r.confidence, 0)} (CHF)`, a: before.var, b: r.var, f: fmt.money }, + { k: `CVaR ${fmt.pct(r.confidence, 0)} (CHF)`, a: before.cvar, b: r.cvar, f: fmt.money }, + { k: "CVaR of wealth", a: before.cvar_fraction, b: r.cvar / r.wealth, f: (v) => fmt.pct(v, 2) }, + ].map((row) => ({ ...row, d: row.b - row.a })); + + return card("Before and after", "the starting portfolio measured on the same scenarios as the proposal", + dataTable([ + { key: "k", label: "" }, + { key: "a", label: "Before", num: true, fmt: (v, r2) => r2.f(v) }, + { key: "b", label: "After", num: true, fmt: (v, r2) => r2.f(v) }, + { key: "d", label: "Change", num: true, fmt: (v, r2) => (r2.f === fmt.int ? fmt.signed(v) : r2.f(v)), + cls: (r2) => (Math.abs(r2.d) < 1e-9 ? "muted" : r2.d > 0 ? "pos" : "neg") }, + ], rows, { sortable: false })); +} + +function positionsTable(positions, rebalancing, wealthValue) { + const cols = [ + { key: "symbol", label: "Symbol", cls: "sym" }, + { key: "name", label: "Name", cls: "name" }, + { key: "sector", label: "Sector" }, + { key: "price", label: "Price", num: true, fmt: (v) => fmt.money2(v) }, + ]; + if (rebalancing) cols.push( + { key: "shares_before", label: "Shares before", num: true, fmt: (v) => fmt.int(v) }, + { key: "shares", label: "Shares after", num: true, fmt: (v) => fmt.int(v) }, + { key: "trade", label: "Trade", num: true, fmt: (v) => fmt.signed(v), + cls: (r) => (r.trade > 0 ? "pos" : r.trade < 0 ? "neg" : "muted") }, + { key: "trade_value", label: "Traded (CHF)", num: true, fmt: (v) => fmt.signed(v) }, + ); + else cols.push({ key: "shares", label: "Shares", num: true, fmt: (v) => fmt.int(v) }); + cols.push( + { key: "value", label: "Value (CHF)", num: true, fmt: (v) => fmt.money(v) }, + { key: "weight", label: "Weight", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "cost", label: "Cost (CHF)", num: true, fmt: (v) => fmt.money2(v) }, + { key: "expected_return_ann", label: "E[r] p.a.", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: "expected_return_horizon", label: "E[r] horizon", num: true, fmt: (v) => fmt.pct(v, 2) }, + ); + return dataTable(cols, positions, { sortKey: "value", sortDir: -1 }); +} + +function provenanceCard(r) { + const m = r.model || {}, s = r.solver_info || {}; + const scen = m.scenarios || {}; + const rg = s.row_generation; + return card("How this number was produced", "everything needed to reproduce the run", + el("div", { class: "grid three" }, + el("div", {}, el("h3", { text: "Scenarios" }), + kv([ + ["model", scen.model], ["scenarios", fmt.int(scen.n_scenarios)], ["sampling", m.sampling?.sampling], + ["moment matching", String(m.sampling?.moment_match)], ["tail tilt", m.sampling?.importance_sampling], + ["effective sample", scen.effective_sample_size ? fmt.int(scen.effective_sample_size) : null], + ["risk measured on", m.sampling?.risk_drift === "zero" ? "centred scenarios" : "scenarios incl. the forecast"], + ["tilt alignment", Number.isFinite(s.tilt_alignment) ? fmt.num(s.tilt_alignment, 2) : null], + ["scenario set", r.scenario_hash], + ["reused from cache", r.solver_info?.scenario_cache ? (r.solver_info.scenario_cache.reused ? "yes" : "no") : null], + ["as of", scen.as_of], ["seed", scen.seed], + ])), + el("div", {}, el("h3", { text: "Solver" }), + kv([ + ["solver", r.solver], ["status", r.status], ["objective", Number.isFinite(s.objective) ? fmt.money2(s.objective) : null], + ["MIP gap", Number.isFinite(s.mip_gap) ? s.mip_gap.toExponential(1) : s.mip_gap], + ["variables", fmt.int(m.milp?.n_vars)], + ["rows solved", rg ? `${fmt.int(rg.rows_final)} of ${fmt.int(scen.n_scenarios)}` : null], + ["row-generation passes", rg?.iterations], ["cuts (Gaussian)", s.outer_approximation?.iterations], + ["solve time", fmt.secs(s.solve_time_s)], ["total", fmt.secs(r.timings?.total_s)], + ["reproducibility", r.reproducibility], + ])), + el("div", {}, el("h3", { text: "Constraints used" }), + kv(constraintRows(m.constraints || {}, m.costs || {}))), + el("div", {}, el("h3", { text: "Expected returns" }), + kv([ + ["model", m.expected_returns?.type], ["market return p.a.", m.expected_returns?.market_return_ann ? fmt.pct(m.expected_returns.market_return_ann, 2) : null], + ["δ", m.expected_returns?.delta ? fmt.num(m.expected_returns.delta, 2) : null], + ["τ", m.expected_returns?.tau], ["views", m.expected_returns?.views?.length ?? 0], + ["drift convention", m.sampling?.drift_convention], + ["covariance", `${scen.covariance_model?.method ?? ""} ${scen.covariance_model?.lam ?? scen.covariance_model?.window ?? ""}`], + ["shrinkage", scen.shrinkage_applied ? fmt.pct(scen.shrinkage_applied, 1) : null], + ])))); +} + +function constraintRows(c, k) { + const sector = c.max_sector_weight && typeof c.max_sector_weight === "object" + ? Object.entries(c.max_sector_weight).map(([sec, v]) => `${sec === "*" ? "all" : sec} ${fmt.pct(v, 0)}`).join(", ") + : fmt.pct(c.max_sector_weight, 0); + return [ + ["max per stock", fmt.pct(c.max_weight, 1)], + ["max per sector", sector], + ["max positions", c.max_positions ?? "no limit"], + ["min position", c.min_position_value ? `${fmt.money(c.min_position_value)} CHF` : "none"], + ["invested", `${fmt.pct(c.min_invested, 0)} – ${fmt.pct(c.max_invested, 0)}`], + ["max turnover", c.max_turnover === null || c.max_turnover === undefined ? "no limit" : fmt.pct(c.max_turnover, 0)], + ["max cost", c.max_transaction_cost === null || c.max_transaction_cost === undefined + ? "no limit" : `${fmt.money(c.max_transaction_cost)} CHF`], + ["trading cost", `${fmt.bp(k.proportional_buy)} buy / ${fmt.bp(k.proportional_sell)} sell`], + ["fixed per trade", k.fixed_per_trade ? `${fmt.money(k.fixed_per_trade)} CHF` : "none"], + ]; +} + +/* ------------------------------------------------------------------ history */ +function historyCard(rerender) { + const runs = allRuns(); + if (!runs.length) return el("span"); + const rows = runs.map((run) => ({ + id: run.id, label: run.label, status: run.result.status, + pnl: run.result.expected_pnl, ret: run.result.expected_return, cvar: run.result.cvar, + cvar_w: run.result.cvar / run.result.wealth, n: run.result.n_positions, + turnover: run.result.turnover, solver: run.result.solver, secs: run.result.timings?.total_s, + hash: run.result.scenario_hash?.slice(0, 8) || "–", + })); + const frontier = chartBox(); + const box = card("Runs in this session", rows.length > 1 + ? "click a row to show that run and make it the one \u201cUse the shown result\u201d adopts; the chart is the risk/return trade-off of the runs" + : "click a row to show that run again", + el("div", {}, + el("div", { class: "btnbar", style: { marginBottom: "8px" } }, + button("Clear history", () => { clearRuns(); rerender(); }, "btn small")), + rows.length > 1 ? frontier : null, + dataTable([ + { key: "label", label: "Run" }, + { key: "status", label: "Status", cls: (r) => (r.status === "Optimal" ? "pos" : "muted") }, + { key: "pnl", label: "E[P&L]", num: true, fmt: (v) => fmt.money(v) }, + { key: "ret", label: "E[return]", num: true, fmt: (v) => fmt.pctSigned(v, 3) }, + { key: "cvar", label: "CVaR", num: true, fmt: (v) => fmt.money(v) }, + { key: "cvar_w", label: "CVaR of wealth", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "n", label: "Positions", num: true, fmt: (v) => fmt.int(v) }, + { key: "turnover", label: "Turnover", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: "solver", label: "Solver" }, + { key: "secs", label: "Time", num: true, fmt: (v) => fmt.secs(v) }, + { key: "hash", label: "Scenario set", cls: "mono" }, + ], rows, { sortable: false, rowClass: (row) => (isSelected(row.id) ? "selected" : ""), + onRowClick: (row) => { selectRun(row.id); rerender(); } }))); + if (rows.length > 1) scatterChart(frontier, { + points: rows.filter((r) => r.status !== "Infeasible").map((r) => ({ x: r.cvar_w, y: r.ret, label: r.label })), + height: 240, xLabel: "CVaR of wealth", yLabel: "expected return over the horizon", + xFormat: (v) => fmt.pct(v, 1), yFormat: (v) => fmt.pct(v, 2), labelPoints: false, + }); + return box; +} diff --git a/quantcore/api/ui/js/tabs/risk.js b/quantcore/api/ui/js/tabs/risk.js new file mode 100644 index 0000000..1690739 --- /dev/null +++ b/quantcore/api/ui/js/tabs/risk.js @@ -0,0 +1,299 @@ +/* Tab 2 -- risk models: the covariance the optimiser sees and the scenarios it is given. */ + +import { + api, button, card, chartBox, dataTable, el, empty, field, fmt, kv, live, scenarioPayload, + segmented, select, setChildren, state, statTiles, warnBox, withBusy, +} from "../core.js"; +import { barChart, heatmap, histogram } from "../charts.js"; +import { info } from "../universe.js"; +import { selectedRun, sharesOf } from "../runs.js"; + +let covResult = null, scenResult = null, noise = null; +let basis = "cap"; // which portfolio the distribution is measured for +let focus = ""; // asset shown as a single-stock distribution +let space = "simple"; // 'simple' (price returns) | 'log' (the modelling space) + +export function render(node) { + node.replaceChildren(); + if (!state.selection.length) { node.appendChild(empty("select candidate stocks in the Stocks tab first")); return; } + node.append(covarianceCard(), scenarioCard()); +} + +/* ------------------------------------------------------------------ covariance / correlation */ +function covarianceCard() { + const out = el("div", { id: "risk-cov" }); + let btn; + const run = async () => { + await withBusy(btn, live("risk-cov", out), async () => { + covResult = await api.post("/model/covariance", { + symbols: state.selection, freq: state.model.freq, horizon: state.model.horizon, + covariance: scenarioPayload().covariance, + }); + drawCovariance(live("risk-cov", out), covResult); + }).catch(() => {}); + }; + btn = button("Estimate covariance", run, "btn primary"); + const body = el("div", {}, el("div", { class: "btnbar", style: { marginBottom: "10px" } }, btn, + el("span", { class: "muted", text: `${state.selection.length} stocks · ${state.model.covariance.method} · shrinkage ${state.model.covariance.shrinkage}` })), out); + if (covResult) drawCovariance(out, covResult); else out.appendChild(empty("estimate the covariance to see correlations, volatilities and conditioning")); + return card("Covariance model", "the estimator the scenario generator and the optimiser use", body); +} + +function drawCovariance(out, r) { + const c = r.conditioning; + const warnings = []; + if (c.rank_deficient) warnings.push(`The estimate is singular: ${c.effective_observations} effective observations for ${c.n_assets} assets. Some portfolios look riskless to the optimiser - use shrinkage 'auto' or a longer window.`); + else if (c.under_determined) warnings.push(`Fewer effective observations (${c.effective_observations}) than assets (${c.n_assets}); shrinkage of ${fmt.pct(c.shrinkage_applied, 0)} keeps the smallest eigenvalue at ${fmt.pct(c.min_eigenvalue_ratio, 1)} of the average variance.`); + + const labels = r.correlation.columns; + const vols = labels.map((s) => ({ label: s, values: [r.volatility_ann[s]] })) + .sort((a, b) => b.values[0] - a.values[0]).slice(0, 25); + const eig = r.eigenvalues.slice(0, Math.min(20, r.eigenvalues.length)) + .map((v, i) => ({ label: `${i + 1}`, values: [v] })); + + const hm = chartBox(), volBox = chartBox(), eigBox = chartBox(); + setChildren(out, + warnBox(warnings), + statTiles([ + { label: "Assets", value: fmt.int(c.n_assets) }, + { label: "Observations", value: fmt.int(r.observations) }, + { label: "Effective obs.", value: fmt.int(c.effective_observations), delta: `${state.model.covariance.method}` }, + { label: "Shrinkage", value: fmt.pct(r.shrinkage_applied, 1), delta: "towards the diagonal" }, + { label: "Mean correlation", value: fmt.num(r.correlation_summary.mean, 2), + delta: `${fmt.num(r.correlation_summary.min, 2)} … ${fmt.num(r.correlation_summary.max, 2)}` }, + { label: "Condition number", value: fmt.compact(c.condition_number) }, + { label: "Min eigenvalue", value: fmt.pct(c.min_eigenvalue_ratio, 2), delta: "of the average variance", + deltaClass: c.rank_deficient ? "neg" : "muted" }, + { label: "Horizon", value: `${r.horizon_periods} × ${r.freq.slice(0, 1)}` }, + ]), + el("div", { class: "grid two", style: { marginTop: "12px" } }, + card("Correlation", labels.length > 26 ? `${labels.length} stocks (labels omitted above 26)` : "of the model's returns", hm), + el("div", {}, + card("Volatility p.a.", "the 25 most volatile of the selection", volBox), + card("Eigenvalue spectrum", "share of total variance per principal component", eigBox))), + ); + heatmap(hm, { labels, matrix: r.correlation.data, domain: [-1, 1], valueName: "correlation" }); + barChart(volBox, { items: vols, valueFormat: (v) => fmt.pct(v, 0), horizontal: true, rowHeight: 18, + labelWidth: 84, valueName: "volatility p.a." }); + barChart(eigBox, { items: eig, valueFormat: (v) => fmt.pct(v, 0), horizontal: false, height: 190, + valueName: "share of variance", bottom: 26 }); +} + +/* ------------------------------------------------------------------ scenarios */ +function exposureFor(kind) { + const sel = state.selection; + const held = Object.entries(state.portfolio.holdings || {}) + .reduce((a, [sym, n]) => a + (info(sym)?.price || 0) * Math.round(Number(n) || 0), 0); + const wealth = (Number(state.portfolio.cash) || 0) + (state.portfolio.mode === "rebalance" ? held : 0) || 1e6; + if (kind === "holdings") return { holdings: state.portfolio.holdings, value: null }; + if (kind === "result") { + const r = selectedRun(); // the run shown in the Optimisation tab, not the newest + return r ? { holdings: sharesOf(r.result), value: null } : null; + } + if (kind === "equal") { + const w = Object.fromEntries(sel.map((s) => [s, 1 / sel.length])); + return { weights: w, value: wealth }; + } + const caps = sel.map((s) => info(s)?.market_cap || 0); + const total = caps.reduce((a, b) => a + b, 0) || 1; + return { weights: Object.fromEntries(sel.map((s, i) => [s, caps[i] / total])), value: wealth }; +} + +function scenarioCard() { + const out = el("div", { id: "risk-scen" }); + let btn, noiseBtn; + + const run = async () => { + await withBusy(btn, live("risk-scen", out), async () => { + const exp = exposureFor(basis); + if (!exp) throw new Error("no optimisation result yet - run one in the Optimisation tab"); + const body = scenarioPayload({ bins: 60, confidence: [0.95, 0.99], focus: focus ? [focus] : [] }); + if (exp.holdings) Object.assign(body, { holdings: exp.holdings, cash: 0 }); + else Object.assign(body, { weights: exp.weights, value: exp.value }); + scenResult = await api.post("/model/scenarios", body); + drawScenarios(live("risk-scen", out), scenResult); + }).catch(() => {}); + }; + + const noiseCheck = async () => { + await withBusy(noiseBtn, live("risk-scen", out), async () => { + const exp = exposureFor(basis); + const seeds = [0, 1, 2].map((d) => Number(state.model.seed) + d); + const results = []; + for (const seed of seeds) { + const body = scenarioPayload({ seed, bins: 20, confidence: [0.95, 0.99] }); + if (exp.holdings) Object.assign(body, { holdings: exp.holdings, cash: 0 }); + else Object.assign(body, { weights: exp.weights, value: exp.value }); + results.push(await api.post("/model/scenarios", body)); + } + noise = { seeds, results }; + drawScenarios(live("risk-scen", out), scenResult, noise); + }).catch(() => {}); + }; + + btn = button("Generate scenarios", run, "btn primary"); + noiseBtn = button("Sampling-noise check (3 seeds)", noiseCheck, "btn"); + + const body = el("div", {}, + el("div", { class: "row", style: { marginBottom: "10px" } }, + field("Measured portfolio", select([ + ["cap", "cap-weighted selection"], ["equal", "equally weighted selection"], + ["holdings", "current holdings"], ["result", "the optimisation result on screen"]], basis, (v) => { basis = v; run(); })), + field("Single-stock view", select([["", "none"], ...state.selection.map((s) => [s, s])], focus, (v) => { focus = v; run(); })), + el("div", { class: "field auto" }, el("label", { text: " " }), el("div", { class: "btnbar" }, btn, noiseBtn))), + out); + if (scenResult) drawScenarios(out, scenResult, noise); + else out.appendChild(empty("generate the scenario set to see the horizon distribution of the portfolio and of each stock")); + return card("Scenarios", `${state.model.scenario_model === "gaussian_exact" ? "GBM (the exact Gaussian mode draws no scenarios)" : state.model.scenario_model} · ${state.model.sampling} sampling · ${state.model.horizon} horizon`, body); +} + +function drawScenarios(out, r, noiseData) { + if (!r) return; + const d = r.description, p = r.portfolio; + const tiles = [ + { label: "Scenarios", value: fmt.int(d.n_scenarios), delta: d.weighted ? "probability weighted" : "equiprobable" }, + { label: "Effective sample", value: fmt.int(d.effective_sample_size), + delta: d.weighted ? `${fmt.pct(d.effective_sample_size / d.n_scenarios, 0)} of the draws` : "no tilt" }, + { label: "Scenario set", value: d.hash?.slice(0, 8) || "–", delta: "same inputs ⇒ same set", title: d.hash }, + { label: "Shocks", value: d.shock_hash?.slice(0, 8) || "–", delta: "drift removed" }, + ]; + if (p) { + tiles.push( + { label: "Expected P&L", value: fmt.money(p.expected_pnl), delta: fmt.pctSigned(p.expected_pnl / p.value, 2), + deltaClass: p.expected_pnl >= 0 ? "pos" : "neg" }, + { label: "P&L volatility", value: fmt.money(p.volatility) }, + { label: "VaR 95 %", value: fmt.money(p.risk["0.95"].var) }, + { label: "CVaR 95 %", value: fmt.money(p.risk["0.95"].cvar), delta: fmt.pct(p.risk["0.95"].cvar / p.value, 2) }, + { label: "CVaR 99 %", value: fmt.money(p.risk["0.99"].cvar), delta: fmt.pct(p.risk["0.99"].cvar / p.value, 2) }, + { label: "Skewness", value: fmt.num(p.skewness, 2), delta: `kurtosis ${fmt.num(p.kurtosis, 1)}` }); + } + + const hist = chartBox(280), assetBox = chartBox(240); + const parts = [statTiles(tiles)]; + + if (p) { + parts.push(el("div", { class: "grid two", style: { marginTop: "12px" } }, + card(`P&L over one ${state.model.horizon}`, `${basisLabel()} · ${fmt.compact(p.value)} CHF`, hist, + el("div", { class: "note", style: { marginTop: "6px" } }, + `Losses beyond the 95 % VaR are shaded; CVaR is their probability-weighted average.`)), + card("Quantiles of the P&L", "probability-weighted, in CHF", quantileTable(p)))); + } + if (r.asset_histograms && Object.keys(r.asset_histograms).length) { + const sym = Object.keys(r.asset_histograms)[0]; + parts.push(card(`${sym} — simple return over one ${state.model.horizon}`, info(sym)?.name || "", assetBox)); + } + if (noiseData) parts.push(noiseCard(noiseData)); + parts.push(summaryCard(r)); + parts.push(card("Provenance", "the inputs this scenario set depends on - any run with the same key reuses it", + kv(Object.entries(r.scenario_key || {}).map(([k, v]) => [ + k, Array.isArray(v) ? `${v.length}: ${v.slice(0, 8).join(", ")}${v.length > 8 ? " …" : ""}` + : typeof v === "object" && v !== null ? Object.entries(v).map(([a, b]) => `${a} ${b}`).join(", ") + : String(v)])))); + + setChildren(out, ...parts); + + if (p) { + histogram(hist, { + edges: p.histogram.edges, mass: p.histogram.mass, height: 280, + valueFormat: (v) => fmt.compact(v), tailFrom: -p.risk["0.95"].var, + mainLabel: "P&L outcomes", tailLabel: "worst 5 %", + markers: [ + { value: -p.risk["0.95"].var, label: `VaR 95 % ${fmt.compact(p.risk["0.95"].var)}` }, + { value: -p.risk["0.95"].cvar, label: `CVaR 95 % ${fmt.compact(p.risk["0.95"].cvar)}` }, + { value: p.expected_pnl, label: `mean ${fmt.compact(p.expected_pnl)}` }, + ], + }); + } + if (r.asset_histograms) { + const sym = Object.keys(r.asset_histograms)[0]; + if (sym) { + const h = r.asset_histograms[sym]; + histogram(assetBox, { edges: h.edges, mass: h.mass, height: 240, valueFormat: (v) => fmt.pct(v, 0), + mainLabel: `${sym} outcomes` }); + } + } +} + +function basisLabel() { + return { cap: "cap-weighted selection", equal: "equally weighted selection", holdings: "current holdings", + result: "the optimisation result on screen" }[basis]; +} + +function quantileTable(p) { + const rows = Object.entries(p.quantiles).map(([q, v]) => ({ q: `${(Number(q) * 100).toFixed(0)} %`, v, r: v / p.value })); + rows.push({ q: "mean", v: p.expected_pnl, r: p.expected_pnl / p.value }); + rows.push({ q: "worst", v: p.min, r: p.min / p.value }); + rows.push({ q: "best", v: p.max, r: p.max / p.value }); + return dataTable([ + { key: "q", label: "Quantile" }, + { key: "v", label: "P&L (CHF)", num: true, fmt: (v) => fmt.money(v), cls: (r) => (r.v >= 0 ? "pos" : "neg") }, + { key: "r", label: "of wealth", num: true, fmt: (v) => fmt.pctSigned(v, 2) }, + ], rows, { sortable: false }); +} + +/** Per-stock scenario table. Price returns by default: a log-return of -2.0 over a quarter is a + price return of -86 %, and only the latter is what a position can actually lose. */ +function summaryCard(r) { + const slot = el("div"); + const fill = () => setChildren(slot, summaryTable(r)); + fill(); + const head = el("div", { class: "row", style: { marginBottom: "8px" } }, + field("Shown as", segmented([["simple", "price returns"], ["log", "log-returns"]], space, + (v) => { space = v; fill(); }))); + return card("Scenario set per stock", + space === "simple" + ? `probability-weighted moments and quantiles of the horizon price return (e^x − 1, never below −100 %)` + : `probability-weighted moments and quantiles of the horizon log-return (the space the model works in)`, + head, slot); +} + +function summaryTable(r) { + const rows = r.summary.map((s) => ({ ...s, mu: r.expected_returns_ann?.[s.symbol] })); + const k = (name) => (space === "simple" ? `${name}_simple` : name); + const cols = [ + { key: "symbol", label: "Symbol", cls: "sym" }, + { key: "mu", label: "E[r] p.a.", num: true, fmt: (v) => fmt.pct(v, 1), title: "the expected-return model" }, + ]; + if (space === "simple") { + cols.push({ key: "expected_simple", label: "E[r] horizon", num: true, fmt: (v) => fmt.pct(v, 2), + title: "the model's expected price return over the horizon" }); + } + cols.push( + { key: k("mean"), label: space === "simple" ? "scenario mean" : "mean log", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: k("std"), label: "sd", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: k("q05"), label: "5 %", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: k("q50"), label: "median", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: k("q95"), label: "95 %", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: k("min"), label: "worst", num: true, fmt: (v) => fmt.pct(v, 1), cls: "neg" }, + { key: k("max"), label: "best", num: true, fmt: (v) => fmt.pct(v, 1), cls: "pos" }, + ); + return dataTable(cols, rows, { sortKey: k("std"), sortDir: -1 }); +} + +function noiseCard(nd) { + const rows = nd.results.map((r, i) => ({ + seed: nd.seeds[i], pnl: r.portfolio.expected_pnl, vol: r.portfolio.volatility, + cvar95: r.portfolio.risk["0.95"].cvar, cvar99: r.portfolio.risk["0.99"].cvar, hash: r.description.hash.slice(0, 8), + })); + const spread = (key) => { + const v = rows.map((r) => r[key]); + const mean = v.reduce((a, b) => a + b, 0) / v.length; + const sd = Math.sqrt(v.reduce((a, b) => a + (b - mean) ** 2, 0) / (v.length - 1)); + return { mean, sd, rel: sd / Math.abs(mean) }; + }; + const c95 = spread("cvar95"), c99 = spread("cvar99"); + return card("Sampling noise across seeds", "the same model and portfolio, three independent scenario sets", + el("div", {}, + statTiles([ + { label: "CVaR 95 % spread", value: fmt.pct(c95.rel, 2), delta: `sd ${fmt.money(c95.sd)} of ${fmt.money(c95.mean)}` }, + { label: "CVaR 99 % spread", value: fmt.pct(c99.rel, 2), delta: `sd ${fmt.money(c99.sd)} of ${fmt.money(c99.mean)}` }, + ]), + el("div", { style: { marginTop: "10px" } }, dataTable([ + { key: "seed", label: "Seed", num: true, fmt: (v) => fmt.int(v) }, + { key: "hash", label: "Scenario set", cls: "mono" }, + { key: "pnl", label: "E[P&L]", num: true, fmt: (v) => fmt.money(v) }, + { key: "vol", label: "sd of P&L", num: true, fmt: (v) => fmt.money(v) }, + { key: "cvar95", label: "CVaR 95 %", num: true, fmt: (v) => fmt.money(v) }, + { key: "cvar99", label: "CVaR 99 %", num: true, fmt: (v) => fmt.money(v) }, + ], rows, { sortable: false })))); +} diff --git a/quantcore/api/ui/js/tabs/stocks.js b/quantcore/api/ui/js/tabs/stocks.js new file mode 100644 index 0000000..54b9fc5 --- /dev/null +++ b/quantcore/api/ui/js/tabs/stocks.js @@ -0,0 +1,217 @@ +/* Tab 1 -- stock data: pick the candidate stocks and look at what the models are fed. */ + +import { + api, button, card, changed, chartBox, dataTable, el, empty, errorBox, field, fmt, input, + live, parseDate, segmented, select, setChildren, state, withBusy, +} from "../core.js"; +import { barChart, lineChart, scatterChart } from "../charts.js"; +import { filterRows, info, universe } from "../universe.js"; + +const RANGES = [["1y", 1], ["3y", 3], ["5y", 5], ["10y", 10], ["max", 0]]; +let range = "5y"; +let analysis = null; // {prices, risk, perf} of the last fetch + +export async function render(node) { + node.replaceChildren(el("div", { class: "empty" }, el("span", { class: "spinner" }), document.createTextNode(" loading the universe …"))); + let uni; + try { uni = await universe(); } catch (err) { node.replaceChildren(errorBox(err)); return; } + node.replaceChildren(); + node.append(pickerCard(uni), analysisCard()); +} + +/* ------------------------------------------------------------------ stock picker */ +function pickerCard(uni) { + const u = state.universe; + const body = el("div"); + const tableSlot = el("div"); + const summary = el("div", { class: "row", style: { marginBottom: "8px" } }); + + const draw = () => { + const rows = filterRows(uni.rows, { region: u.region, sector: u.sector, country: u.country, search: u.search }); + const selected = new Set(state.selection); + summary.replaceChildren( + el("span", { class: "muted", text: `${rows.length} of ${uni.rows.length} stocks match · ${selected.size} selected` }), + el("span", { class: "spacer", style: { marginLeft: "auto" } }), + el("div", { class: "btnbar" }, + button(`Select top ${u.top}`, () => { + state.selection = rows.slice(0, Number(u.top)).map((r) => r.symbol); + changed("selection", "stocks"); draw(); + }, "btn small primary"), + button("Add all matching", () => { + const add = rows.slice(0, 500).map((r) => r.symbol); + state.selection = [...new Set([...state.selection, ...add])]; + changed("selection", "stocks"); draw(); + }, "btn small"), + button("Clear", () => { state.selection = []; changed("selection", "stocks"); draw(); }, "btn small"))); + + tableSlot.replaceChildren(dataTable([ + { key: "selected", label: "", render: (r) => { + const box = el("input", { type: "checkbox" }); + box.checked = selected.has(r.symbol); + box.addEventListener("change", () => { + state.selection = box.checked + ? [...new Set([...state.selection, r.symbol])] + : state.selection.filter((s) => s !== r.symbol); + changed("selection", "stocks"); + draw(); + }); + return box; + } }, + { key: "symbol", label: "Symbol", cls: "sym" }, + { key: "name", label: "Name", cls: "name" }, + { key: "sector", label: "Sector" }, + { key: "country", label: "Country" }, + { key: "market_cap", label: "Market cap (CHF)", num: true, fmt: (v) => fmt.compact(v) }, + { key: "market_weight", label: "Weight", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "price", label: "Price (CHF)", num: true, fmt: (v) => fmt.money2(v) }, + { key: "price_coverage", label: "Coverage", num: true, fmt: (v) => fmt.pct(v, 1) }, + ], rows, { sortKey: "market_cap", sortDir: -1, limit: 400 })); + }; + + const sectors = ["", ...Object.keys(uni.facets.sectors)]; + const countries = ["", ...Object.keys(uni.facets.countries).sort()]; + body.append( + el("div", { class: "row" }, + field("Region", select([["", "world"], ...Object.keys(uni.facets.regions).map((r) => [r, r])], u.region, + (v) => { u.region = v; changed("universe", "stocks"); draw(); })), + field("Sector", select(sectors.map((s) => [s, s || "all sectors"]), u.sector, + (v) => { u.sector = v; changed("universe", "stocks"); draw(); })), + field("Country", select(countries.map((s) => [s, s || "all countries"]), u.country || "", + (v) => { u.country = v; changed("universe", "stocks"); draw(); })), + field("Search", input("search", u.search, (v) => { u.search = v; changed("universe", "stocks"); draw(); }, + { placeholder: "symbol or name" }))), + summary, tableSlot); + draw(); + return card("Investment universe", `${uni.n_universe} stocks, market caps as of ${fmt.date(uni.as_of)}`, body); +} + +/* ------------------------------------------------------------------ time series and statistics */ +function analysisCard() { + const body = el("div"); + const out = el("div", { id: "stocks-analysis" }); + const controls = el("div", { class: "row", style: { marginBottom: "10px" } }); + let btn; + + const run = async () => { + const symbols = state.selection; + const panel = live("stocks-analysis", out); + if (!symbols.length) { panel.replaceChildren(empty("select stocks above to see prices, risk and performance")); return; } + await withBusy(btn, panel, async () => { + const years = RANGES.find(([k]) => k === range)?.[1] ?? 5; + const start = years ? new Date(Date.now() - years * 365.25 * 864e5).toISOString().slice(0, 10) : null; + const list = symbols.slice(0, 150).join(","); + const [prices, risk, perf] = await Promise.all([ + api.get("/prices", { symbols: list, start, freq: state.data.freq, currency: "base" }), + api.post("/risk/assets", { symbols: symbols.slice(0, 150), freq: state.data.freq, horizon: state.model.horizon, + start, currency: "base", space: "value", drift: "zero" }), + api.post("/performance/assets", { symbols: symbols.slice(0, 150), freq: state.data.freq, start, currency: "base" }), + ]); + analysis = { prices, risk, perf }; + drawAnalysis(live("stocks-analysis", panel), prices, risk, perf); + }).catch(() => {}); + }; + + btn = button("Load data", run, "btn primary"); + controls.append( + field("History", segmented(RANGES.map(([k]) => [k, k]), range, (v) => { range = v; run(); })), + field("Frequency", select([["daily", "daily"], ["weekly", "weekly"], ["monthly", "monthly"]], state.data.freq, + (v) => { state.data.freq = v; changed("data", "stocks"); run(); })), + el("div", { class: "field auto" }, el("label", { text: " " }), btn)); + body.append(controls, out); + out.appendChild(empty("select stocks above, then load the data")); + if (state.selection.length && analysis) drawAnalysis(out, analysis.prices, analysis.risk, analysis.perf); + return card("Time series, risk and performance", "prices in CHF; log-returns; the window is the range above", body); +} + +function drawAnalysis(out, prices, risk, perf) { + const { index, columns, data } = prices; + const x = index.map(parseDate); + const shown = [...columns].sort((a, b) => (info(b)?.market_cap ?? 0) - (info(a)?.market_cap ?? 0)).slice(0, 8); + + // indexed prices (a common base is the only honest way to put several stocks on one axis) + const series = shown.map((c) => { + const j = columns.indexOf(c); + const col = data.map((row) => row[j]); + const base = col.find((v) => Number.isFinite(v)) ?? 1; + return { name: c, points: col.map((v, i) => [x[i], Number.isFinite(v) ? (v / base) * 100 : NaN]) }; + }); + const dd = shown.map((c) => { + const j = columns.indexOf(c); + let peak = -Infinity; + const pts = []; + data.forEach((row, i) => { + const v = row[j]; + if (!Number.isFinite(v)) return; + peak = Math.max(peak, v); + pts.push([x[i], v / peak - 1]); + }); + return { name: c, points: pts }; + }); + + const perfBy = Object.fromEntries(perf.map((p) => [p.symbol, p])); + const riskBy = Object.fromEntries(risk.map((p) => [p.symbol, p])); + const caps = state.selection.map((s) => ({ label: s, values: [info(s)?.market_cap ?? NaN] })) + .filter((r) => Number.isFinite(r.values[0])).sort((a, b) => b.values[0] - a.values[0]).slice(0, 20); + + const priceChart = chartBox(260), ddChart = chartBox(220), capChart = chartBox(), scatter = chartBox(280); + setChildren(out, + el("div", { class: "grid two" }, + card("Price, indexed to 100", `${shown.length} of ${columns.length} stocks · ${index.length} ${state.data.freq} observations`, priceChart), + card("Drawdown from the running peak", "same stocks", ddChart)), + el("div", { class: "grid two" }, + card("Risk and return", `annualised, from ${fmt.date(index[0])}`, scatter), + card("Market capitalisation", "the 20 largest of the selection, CHF", capChart)), + card("Per-stock statistics", + `risk over a ${state.model.horizon} horizon, ${state.data.freq} data; VaR and CVaR as a share of the ` + + `position's value, volatility only (no drift extrapolated into the horizon)`, + statsTable(state.selection, riskBy, perfBy)), + ); + + lineChart(priceChart, { series, height: 260, xFormat: (v) => new Date(v).toISOString().slice(0, 7), + yFormat: (v) => fmt.num(v, 0) }); + lineChart(ddChart, { series: dd, height: 220, xFormat: (v) => new Date(v).toISOString().slice(0, 7), + yFormat: (v) => fmt.pct(v, 0), area: dd.length === 1 }); + barChart(capChart, { items: caps, valueFormat: (v) => fmt.compact(v), horizontal: true, rowHeight: 20, + labelWidth: 84, valueName: "market cap" }); + scatterChart(scatter, { + points: state.selection.filter((s) => perfBy[s]).map((s) => ({ + x: perfBy[s].annualised_volatility, y: perfBy[s].annualised_return, label: s })), + height: 280, xLabel: "volatility p.a.", yLabel: "return p.a.", + xFormat: (v) => fmt.pct(v, 0), yFormat: (v) => fmt.pct(v, 0), labelPoints: state.selection.length <= 14, + }); +} + +function statsTable(symbols, riskBy, perfBy) { + const rows = symbols.filter((s) => riskBy[s] || perfBy[s]).map((s) => ({ + symbol: s, + vol: perfBy[s]?.annualised_volatility, + vol_ewm: riskBy[s]?.vol_ewm_ann, + ret: perfBy[s]?.annualised_return, + sharpe: perfBy[s]?.sharpe, + mdd: perfBy[s]?.max_drawdown, + skew: perfBy[s]?.skewness, + kurt: perfBy[s]?.excess_kurtosis, + var95: riskBy[s]?.var95_normal_ewm, + cvar95: riskBy[s]?.cvar95_normal_ewm, + cvar95_hist: riskBy[s]?.cvar95_hist, + cvar99_hist: riskBy[s]?.cvar99_hist, + n: perfBy[s]?.n_obs, + })); + return dataTable([ + { key: "symbol", label: "Symbol", cls: "sym" }, + { key: "ret", label: "Return p.a.", num: true, fmt: (v) => fmt.pct(v, 1), cls: (r) => (r.ret >= 0 ? "pos" : "neg") }, + { key: "vol", label: "Vol p.a.", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: "vol_ewm", label: "Vol p.a. (EWM)", num: true, fmt: (v) => fmt.pct(v, 1), + title: "exponentially weighted: the volatility the models use today" }, + { key: "sharpe", label: "Sharpe", num: true, fmt: (v) => fmt.num(v, 2) }, + { key: "mdd", label: "Max drawdown", num: true, fmt: (v) => fmt.pct(v, 0), cls: "neg" }, + { key: "skew", label: "Skew", num: true, fmt: (v) => fmt.num(v, 2) }, + { key: "kurt", label: "Excess kurtosis", num: true, fmt: (v) => fmt.num(v, 1) }, + { key: "var95", label: "VaR 95 %", num: true, fmt: (v) => fmt.pct(v, 2), + title: "normal, EWM volatility, over the horizon; the loss as a share of the position's value" }, + { key: "cvar95", label: "CVaR 95 %", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "cvar95_hist", label: "CVaR 95 % hist.", num: true, fmt: (v) => fmt.pct(v, 2), title: "empirical, same horizon" }, + { key: "cvar99_hist", label: "CVaR 99 % hist.", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "n", label: "Obs.", num: true, fmt: (v) => fmt.int(v) }, + ], rows, { sortKey: "vol", sortDir: -1 }); +} diff --git a/quantcore/api/ui/js/tabs/views.js b/quantcore/api/ui/js/tabs/views.js new file mode 100644 index 0000000..154aa55 --- /dev/null +++ b/quantcore/api/ui/js/tabs/views.js @@ -0,0 +1,172 @@ +/* Tab 3 -- expected returns: the implied (equilibrium) returns, the views and the posterior. */ + +import { + api, button, card, changed, chartBox, dataTable, el, empty, field, fmt, input, live, select, setChildren, + state, statTiles, viewPayload, withBusy, +} from "../core.js"; +import { barChart } from "../charts.js"; +import { info } from "../universe.js"; + +let result = null; + +export function render(node) { + node.replaceChildren(); + if (!state.selection.length) { node.appendChild(empty("select candidate stocks in the Stocks tab first")); return; } + node.append(viewsCard(), resultCard()); +} + +/* ------------------------------------------------------------------ the views */ +function viewsCard() { + const body = el("div"); + const list = el("div"); + + const drawList = () => { + list.replaceChildren(); + if (!state.views.length) { list.appendChild(el("div", { class: "note", text: "No views: the posterior equals the market-implied (equilibrium) returns." })); return; } + list.appendChild(dataTable([ + { key: "label", label: "View", render: (v) => document.createTextNode(describe(v)) }, + { key: "confidence", label: "Confidence", num: true, fmt: (v) => fmt.pct(v, 0) }, + { key: "del", label: "", render: (v) => button("remove", () => { + state.views = state.views.filter((x) => x !== v); changed("views", "views"); drawList(); }, "btn small") }, + ], state.views, { sortable: false })); + }; + drawList(); + + const draft = { kind: "sentiment", symbol: state.selection[0] || "", sentiment: 0.5, confidence: 0.5, strength: 0.25, + assets: "", q: 0.1 }; + const form = el("div"); + const drawForm = () => { + form.replaceChildren( + el("div", { class: "row" }, + field("Kind", select([["sentiment", "sentiment on one stock"], ["explicit", "explicit return view"]], draft.kind, + (v) => { draft.kind = v; drawForm(); })), + draft.kind === "sentiment" + ? field("Stock", select(state.selection.map((s) => [s, `${s} — ${info(s)?.name || ""}`.slice(0, 44)]), draft.symbol, + (v) => { draft.symbol = v; })) + : field("Portfolio of the view", input("text", draft.assets, (v) => { draft.assets = v; }, + { placeholder: "AAPL:1, MSFT:-1" }), "weights of a long/short view portfolio"), + draft.kind === "sentiment" + ? field("Sentiment −1 … +1", input("number", draft.sentiment, (v) => { draft.sentiment = v; }, + { min: -1, max: 1, step: 0.1 })) + : field("Expected return p.a.", input("number", draft.q, (v) => { draft.q = v; }, { step: 0.01 }), + "as a fraction, e.g. 0.12"), + field("Confidence", input("number", draft.confidence, (v) => { draft.confidence = v; }, + { min: 0.01, max: 1, step: 0.05 }), "Idzorek: 1 = fully trust the view"), + draft.kind === "sentiment" + ? field("Strength", input("number", draft.strength, (v) => { draft.strength = v; }, { min: 0, step: 0.05 }), + "return added at sentiment +1") + : null, + el("div", { class: "field auto" }, el("label", { text: " " }), button("Add view", () => { + const v = draft.kind === "sentiment" + ? { kind: "sentiment", symbol: draft.symbol, sentiment: Number(draft.sentiment), + confidence: Number(draft.confidence), strength: Number(draft.strength) } + : { kind: "explicit", assets: parseAssets(draft.assets), q: Number(draft.q), confidence: Number(draft.confidence) }; + if (v.kind === "explicit" && !Object.keys(v.assets).length) { alert("give the view portfolio as SYMBOL:weight pairs"); return; } + state.views = [...state.views, v]; + changed("views", "views"); drawList(); + }, "btn primary")))); + }; + drawForm(); + + body.append(list, el("div", { style: { marginTop: "10px" } }, form)); + return card("Views", "Black-Litterman: a view moves the posterior away from the market-implied return, by as much as its confidence allows", body); +} + +function describe(v) { + if (v.kind === "sentiment") return `${v.symbol}: sentiment ${v.sentiment > 0 ? "+" : ""}${v.sentiment} (strength ${fmt.pct(v.strength, 0)})`; + const parts = Object.entries(v.assets).map(([s, w]) => `${w > 0 ? "+" : ""}${w} ${s}`).join(" "); + return `${parts} → ${fmt.pct(v.q, 1)} p.a.`; +} + +function parseAssets(text) { + const out = {}; + for (const part of String(text).split(/[,;]/)) { + const [s, w] = part.split(":").map((x) => x.trim()); + if (s && w !== undefined && !Number.isNaN(Number(w))) out[s.toUpperCase()] = Number(w); + } + return out; +} + +/* ------------------------------------------------------------------ the model output */ +function resultCard() { + const out = el("div", { id: "bl-result" }); + let btn; + const run = async () => { + await withBusy(btn, live("bl-result", out), async () => { + result = await api.post("/model/expected-returns", { + symbols: state.selection, max_assets: state.selection.length, views: viewPayload(), + tau: Number(state.model.bl_tau), years: Number(state.model.bl_years), + omega_method: state.model.bl_omega_method || "idzorek", + }); + draw(live("bl-result", out), result); + }).catch(() => {}); + }; + btn = button("Compute expected returns", run, "btn primary"); + const body = el("div", {}, + el("div", { class: "row", style: { marginBottom: "10px" } }, + field("Ω (view uncertainty)", select([["idzorek", "Idzorek (from the confidence)"], ["he_litterman", "He-Litterman"]], + state.model.bl_omega_method || "idzorek", (v) => { state.model.bl_omega_method = v; changed("model", "views"); run(); })), + field("τ", input("number", state.model.bl_tau, (v) => { state.model.bl_tau = v; changed("model", "views"); }, { step: 0.01, min: 0.001 })), + field("History (years)", input("number", state.model.bl_years, (v) => { state.model.bl_years = v; changed("model", "views"); }, { min: 1, max: 20 })), + el("div", { class: "field auto" }, el("label", { text: " " }), btn)), + out); + if (result) draw(out, result); else out.appendChild(empty("compute the model to see implied and posterior expected returns")); + return card("Implied and posterior expected returns", "reverse optimisation of the cap-weighted market, then the views", body); +} + +function draw(out, r) { + const symbols = Object.keys(r.prior_ann); + const rows = symbols.map((s) => ({ + symbol: s, name: info(s)?.name || s, weight: r.market_weights[s], vol: r.volatility_ann[s], + prior: r.prior_ann[s], post: r.posterior_ann[s], change: r.posterior_ann[s] - r.prior_ann[s], + sharpe: r.volatility_ann[s] ? (r.posterior_ann[s] - r.risk_free_ann) / r.volatility_ann[s] : null, + })); + const moved = [...rows].sort((a, b) => Math.abs(b.change) - Math.abs(a.change)); + const top = moved.slice(0, 20); + const chart = chartBox(), changeChart = chartBox(); + + setChildren(out, + statTiles([ + { label: "Market return p.a.", value: fmt.pct(r.market_return_ann, 2), delta: "arithmetic, cap-weighted index" }, + { label: "Market volatility", value: fmt.pct(r.market_volatility_ann, 1) }, + { label: "Implied risk aversion δ", value: fmt.num(r.delta, 2) }, + { label: "τ", value: fmt.num(r.tau, 3) }, + { label: "Risk-free", value: fmt.pct(r.risk_free_ann, 2) }, + { label: "Views", value: fmt.int(r.views.length), delta: r.omega_method }, + ]), + el("div", { class: "grid two", style: { marginTop: "12px" } }, + card("Implied vs posterior", `the ${top.length} stocks the views move most`, chart), + card("Change from the views", "posterior − implied, p.a.", changeChart)), + r.views.length + ? card("How the views were translated", "q is the view's expected return p.a.; ω its variance (smaller = more trusted)", + dataTable([ + { key: "label", label: "View" }, + { key: "q_ann", label: "q p.a.", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "confidence", label: "Confidence", num: true, fmt: (v) => fmt.pct(v, 0) }, + { key: "omega", label: "ω", num: true, fmt: (v) => v?.toExponential(2) }, + ], r.views, { sortable: false })) + : null, + card("Per-stock expected returns", "implied = what the market's weights say; posterior = after the views", + dataTable([ + { key: "symbol", label: "Symbol", cls: "sym" }, + { key: "name", label: "Name", cls: "name" }, + { key: "weight", label: "Market weight", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "vol", label: "Vol p.a.", num: true, fmt: (v) => fmt.pct(v, 1) }, + { key: "prior", label: "Implied p.a.", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "post", label: "Posterior p.a.", num: true, fmt: (v) => fmt.pct(v, 2) }, + { key: "change", label: "Change", num: true, fmt: (v) => fmt.pctSigned(v, 2), + cls: (row) => (Math.abs(row.change) < 1e-9 ? "muted" : row.change > 0 ? "pos" : "neg") }, + { key: "sharpe", label: "Return / vol", num: true, fmt: (v) => fmt.num(v, 2) }, + ], rows, { sortKey: "weight", sortDir: -1 })), + ); + + barChart(chart, { + items: top.map((r2) => ({ label: r2.symbol, values: [r2.prior, r2.post] })), + groups: ["implied", "posterior"], valueFormat: (v) => fmt.pct(v, 0), horizontal: true, rowHeight: 24, labelWidth: 84, + }); + barChart(changeChart, { + items: top.map((r2) => ({ label: r2.symbol, values: [r2.change] })), + valueFormat: (v) => fmt.pctSigned(v, 1), horizontal: true, rowHeight: 24, labelWidth: 84, diverging: true, + valueName: "change", + }); +} diff --git a/quantcore/api/ui/js/universe.js b/quantcore/api/ui/js/universe.js new file mode 100644 index 0000000..a0807c4 --- /dev/null +++ b/quantcore/api/ui/js/universe.js @@ -0,0 +1,39 @@ +/* The investable universe, fetched once and filtered in the browser (745 rows). */ + +import { api } from "./core.js"; + +let cache = null; + +export async function universe() { + if (!cache) { + const raw = await api.get("/data/symbols", { limit: 2000 }); + const bySymbol = new Map(raw.symbols.map((s) => [s.symbol, s])); + cache = { ...raw, rows: raw.symbols, bySymbol }; + } + return cache; +} + +export function clearUniverse() { cache = null; } + +export function info(symbol) { return cache?.bySymbol.get(symbol) || null; } + +export function name(symbol) { return info(symbol)?.name || symbol; } + +export function sector(symbol) { return info(symbol)?.sector || "Unknown"; } + +export function price(symbol) { return info(symbol)?.price ?? null; } + +/** Filter the cached rows. `{region, sector, country, search}` -- all optional, all case-insensitive. */ +export function filterRows(rows, f = {}) { + const q = (f.search || "").trim().toLowerCase(); + return rows.filter((r) => + (!f.region || r.region === f.region) && + (!f.sector || (r.sector || "Unknown") === f.sector) && + (!f.country || r.country === f.country) && + (!q || r.symbol.toLowerCase().includes(q) || (r.name || "").toLowerCase().includes(q))); +} + +/** The `n` largest by market capitalisation under the filter. */ +export function topSymbols(rows, f, n) { + return filterRows(rows, f).slice(0, n).map((r) => r.symbol); +} diff --git a/quantcore/api/ui/styles.css b/quantcore/api/ui/styles.css new file mode 100644 index 0000000..a161b85 --- /dev/null +++ b/quantcore/api/ui/styles.css @@ -0,0 +1,254 @@ +/* quantcore test bench -- design tokens and layout. + Colours: the validated reference palette (categorical slots 1-8, a blue sequential ramp and a + blue<->red diverging pair with a neutral grey midpoint). Dark mode is a selected set of steps + for the dark surface, not an automatic flip. */ + +:root { + color-scheme: light; + --surface-0: #f2f1ee; /* page */ + --surface-1: #fcfcfb; /* card / chart surface */ + --surface-2: #f6f5f2; /* inset: table heads, wells */ + --border: #e3e2dd; + --border-strong: #cfcec8; + --text-primary: #14130f; + --text-secondary: #52514e; + --text-muted: #85837c; + + --series-1: #2a78d6; --series-2: #eb6834; --series-3: #1baf7a; --series-4: #eda100; + --series-5: #e87ba4; --series-6: #008300; --series-7: #4a3aa7; --series-8: #e34948; + + --seq-100: #cde2fb; --seq-250: #86b6ef; --seq-400: #3987e5; --seq-550: #1c5cab; --seq-700: #0d366b; + --div-pos: #2a78d6; /* diverging: cool pole */ + --div-neg: #e34948; /* diverging: warm pole */ + --div-mid: #f0efec; /* neutral midpoint */ + + --good: #0f7a4a; --warning: #b26a00; --serious: #c2410c; --critical: #c02626; + --accent: #2a78d6; + --shadow: 0 1px 2px rgba(20, 19, 15, .05), 0 4px 16px rgba(20, 19, 15, .05); + --radius: 10px; + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Inter, Roboto, Helvetica, Arial, sans-serif; +} + +@media (prefers-color-scheme: dark) { + :root:where(:not([data-theme="light"])) { + color-scheme: dark; + --surface-0: #121211; --surface-1: #1a1a19; --surface-2: #232321; + --border: #333331; --border-strong: #46453f; + --text-primary: #ffffff; --text-secondary: #c3c2b7; --text-muted: #8f8e85; + --series-1: #3987e5; --series-2: #d95926; --series-3: #199e70; --series-4: #c98500; + --series-5: #d55181; --series-6: #008300; --series-7: #9085e9; --series-8: #e66767; + --seq-100: #0d366b; --seq-250: #184f95; --seq-400: #256abf; --seq-550: #3987e5; --seq-700: #9ec5f4; + --div-pos: #3987e5; --div-neg: #e66767; --div-mid: #383835; + --good: #37b57d; --warning: #d9a02c; --serious: #e07a3c; --critical: #e66767; + --accent: #3987e5; + --shadow: 0 1px 2px rgba(0, 0, 0, .4), 0 4px 16px rgba(0, 0, 0, .3); + } +} +:root[data-theme="dark"] { + color-scheme: dark; + --surface-0: #121211; --surface-1: #1a1a19; --surface-2: #232321; + --border: #333331; --border-strong: #46453f; + --text-primary: #ffffff; --text-secondary: #c3c2b7; --text-muted: #8f8e85; + --series-1: #3987e5; --series-2: #d95926; --series-3: #199e70; --series-4: #c98500; + --series-5: #d55181; --series-6: #008300; --series-7: #9085e9; --series-8: #e66767; + --seq-100: #0d366b; --seq-250: #184f95; --seq-400: #256abf; --seq-550: #3987e5; --seq-700: #9ec5f4; + --div-pos: #3987e5; --div-neg: #e66767; --div-mid: #383835; + --good: #37b57d; --warning: #d9a02c; --serious: #e07a3c; --critical: #e66767; + --accent: #3987e5; + --shadow: 0 1px 2px rgba(0, 0, 0, .4), 0 4px 16px rgba(0, 0, 0, .3); +} + +* { box-sizing: border-box; } +html, body { height: 100%; } +body { + margin: 0; background: var(--surface-0); color: var(--text-primary); + font-family: var(--sans); font-size: 13px; line-height: 1.45; + -webkit-font-smoothing: antialiased; +} +h1, h2, h3, h4 { margin: 0; font-weight: 600; letter-spacing: -.01em; } +h2 { font-size: 15px; } +h3 { font-size: 13px; } +a { color: var(--accent); } + +/* ---------------------------------------------------------------- app frame */ +.app { display: flex; flex-direction: column; min-height: 100vh; } + +.topbar { + display: flex; align-items: center; gap: 16px; padding: 10px 18px; + background: var(--surface-1); border-bottom: 1px solid var(--border); position: sticky; top: 0; z-index: 30; +} +.brand { display: flex; align-items: baseline; gap: 9px; } +.brand b { font-size: 15px; letter-spacing: -.02em; } +.brand span { color: var(--text-muted); font-size: 11px; } +.status-row { display: flex; gap: 6px; flex-wrap: wrap; margin-left: auto; align-items: center; } + +.pill { + display: inline-flex; align-items: center; gap: 6px; padding: 3px 9px; border-radius: 999px; + background: var(--surface-2); border: 1px solid var(--border); font-size: 11px; color: var(--text-secondary); + white-space: nowrap; +} +.pill b { color: var(--text-primary); font-weight: 600; } +.dot { width: 7px; height: 7px; border-radius: 50%; background: var(--text-muted); flex: none; } +.dot.ok { background: var(--good); } .dot.warn { background: var(--warning); } .dot.bad { background: var(--critical); } + +.tabs { display: flex; gap: 2px; padding: 0 18px; background: var(--surface-1); border-bottom: 1px solid var(--border); + position: sticky; top: 49px; z-index: 29; overflow-x: auto; } +.tabs button { + appearance: none; background: none; border: 0; border-bottom: 2px solid transparent; cursor: pointer; + padding: 9px 13px; font: inherit; font-weight: 500; color: var(--text-secondary); white-space: nowrap; +} +.tabs button:hover { color: var(--text-primary); } +.tabs button[aria-selected="true"] { color: var(--text-primary); border-bottom-color: var(--accent); } + +.body { display: flex; align-items: flex-start; gap: 0; flex: 1; min-height: 0; } +.sidebar { + width: 302px; flex: none; background: var(--surface-1); border-right: 1px solid var(--border); + padding: 14px; position: sticky; top: 84px; max-height: calc(100vh - 84px); overflow-y: auto; +} +.main { flex: 1; min-width: 0; padding: 16px 18px 48px; } +.tabpanel[hidden] { display: none; } + +/* ---------------------------------------------------------------- cards */ +.card { + background: var(--surface-1); border: 1px solid var(--border); border-radius: var(--radius); + padding: 14px; margin-bottom: 14px; box-shadow: var(--shadow); +} +.card > header { display: flex; align-items: baseline; gap: 10px; margin-bottom: 10px; flex-wrap: wrap; } +.card > header .sub { color: var(--text-muted); font-size: 11px; } +.card > header .spacer { margin-left: auto; } +.grid { display: grid; gap: 14px; } +.grid.two { grid-template-columns: repeat(auto-fit, minmax(380px, 1fr)); align-items: start; } +.grid.three { grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); } +@media (max-width: 900px) { .body { flex-direction: column; } .sidebar { width: 100%; position: static; max-height: none; border-right: 0; border-bottom: 1px solid var(--border); } } + +/* ---------------------------------------------------------------- forms */ +.field { display: flex; flex-direction: column; gap: 3px; margin-bottom: 9px; min-width: 0; } +.field > label { font-size: 11px; color: var(--text-secondary); font-weight: 500; } +.field .hint { font-size: 10.5px; color: var(--text-muted); } +input, select, textarea, button { font: inherit; font-size: 12.5px; color: var(--text-primary); } +input[type="text"], input[type="number"], input[type="date"], input[type="search"], select, textarea { + background: var(--surface-1); border: 1px solid var(--border-strong); border-radius: 7px; + padding: 5px 8px; width: 100%; min-width: 0; +} +input:focus-visible, select:focus-visible, textarea:focus-visible, button:focus-visible { + outline: 2px solid var(--accent); outline-offset: 1px; +} +input[type="number"] { font-variant-numeric: tabular-nums; } +.row { display: flex; gap: 8px; align-items: flex-end; flex-wrap: wrap; } +.row > .field { flex: 1; } +.row > .field.auto { flex: 0 0 auto; } +.row > .field.auto > .btnbar { height: 27px; } +.check { display: flex; align-items: center; gap: 6px; font-size: 12px; color: var(--text-secondary); } +.check input { width: auto; } + +button.btn { + background: var(--surface-2); border: 1px solid var(--border-strong); border-radius: 7px; + padding: 5px 11px; cursor: pointer; font-weight: 500; white-space: nowrap; +} +button.btn:hover { border-color: var(--text-muted); } +button.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; } +button.btn.primary:hover { filter: brightness(1.07); } +button.btn.small { padding: 2px 7px; font-size: 11px; } +button.btn:disabled { opacity: .5; cursor: not-allowed; } +.btnbar { display: flex; gap: 7px; flex-wrap: wrap; align-items: center; } + +/* sidebar sections */ +details.section { border-top: 1px solid var(--border); padding: 9px 0; } +details.section:first-of-type { border-top: 0; } +details.section > summary { + cursor: pointer; font-weight: 600; font-size: 12px; list-style: none; display: flex; align-items: center; gap: 7px; + color: var(--text-primary); padding: 2px 0; +} +details.section > summary::-webkit-details-marker { display: none; } +details.section > summary::before { + content: ""; width: 0; height: 0; border-left: 4px solid var(--text-muted); + border-top: 3.5px solid transparent; border-bottom: 3.5px solid transparent; transition: transform .12s; +} +details.section[open] > summary::before { transform: rotate(90deg); } +details.section > summary .count { margin-left: auto; color: var(--text-muted); font-weight: 400; font-size: 11px; } +details.section > div { padding-top: 9px; } + +/* chips */ +.chips { display: flex; flex-wrap: wrap; gap: 5px; } +.chip { + display: inline-flex; align-items: center; gap: 5px; background: var(--surface-2); + border: 1px solid var(--border); border-radius: 999px; padding: 2px 4px 2px 9px; font-size: 11px; + font-family: var(--mono); +} +.chip button { appearance: none; background: none; border: 0; cursor: pointer; color: var(--text-muted); + font-size: 13px; line-height: 1; padding: 0 3px; border-radius: 50%; } +.chip button:hover { color: var(--critical); } + +/* ---------------------------------------------------------------- tables */ +.table-wrap { overflow: auto; max-height: 460px; border: 1px solid var(--border); border-radius: 8px; } +table.data { border-collapse: separate; border-spacing: 0; width: 100%; font-size: 12px; } +table.data th, table.data td { padding: 5px 9px; text-align: left; white-space: nowrap; border-bottom: 1px solid var(--border); } +table.data thead th { + position: sticky; top: 0; background: var(--surface-2); z-index: 2; font-weight: 600; font-size: 11px; + color: var(--text-secondary); cursor: default; border-bottom: 1px solid var(--border-strong); +} +table.data thead th.sortable { cursor: pointer; user-select: none; } +table.data thead th.sortable:hover { color: var(--text-primary); } +table.data thead th .arrow { color: var(--text-muted); font-size: 9px; } +table.data td.num, table.data th.num { text-align: right; font-variant-numeric: tabular-nums; font-family: var(--mono); font-size: 11.5px; } +table.data tbody tr:hover td { background: var(--surface-2); } +/* the row whose result is on screen: an accent edge plus a tint, so it reads without colour alone */ +table.data tbody tr.selected td { background: var(--surface-2); font-weight: 600; } +table.data tbody tr.selected td:first-child { box-shadow: inset 3px 0 0 var(--accent); } +table.data tbody tr:last-child td { border-bottom: 0; } +table.data td.sym { font-family: var(--mono); font-weight: 600; } +table.data td.name { max-width: 230px; overflow: hidden; text-overflow: ellipsis; } +.pos { color: var(--good); } .neg { color: var(--critical); } +.muted { color: var(--text-muted); } +.mono { font-family: var(--mono); } + +/* ---------------------------------------------------------------- stat tiles */ +.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(132px, 1fr)); gap: 1px; + background: var(--border); border: 1px solid var(--border); border-radius: 8px; overflow: hidden; } +.stat { background: var(--surface-1); padding: 9px 11px; } +.stat .label { font-size: 10.5px; color: var(--text-secondary); } +.stat .value { font-size: 17px; font-weight: 600; font-variant-numeric: tabular-nums; letter-spacing: -.02em; margin-top: 1px; } +.stat .delta { font-size: 10.5px; font-variant-numeric: tabular-nums; } +.hero { font-size: 30px; font-weight: 600; letter-spacing: -.03em; font-variant-numeric: tabular-nums; } + +/* ---------------------------------------------------------------- misc */ +.chart { width: 100%; min-width: 0; } +.chart svg { margin: 0 auto; } +.chart svg { display: block; width: 100%; height: auto; overflow: visible; } +.legend { display: flex; flex-wrap: wrap; gap: 4px 14px; margin: 6px 0 2px; font-size: 11px; color: var(--text-secondary); } +.legend .item { display: inline-flex; align-items: center; gap: 5px; cursor: default; } +.legend .key { width: 12px; height: 2px; border-radius: 1px; flex: none; } +.legend .key.rect { height: 9px; width: 9px; border-radius: 2px; } + +#tooltip { + position: fixed; z-index: 100; pointer-events: none; opacity: 0; transition: opacity .08s; + background: var(--surface-1); border: 1px solid var(--border-strong); border-radius: 7px; + box-shadow: var(--shadow); padding: 7px 9px; font-size: 11.5px; max-width: 280px; +} +#tooltip .tt-title { font-weight: 600; margin-bottom: 3px; } +#tooltip .tt-row { display: flex; align-items: center; gap: 7px; } +#tooltip .tt-row .key { width: 10px; height: 2px; border-radius: 1px; flex: none; } +#tooltip .tt-row .nm { color: var(--text-secondary); } +#tooltip .tt-row .v { margin-left: auto; font-variant-numeric: tabular-nums; font-weight: 600; } + +.note { border-left: 2px solid var(--border-strong); padding: 2px 0 2px 9px; color: var(--text-secondary); font-size: 11.5px; } +.warn-box { border-left: 2px solid var(--warning); background: var(--surface-2); padding: 7px 10px; border-radius: 0 7px 7px 0; + font-size: 11.5px; color: var(--text-secondary); margin-bottom: 8px; } +.error-box { border-left: 2px solid var(--critical); background: var(--surface-2); padding: 7px 10px; border-radius: 0 7px 7px 0; + font-size: 11.5px; color: var(--text-primary); margin-bottom: 8px; white-space: pre-wrap; } +.kv { display: grid; grid-template-columns: max-content 1fr; gap: 2px 12px; font-size: 11.5px; } +.kv dt { color: var(--text-secondary); } +.kv dd { margin: 0; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; min-width: 0; } +.empty { color: var(--text-muted); font-size: 12px; padding: 18px; text-align: center; border: 1px dashed var(--border-strong); border-radius: 8px; } +.busy { opacity: .45; transition: opacity .1s; } +.spinner { width: 12px; height: 12px; border: 2px solid var(--border-strong); border-top-color: var(--accent); + border-radius: 50%; animation: spin .7s linear infinite; display: inline-block; vertical-align: -2px; } +@keyframes spin { to { transform: rotate(360deg); } } +.seg { display: inline-flex; align-self: flex-start; flex: none; border: 1px solid var(--border-strong); + border-radius: 7px; overflow: hidden; } +.seg button { appearance: none; flex: none; background: var(--surface-1); border: 0; border-left: 1px solid var(--border); + padding: 4px 10px; cursor: pointer; color: var(--text-secondary); font-size: 12px; } +.seg button:first-child { border-left: 0; } +.seg button[aria-pressed="true"] { background: var(--accent); color: #fff; } +.tabbar-sub { display: flex; gap: 4px; margin-bottom: 10px; flex-wrap: wrap; } diff --git a/quantcore/apicli.py b/quantcore/apicli.py new file mode 100644 index 0000000..9be3784 --- /dev/null +++ b/quantcore/apicli.py @@ -0,0 +1,672 @@ +"""Command-line client of the quant-core API: ``quantcore-api ``. + +``quantcore`` (the other entry point) runs the services *in this process* and needs the data +directory locally. ``quantcore-api`` talks to a *running* service over HTTP and needs only its URL, +so it works from any machine on the network and is the quickest way to see what the API returns -- +and, with ``selftest``, to check that a deployment answers correctly on every endpoint. + + quantcore-api --url http://spark-6d4a:5001 health + quantcore-api selftest # exercise every endpoint, pass/fail table + quantcore-api endpoints # the API surface, from its own OpenAPI schema + + quantcore-api universe --region Europe --sector "Health Care" + quantcore-api prices NESN.SW AAPL --freq weekly --tail 5 + quantcore-api risk NESN.SW AAPL --horizon week --drift zero + quantcore-api correlation NESN.SW NOVN.SW ROG.SW + quantcore-api expected-returns --max-assets 50 --view AAPL:0.6:0.4 + quantcore-api scenarios NESN.SW AAPL --model gbm --n-scenarios 4096 + quantcore-api optimize --cash 1000000 --cvar-limit 40000 --horizon week --risk-drift zero + quantcore-api optimize --request my_request.json --json + quantcore-api update --wait # start the data-refresh job and follow it + +The base URL comes from ``--url``, else ``QUANTCORE_API_URL``, else ``http://127.0.0.1:5001``. +Every command takes ``--json`` to print the raw answer instead of a table; that is the form to pipe +into ``jq`` or to save as a fixture. +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +import time +from typing import Any + +from quantcore.client import QuantcoreClient, QuantcoreError, default_url + +WIDE = {"display.width": 200, "display.max_columns": 40, "display.max_rows": 200} + + +# ------------------------------------------------------------------------------------------------ +# output helpers +# ------------------------------------------------------------------------------------------------ +def _json(obj: Any) -> int: + print(json.dumps(obj, indent=2, default=str)) + return 0 + + +def _records(rows: list[dict], transpose: bool = False, round_to: int = 4) -> int: + """A list of JSON objects as a table; transposed when the rows are wide (risk tables).""" + import pandas as pd + + if not rows: + print("(no rows)") + return 0 + df = pd.DataFrame(rows) + if transpose: + df = df.set_index(df.columns[0]).T + with pd.option_context(*[x for kv in WIDE.items() for x in kv]): + print(df.round(round_to).to_string()) + return 0 + + +def _frame(body: dict, tail: int | None = None) -> int: + """A ``{"index", "columns", "data"}`` answer as a table, optionally only its last rows.""" + import pandas as pd + + df = pd.DataFrame(body.get("data") or [], index=body.get("index") or [], columns=body.get("columns") or []) + head = {k: v for k, v in body.items() if k not in ("index", "columns", "data")} + if head: + print(" · ".join(f"{k}={v}" for k, v in head.items())) + print(f"{len(df)} rows x {len(df.columns)} columns" + (f", last {tail}" if tail and len(df) > tail else "")) + if tail: + df = df.tail(tail) + with pd.option_context(*[x for kv in WIDE.items() for x in kv]): + print(df.round(6).to_string()) + return 0 + + +def _matrix(body: dict, key: str) -> int: + inner = body.get(key, body) + return _frame(inner if isinstance(inner, dict) and "data" in inner else body) + + +def _client(args: argparse.Namespace) -> QuantcoreClient: + return QuantcoreClient(args.url, timeout=args.timeout) + + +def _views(specs: list[str] | None) -> list[dict]: + """``--view SYMBOL:SENTIMENT[:CONFIDENCE[:STRENGTH]]`` -> the API's view objects. + + ``--view AAPL:0.6:0.4`` reads "positive news on Apple (sentiment 0.6), moderate confidence". + """ + views = [] + for spec in specs or []: + parts = spec.split(":") + if len(parts) < 2: + raise SystemExit(f"--view needs SYMBOL:SENTIMENT[:CONFIDENCE[:STRENGTH]], got {spec!r}") + view: dict[str, Any] = {"symbol": parts[0], "sentiment": float(parts[1])} + if len(parts) > 2: + view["confidence"] = float(parts[2]) + if len(parts) > 3: + view["strength"] = float(parts[3]) + views.append(view) + return views + + +# ------------------------------------------------------------------------------------------------ +# commands +# ------------------------------------------------------------------------------------------------ +def cmd_health(args) -> int: + body = _client(args).health() + if args.json: + return _json(body) + store, cuopt = body.get("store") or {}, body.get("cuopt") or {} + print(f"{args.url or default_url()} version {body.get('version')} currency {body.get('base_currency')}") + print(f"store : {store.get('n_universe')} stocks in the universe, {store.get('n_symbols')} with prices, " + f"{store.get('n_days')} days, last {store.get('last_price_date')}") + print(f"cuOpt : {'reachable, ' + str(cuopt.get('version')) if cuopt.get('reachable') else 'NOT reachable'}" + f"{'' if cuopt.get('reachable') else ' - optimisations fall back to HiGHS on the CPU'}") + return 0 + + +def cmd_endpoints(args) -> int: + schema = _client(args).openapi() + rows = [{"method": method.upper(), "path": path, "summary": op.get("summary") or op.get("operationId", ""), + "tag": (op.get("tags") or [""])[0]} + for path, ops in sorted((schema.get("paths") or {}).items()) + for method, op in ops.items() if method in ("get", "post", "put", "delete")] + if args.json: + return _json(rows) + print(f"{schema.get('info', {}).get('title', 'quantcore')} -- {len(rows)} endpoints " + f"(interactive docs at {(args.url or default_url()).rstrip('/')}/docs)") + return _records(rows, round_to=0) + + +def cmd_universe(args) -> int: + rows = _client(args).universe(region=args.region, sector=args.sector, country=args.country) + if args.json: + return _json(rows) + keep = ["symbol", "name", "region", "country", "gics_sector", "listing_currency", "market_cap_base"] + rows = sorted(rows, key=lambda r: -(r.get("market_cap_base") or 0))[: args.limit] + print(f"{len(rows)} of the matching stocks, largest first") + return _records([{k: r.get(k) for k in keep} for r in rows], round_to=0) + + +def cmd_symbol(args) -> int: + return _json(_client(args).symbol(args.symbol)) + + +def cmd_prices(args) -> int: + body = _client(args).prices(args.symbols, start=args.start, end=args.end, currency=args.currency, + freq=args.freq) + return _json(body) if args.json else _frame(body, args.tail) + + +def cmd_returns(args) -> int: + body = _client(args).returns(args.symbols, start=args.start, end=args.end, currency=args.currency, + freq=args.freq) + return _json(body) if args.json else _frame(body, args.tail) + + +def cmd_market(args) -> int: + qc = _client(args) + if args.what == "weights": + body = qc.market_weights(max_assets=args.max_assets) + if args.json: + return _json(body) + weights = body.get("weights") or {} + print(f"as of {body.get('as_of')}, {len(weights)} stocks, sum {sum(weights.values()):.6f}") + return _records([{"symbol": k, "weight": v} for k, v in list(weights.items())[: args.limit]], round_to=6) + body = qc.market_index(start=args.start, end=args.end, freq=args.freq) + if args.json: + return _json(body) + print(f"cap-weighted index, {body.get('freq')}: mean_ann {body.get('mean_ann'):.4%}, " + f"vol_ann {body.get('vol_ann'):.4%}, {body.get('n')} observations") + return 0 + + +def cmd_risk(args) -> int: + rows = _client(args).asset_risk(args.symbols, horizon=args.horizon, freq=args.freq, window=args.window, + lam=args.lam, currency=args.currency, drift=args.drift, space=args.space, + confidence=args.confidence) + if args.json: + return _json(rows) + print(f"horizon {args.horizon} · {args.freq} · window {args.window} · lambda {args.lam} · " + f"drift {args.drift} · space {args.space}") + return _records(rows, transpose=True, round_to=6) + + +def cmd_risk_portfolio(args) -> int: + body = _client(args).portfolio_risk(json.loads(args.holdings), cash=args.cash, horizon=args.horizon, + method=args.method, confidence=args.confidence) + return _json(body) + + +def cmd_matrix(args) -> int: + qc = _client(args) + call = qc.correlation if args.command == "correlation" else qc.covariance + body = call(args.symbols, method=args.method, window=args.window, lam=args.lam, freq=args.freq) + return _json(body) if args.json else _frame(body) + + +def cmd_performance(args) -> int: + rows = _client(args).asset_performance(args.symbols, freq=args.freq, currency=args.currency, start=args.start) + return _json(rows) if args.json else _records(rows, transpose=True) + + +def cmd_expected_returns(args) -> int: + body = _client(args).expected_returns(max_assets=args.max_assets, symbols=args.symbols, tau=args.tau, + years=args.years, omega_method=args.omega_method, + views=_views(args.view)) + if args.json: + return _json(body) + rows = body.get("expected_returns") or body.get("returns") or [] + for k, v in body.items(): + if isinstance(v, (int, float, str)): + print(f"{k:24} {v}") + if isinstance(rows, dict): + rows = [{"symbol": k, **(v if isinstance(v, dict) else {"value": v})} for k, v in rows.items()] + return _records(rows[: args.limit] if isinstance(rows, list) else [], round_to=6) + + +def cmd_model_covariance(args) -> int: + body = _client(args).model_covariance(args.symbols, freq=args.freq, horizon=args.horizon, + covariance=_cov_spec(args)) + if args.json: + return _json(body) + print(f"model {body.get('model')} · as of {body.get('as_of')} · shrinkage applied " + f"{body.get('shrinkage_applied')}") + print("conditioning:", json.dumps(body.get("conditioning"), default=str)) + return _matrix(body, "covariance") + + +def cmd_scenarios(args) -> int: + body = _client(args).scenarios(args.symbols, model=args.model, n_scenarios=args.n_scenarios, + horizon=args.horizon, freq=args.freq, seed=args.seed, + sampling=args.sampling, covariance=_cov_spec(args), + expected_returns=args.expected_returns, + importance_sampling=args.importance_sampling, + holdings=json.loads(args.holdings) if args.holdings else None, + value=args.value, confidence=args.confidence) + if args.json: + return _json(body) + desc, port = body.get("description") or {}, body.get("portfolio") or {} + print(f"{desc.get('model')} · {desc.get('n_scenarios')} scenarios x {desc.get('n_assets')} assets · " + f"horizon {desc.get('horizon_periods')} periods · seed {desc.get('seed')}") + print(f"hash {desc.get('hash')} · shocks {desc.get('shock_hash')} · weighted {desc.get('weighted')} · " + f"effective sample size {desc.get('effective_sample_size')}") + if port: + print(f"portfolio: invested {port.get('invested'):,.0f}") + for level, fig in sorted((port.get("risk") or {}).items()): + print(f" {float(level):.0%} VaR {fig.get('var'):>14,.2f} CVaR {fig.get('cvar'):>14,.2f}") + return _records(body.get("summary") or [], transpose=True, round_to=6) + + +def cmd_optimize(args) -> int: + body: dict[str, Any] = json.loads(pathlib.Path(args.request).read_text()) if args.request else {} + flags = {"cash": args.cash, "max_assets": args.max_assets, "horizon": args.horizon, + "confidence": args.confidence, "cvar_limit": args.cvar_limit, "objective": args.objective, + "scenario_model": args.scenario_model, "n_scenarios": args.n_scenarios, "seed": args.seed, + "solver": args.solver, "time_limit": args.time_limit, "risk_drift": args.risk_drift, + "expected_returns": args.expected_returns, "importance_sampling": args.importance_sampling, + "row_generation": args.row_generation, "freq": args.freq, "symbols": args.symbols} + for k, v in flags.items(): # the request file wins over a flag default + if v is not None and k not in body: + body[k] = v + if args.holdings: + body["holdings"] = json.loads(args.holdings) + body.setdefault("mode", "rebalance") + if args.view: + body["views"] = _views(args.view) + if args.max_weight is not None or args.max_sector_weight is not None: + constraints = dict(body.get("constraints") or {}) + if args.max_weight is not None: + constraints["max_weight"] = args.max_weight + if args.max_sector_weight is not None: + constraints["max_sector_weight"] = args.max_sector_weight + body["constraints"] = constraints + + t0 = time.monotonic() + res = _client(args).optimize(**body) + elapsed = time.monotonic() - t0 + if args.json: + return _json(res) + print(f"status {res.get('status')} · solver {res.get('solver')} · reproducibility " + f"{res.get('reproducibility')} · {elapsed:.1f} s wall clock") + info = res.get("solver_info") or {} + if info.get("mip_gap") is not None: + print(f"mip gap {info['mip_gap']:.2e}" + (f" · {info.get('message')}" if info.get("message") else "")) + key = (res.get("scenario_key") or {}).get("id") + print(f"scenario key {key} · hash {res.get('scenario_hash')} · timings {res.get('timings')}") + print(f"wealth {res.get('wealth', 0):,.0f} · invested {res.get('invested', 0):,.0f} · " + f"cash left {res.get('cash_end', 0):,.0f} · cost {res.get('total_cost', 0):,.0f} · " + f"turnover {res.get('turnover', 0):,.0f}") + print(f"E[P&L] {res.get('expected_pnl', 0):,.0f} ({res.get('expected_return', 0):.3%}) · " + f"VaR {res.get('var', 0):,.0f} · CVaR {res.get('cvar', 0):,.0f} " + f"(limit {res.get('cvar_limit')}) at {res.get('confidence', 0):.0%} over {res.get('horizon')}") + for warning in res.get("warnings") or []: + print(f" ! {warning}") + positions = res.get("positions") or [] + if positions and not args.quiet: + keep = [k for k in ("symbol", "name", "sector", "price", "shares", "trade", "value", "weight") + if k in positions[0]] + _records([{k: p.get(k) for k in keep} for p in positions], round_to=3) + print("sectors:", {k: round(v, 3) for k, v in (res.get("sector_exposure") or {}).items()}) + return 0 if res.get("status") in ("Optimal", "FeasibleFound") else 2 + + +def cmd_jobs(args) -> int: + qc = _client(args) + if args.job_id: + return _json(qc.job(args.job_id)) + return _json(qc.jobs()) + + +def cmd_setup(args) -> int: + return _start_job(args, _client(args).start_setup(offline=args.offline, max_per_region=args.max_per_region)) + + +def cmd_update(args) -> int: + return _start_job(args, _client(args).start_update(full=args.full)) + + +def _start_job(args, started: dict) -> int: + job_id = started.get("job_id") or started.get("id") + print(f"job {job_id} started ({started.get('status', 'queued')})") + if not args.wait: + print(f"follow it with: quantcore-api --url {args.url or default_url()} jobs --job-id {job_id}") + return 0 + record = _client(args).wait_for_job(job_id, poll=args.poll, + on_poll=lambda r: print(f" ... {r.get('status')}", flush=True)) + _json(record) + return 0 if record.get("status") == "done" else 2 + + +def cmd_snapshots(args) -> int: + qc = _client(args) + if args.create: + return _json(qc.create_snapshot(args.name)) + return _json(qc.snapshots()) + + +def _cov_spec(args) -> dict: + return {"method": args.cov_method, "lam": args.lam, "window": args.window, "shrinkage": args.shrinkage} + + +# ------------------------------------------------------------------------------------------------ +# selftest -- exercise every endpoint against a running service +# ------------------------------------------------------------------------------------------------ +def cmd_selftest(args) -> int: + """Call every endpoint with a small request and report what each one answered. + + Read-only by default: the two background jobs and the snapshot endpoint only run with + ``--write``, because they download data and write to the store. Exit code 1 if any probe failed. + """ + qc = _client(args) + results: list[dict] = [] + + def probe(name: str, call: Any, evidence: Any = None, skip: str = "") -> Any: + """Run one endpoint. ``call()`` performs it, ``evidence(value)`` says what came back.""" + if skip: + results.append({"endpoint": name, "result": "skipped", "ms": 0, "evidence": skip}) + return None + t0 = time.monotonic() + try: + value = call() + note = evidence(value) if evidence else "" + except QuantcoreError as exc: + note, value, ok = str(exc)[:120], None, False + except Exception as exc: # noqa: BLE001 (a probe's own assertion, or a malformed answer) + note, value, ok = f"{type(exc).__name__}: {exc}"[:120], None, False + else: + ok = True + results.append({"endpoint": name, "result": "ok" if ok else "FAIL", + "ms": round((time.monotonic() - t0) * 1000), "evidence": note}) + return value + + def health() -> dict: + body = qc.health() + if not (body.get("store") or {}).get("prices"): + raise AssertionError("the store has no prices - run the setup job first") + return body + + body = probe("GET /health", health, lambda b: ( + f"{b['store'].get('n_universe')} stocks, last {b['store'].get('last_price_date')}, " + f"cuOpt {'up' if (b.get('cuopt') or {}).get('reachable') else 'down'}")) + if body is None: + _records(results, round_to=0) + print("\nthe service is not usable - stopping here") + return 1 + + rows = probe("GET /universe", qc.universe, lambda r: f"{len(r)} rows") or [] + syms = args.symbols or [r["symbol"] for r in + sorted(rows, key=lambda r: -(r.get("market_cap_base") or 0))[: args.n_symbols]] + if not syms: + print("no symbols to probe with") + return 1 + print(f"probing with {', '.join(syms)}\n") + holdings = {syms[0]: 10} + + probe("GET /openapi.json", qc.openapi, lambda d: f"{len(d.get('paths') or {})} paths") + probe("GET /universe/{symbol}", lambda: qc.symbol(syms[0]), + lambda d: f"{d.get('symbol')}: {d.get('name')}") + probe("GET /data/symbols", lambda: qc.symbols_table(limit=10), + lambda d: f"{len(d.get('symbols') or [])} of {d.get('n_matched')} rows, " + f"facets {list((d.get('facets') or {}).keys())}") + probe("GET /prices", lambda: qc.prices(syms, start=args.start), + lambda d: f"{len(d.get('index') or [])} dates x {len(d.get('columns') or [])} symbols") + probe("GET /returns", lambda: qc.returns(syms, start=args.start), + lambda d: f"{d.get('kind')}, {len(d.get('index') or [])} dates") + probe("GET /market/index", qc.market_index, + lambda d: f"mean_ann {d.get('mean_ann'):.2%}, vol_ann {d.get('vol_ann'):.2%}") + probe("GET /market/weights", lambda: qc.market_weights(max_assets=20), + lambda d: f"{len(d.get('weights') or {})} weights, sum {sum((d.get('weights') or {}).values()):.4f}") + + probe("POST /risk/assets", lambda: qc.asset_risk(syms, horizon="week"), + lambda r: f"{len(r)} rows, cvar95_hist {r[0].get('cvar95_hist'):.4f}") + probe("POST /risk/assets (drift=zero, space=value)", + lambda: qc.asset_risk(syms, horizon="week", drift="zero", space="value"), + lambda r: f"cvar95_normal_ewm {r[0].get('cvar95_normal_ewm'):.4f}") + probe("POST /risk/series", lambda: qc.risk_series(syms[0], horizon="week"), + lambda d: f"{len(d.get('index') or [])} dates") + probe("POST /risk/covariance", lambda: qc.covariance(syms), + lambda d: f"{len(d.get('columns') or [])} assets") + probe("POST /risk/correlation", lambda: qc.correlation(syms), + lambda d: f"diagonal {d['data'][0][0]:.3f}") + probe("POST /risk/portfolio", lambda: qc.portfolio_risk(holdings, cash=10_000, horizon="week"), + lambda d: f"value {d.get('value', 0):,.0f}, " + f"CVaR95 {((d.get('confidence') or {}).get('0.95') or {}).get('cvar_money', 0):,.0f}") + + probe("POST /performance/assets", lambda: qc.asset_performance(syms), + lambda r: f"{len(r)} rows, sharpe {r[0].get('sharpe')}") + probe("POST /performance/rolling", lambda: qc.rolling_performance(syms[0], window=252), + lambda d: f"{len(d.get('index') or [])} dates") + probe("POST /performance/portfolio", lambda: qc.portfolio_performance(holdings), + lambda d: f"{len(d)} figures") + + probe("POST /model/expected-returns", + lambda: qc.expected_returns(max_assets=args.n_symbols, symbols=args.symbols), + lambda d: f"market return {(d.get('market_return_ann') or 0):.2%}") + probe("POST /model/covariance", lambda: qc.model_covariance(syms), + lambda d: f"shrinkage {d.get('shrinkage_applied')}") + probe("POST /model/scenarios", + lambda: qc.scenarios(syms, model="gbm", n_scenarios=1024, holdings=holdings), + lambda d: f"{(d.get('description') or {}).get('n_scenarios')} scenarios, " + f"hash {(d.get('description') or {}).get('hash')}") + + def optimise() -> dict: + body = {"cash": 1_000_000, "max_assets": max(len(syms), 12), "horizon": "week", + "symbols": args.symbols, "solver": args.solver, "time_limit": args.time_limit} + body.update({"scenario_model": "gbm", "n_scenarios": 4096, "cvar_limit_fraction": 0.05} + if args.full else {"scenario_model": "gaussian_exact"}) + res = qc.optimize(**{k: v for k, v in body.items() if v is not None}) + if res.get("status") not in ("Optimal", "FeasibleFound"): + raise AssertionError(f"status {res.get('status')}: {'; '.join(res.get('warnings') or [])}"[:120]) + return res + + probe(f"POST /optimize ({'gbm, 4096 scenarios' if args.full else 'gaussian_exact'})", optimise, + lambda r: f"{r.get('status')} on {r.get('solver')}, {r.get('n_positions')} positions, " + f"CVaR {r.get('cvar', 0):,.0f}") + + probe("GET /jobs", qc.jobs, lambda r: f"{len(r)} jobs") + probe("GET /snapshots", qc.snapshots, lambda r: f"{len(r)} snapshots") + probe("POST /snapshots", lambda: qc.create_snapshot(f"selftest-{int(time.time())}"), + lambda d: str(d.get("path"))[:60], + skip="" if args.write else "writes to the store (--write)") + probe("POST /jobs/update", qc.start_update, lambda d: f"job {d.get('job_id')}", + skip="" if args.write else "downloads data (--write)") + probe("POST /jobs/setup", lambda: qc.start_setup(offline=True), lambda d: f"job {d.get('job_id')}", + skip="" if args.write else "rebuilds the universe (--write)") + + if args.json: + return _json(results) + _records(results, round_to=0) + failed = [r for r in results if r["result"] == "FAIL"] + ok = [r for r in results if r["result"] == "ok"] + skipped = [r for r in results if r["result"] == "skipped"] + print(f"\n{len(ok)} ok, {len(failed)} failed, {len(skipped)} skipped " + f"in {sum(r['ms'] for r in results) / 1000:.1f} s against {qc.base_url}") + for r in failed: + print(f" FAIL {r['endpoint']}: {r['evidence']}") + return 1 if failed else 0 + + +# ------------------------------------------------------------------------------------------------ +# parser +# ------------------------------------------------------------------------------------------------ +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="quantcore-api", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--url", default=None, help=f"API base URL (default: $QUANTCORE_API_URL or {default_url()})") + p.add_argument("--timeout", type=float, default=900.0, help="seconds per request") + p.add_argument("--json", action="store_true", help="print the raw JSON answer instead of a table") + sub = p.add_subparsers(dest="command", required=True) + + # the same three options after the subcommand, where anyone would try them first + # ('quantcore-api optimize --json'). SUPPRESS keeps the value parsed before the subcommand + # unless the flag is repeated after it. + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--url", default=argparse.SUPPRESS) + common.add_argument("--timeout", type=float, default=argparse.SUPPRESS) + common.add_argument("--json", action="store_true", default=argparse.SUPPRESS) + + def add(name: str, help_: str, func: Any) -> argparse.ArgumentParser: + sp = sub.add_parser(name, help=help_, parents=[common]) + sp.set_defaults(func=func) + return sp + + def symbols_arg(sp: argparse.ArgumentParser) -> argparse.ArgumentParser: + sp.add_argument("symbols", nargs="+", help="Yahoo symbols, e.g. NESN.SW AAPL ASML.AS") + return sp + + def window_args(sp: argparse.ArgumentParser) -> argparse.ArgumentParser: + sp.add_argument("--window", type=int, default=200, help="moving-window length in periods") + sp.add_argument("--lam", type=float, default=0.94, help="EWM decay") + return sp + + add("health", "store status, cuOpt reachability, version", cmd_health) + add("endpoints", "list every endpoint from the service's own OpenAPI schema", cmd_endpoints) + + up = add("universe", "the investable universe (filter by region, sector, country)", cmd_universe) + up.add_argument("--region"), up.add_argument("--sector"), up.add_argument("--country") + up.add_argument("--limit", type=int, default=25) + + sy = add("symbol", "one universe row", cmd_symbol) + sy.add_argument("symbol") + + for name, func, what in (("prices", cmd_prices, "adjusted closes"), + ("returns", cmd_returns, "log-returns")): + sp = symbols_arg(add(name, what, func)) + sp.add_argument("--freq", default="daily", choices=["daily", "weekly", "monthly"]) + sp.add_argument("--currency", default="base", choices=["base", "local"]) + sp.add_argument("--start"), sp.add_argument("--end") + sp.add_argument("--tail", type=int, default=10, help="print only the last N rows (0: all)") + + mk = add("market", "the cap-weighted broad market: 'index' or 'weights'", cmd_market) + mk.add_argument("what", nargs="?", default="index", choices=["index", "weights"]) + mk.add_argument("--freq", default="weekly", choices=["daily", "weekly", "monthly"]) + mk.add_argument("--start"), mk.add_argument("--end") + mk.add_argument("--max-assets", type=int, default=None), mk.add_argument("--limit", type=int, default=25) + + rk = window_args(symbols_arg(add("risk", "per-stock volatility, VaR and CVaR", cmd_risk))) + rk.add_argument("--horizon", default="week", help="'day' | 'week' | 'month' | 'quarter' | 'year' | periods") + rk.add_argument("--freq", default="daily", choices=["daily", "weekly", "monthly"]) + rk.add_argument("--currency", default="base", choices=["base", "local"]) + rk.add_argument("--drift", default="estimated", choices=["estimated", "zero"], + help="'zero': volatility only, the market-risk convention") + rk.add_argument("--space", default="log", choices=["log", "value"], + help="'value': as a fraction of the position's value, which cannot exceed 100 %%") + rk.add_argument("--confidence", type=float, nargs="+", default=[0.95, 0.99]) + + rp = add("risk-portfolio", "VaR/CVaR of a portfolio held in share numbers", cmd_risk_portfolio) + rp.add_argument("--holdings", required=True, help='JSON: {"NESN.SW": 500, "AAPL": 100}') + rp.add_argument("--cash", type=float, default=0.0) + rp.add_argument("--horizon", default="week") + rp.add_argument("--method", default="normal", choices=["normal", "historical"]) + rp.add_argument("--confidence", type=float, nargs="+", default=[0.95, 0.99]) + + for name in ("correlation", "covariance"): + sp = window_args(symbols_arg(add(name, f"{name} matrix as measured", cmd_matrix))) + sp.add_argument("--method", default="ewm", choices=["rolling", "ewm"]) + sp.add_argument("--freq", default="daily", choices=["daily", "weekly", "monthly"]) + + pf = symbols_arg(add("performance", "annualised return, Sharpe, drawdown, skew, kurtosis", cmd_performance)) + pf.add_argument("--freq", default="weekly", choices=["daily", "weekly", "monthly"]) + pf.add_argument("--currency", default="base", choices=["base", "local"]) + pf.add_argument("--start") + + er = add("expected-returns", "Black-Litterman equilibrium and posterior returns", cmd_expected_returns) + er.add_argument("--symbols", nargs="*", default=None) + er.add_argument("--max-assets", type=int, default=100) + er.add_argument("--tau", type=float, default=0.05) + er.add_argument("--years", type=int, default=10) + er.add_argument("--omega-method", default="idzorek", choices=["idzorek", "he_litterman"]) + er.add_argument("--view", action="append", metavar="SYM:SENTIMENT[:CONF[:STRENGTH]]", + help="a sentiment view; repeatable") + er.add_argument("--limit", type=int, default=25) + + mc = window_args(symbols_arg(add("model-covariance", "the covariance the models use, with its " + "conditioning", cmd_model_covariance))) + mc.add_argument("--cov-method", default="ewm", choices=["rolling", "ewm"]) + mc.add_argument("--shrinkage", default="auto") + mc.add_argument("--freq", default="daily", choices=["daily", "weekly"]) + mc.add_argument("--horizon", default="week") + + sc = window_args(symbols_arg(add("scenarios", "a scenario set and its statistics", cmd_scenarios))) + sc.add_argument("--model", default="gbm", choices=["gbm", "empirical"]) + sc.add_argument("--n-scenarios", type=int, default=4096) + sc.add_argument("--horizon", default="week") + sc.add_argument("--freq", default="daily", choices=["daily", "weekly"]) + sc.add_argument("--seed", type=int, default=42) + sc.add_argument("--sampling", default="sobol", choices=["sobol", "mc", "antithetic", "lhs", "balanced"]) + sc.add_argument("--cov-method", default="ewm", choices=["rolling", "ewm"]) + sc.add_argument("--shrinkage", default="auto") + sc.add_argument("--expected-returns", default="zero", choices=["zero", "historical", "black_litterman"]) + sc.add_argument("--importance-sampling", default="auto", choices=["auto", "market", "adaptive", "off"]) + sc.add_argument("--holdings", help='JSON: {"NESN.SW": 500}; else the set is reported per stock') + sc.add_argument("--value", type=float, default=1_000_000.0) + sc.add_argument("--confidence", type=float, nargs="+", default=[0.95, 0.99]) + + op = add("optimize", "mean-CVaR MILP in integer shares", cmd_optimize) + op.add_argument("--request", help="JSON file with the full request; its fields win over the flags") + op.add_argument("--cash", type=float, default=1_000_000.0) + op.add_argument("--holdings", help='JSON dict -> rebalancing mode') + op.add_argument("--symbols", nargs="*", default=None, help="candidates; default: largest by market cap") + op.add_argument("--max-assets", type=int, default=100) + op.add_argument("--horizon", default="week") + op.add_argument("--freq", default="daily", choices=["daily", "weekly"]) + op.add_argument("--confidence", type=float, default=0.95) + op.add_argument("--cvar-limit", type=float, default=None, help="max CVaR in CHF over the horizon") + op.add_argument("--objective", default="max_return", choices=["max_return", "min_cvar", "utility"]) + op.add_argument("--scenario-model", default="empirical", choices=["gbm", "empirical", "gaussian_exact"]) + op.add_argument("--n-scenarios", type=int, default=None, + help="default: 16 per candidate within [4096, 16384], rounded up to a power of two") + op.add_argument("--seed", type=int, default=42) + op.add_argument("--expected-returns", default="black_litterman", + choices=["black_litterman", "historical", "zero"]) + op.add_argument("--risk-drift", default="model", choices=["model", "zero"], + help="'zero': measure the CVaR on centred scenarios, so a bullish forecast " + "cannot relax the risk limit") + op.add_argument("--importance-sampling", default="auto", choices=["auto", "market", "adaptive", "off"]) + op.add_argument("--row-generation", default="auto", choices=["auto", "on", "off"]) + op.add_argument("--max-weight", type=float, default=None, help="cap per stock (default 0.10)") + op.add_argument("--max-sector-weight", type=float, default=None, help="cap per GICS sector (default 0.25)") + op.add_argument("--view", action="append", metavar="SYM:SENTIMENT[:CONF[:STRENGTH]]", help="repeatable") + op.add_argument("--solver", default="auto", choices=["auto", "cuopt", "highs"]) + op.add_argument("--time-limit", type=float, default=300.0, help="seconds for the whole request") + op.add_argument("--quiet", action="store_true", help="summary only, no position table") + + jb = add("jobs", "list background jobs, or one with --job-id", cmd_jobs) + jb.add_argument("--job-id", default=None) + + st = add("setup", "start the universe/history build (background job)", cmd_setup) + st.add_argument("--offline", action="store_true", help="use the candidate lists shipped in the package") + st.add_argument("--max-per-region", type=int, default=None) + st.add_argument("--wait", action="store_true"), st.add_argument("--poll", type=float, default=10.0) + + ud = add("update", "start the price/FX refresh (background job)", cmd_update) + ud.add_argument("--full", action="store_true") + ud.add_argument("--wait", action="store_true"), ud.add_argument("--poll", type=float, default=5.0) + + sn = add("snapshots", "list snapshots, or --create one", cmd_snapshots) + sn.add_argument("--create", action="store_true"), sn.add_argument("--name", default=None) + + se = add("selftest", "call every endpoint and report what it answered", cmd_selftest) + se.add_argument("--full", action="store_true", help="optimise on a real scenario set, not gaussian_exact") + se.add_argument("--write", action="store_true", help="also probe the endpoints that write or download") + se.add_argument("--symbols", nargs="*", default=None, help="default: the largest few in the universe") + se.add_argument("--n-symbols", type=int, default=3) + se.add_argument("--start", default=None, help="first date for the price/return probes") + se.add_argument("--solver", default="auto", choices=["auto", "cuopt", "highs"]) + se.add_argument("--time-limit", type=float, default=120.0) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return args.func(args) + except QuantcoreError as exc: + print(f"error: {exc}", file=sys.stderr) + if exc.unreachable: + print("is the service running? start it with 'quantcore serve', or pass --url", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("interrupted", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quantcore/cli.py b/quantcore/cli.py new file mode 100644 index 0000000..3dd245b --- /dev/null +++ b/quantcore/cli.py @@ -0,0 +1,253 @@ +"""Command-line interface: ``quantcore ``. + + quantcore setup [--offline] [--max-per-region N] build the universe once (candidates, + metadata, full price + FX history) + quantcore reselect re-run only the selection step + quantcore clean re-apply price cleaning to stored data + quantcore update [--full] fetch the newest prices / FX + quantcore snapshot [--name NAME] freeze the data for reproducible demos + quantcore status what the store holds, cuOpt health + quantcore risk SYMBOL [SYMBOL ...] [--horizon week] [--freq daily] [--window 200] [--lam 0.94] + quantcore performance SYMBOL [SYMBOL ...] [--freq weekly] + quantcore optimize [--cash 1e6] [--cvar-limit 50000] [--confidence 0.95] [--horizon week] ... + quantcore serve [--host 127.0.0.1] [--port 5001] run the HTTP API (+ the test bench at /ui/) + +These commands run the services in this process and need the data directory locally. To drive a +*running* service over HTTP instead -- from any machine, without the data -- use ``quantcore-api`` +(:mod:`quantcore.apicli`), whose ``selftest`` also checks a deployment endpoint by endpoint. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys + +import pandas as pd + +from quantcore.config import Settings, get_settings + +log = logging.getLogger("quantcore.cli") + + +def _settings(args) -> Settings: + s = get_settings() + if getattr(args, "data_dir", None): + s = s.model_copy(update={"data_dir": args.data_dir}) + if getattr(args, "cuopt_url", None): + s = s.model_copy(update={"cuopt_url": args.cuopt_url}) + return s + + +def cmd_setup(args) -> int: + from quantcore.universe.setup_service import SetupService + + report = SetupService(_settings(args)).run(allow_web=not args.offline, max_per_region=args.max_per_region) + print(json.dumps(report.as_dict(), indent=2, default=str)) + return 0 + + +def cmd_reselect(args) -> int: + from quantcore.universe.setup_service import SetupService + + report = SetupService(_settings(args)).rebuild_selection() + print(json.dumps(report.as_dict(), indent=2, default=str)) + return 0 + + +def cmd_update(args) -> int: + from quantcore.data.download_service import DataDownloadService + + report = DataDownloadService(_settings(args)).update(full=args.full) + print(json.dumps(report.as_dict(), indent=2, default=str)) + return 0 + + +def cmd_clean(args) -> int: + from quantcore.data.download_service import DataDownloadService + + report = DataDownloadService(_settings(args)).clean() + print(json.dumps(report.as_dict(), indent=2, default=str)) + return 0 + + +def cmd_snapshot(args) -> int: + from quantcore.data.store import DataStore + + path = DataStore(_settings(args)).create_snapshot(args.name) + print(path) + return 0 + + +def cmd_status(args) -> int: + from quantcore.data.store import DataStore + from quantcore.optimization.solvers import CuOptSolver + + s = _settings(args) + out = {"store": DataStore(s).status(), "cuopt": CuOptSolver(s).health(), "cuopt_url": s.cuopt_url} + print(json.dumps(out, indent=2, default=str)) + return 0 + + +def cmd_risk(args) -> int: + from quantcore.data.market_data import MarketDataService + from quantcore.measurement.risk import RiskService + + md = MarketDataService(_settings(args), snapshot=args.snapshot) + table = RiskService(md).asset_risk(args.symbols, freq=args.freq, window=args.window, lam=args.lam, + horizon=args.horizon, currency=args.currency) + with pd.option_context("display.width", 200, "display.max_columns", 30): + print(table.round(5).T) + return 0 + + +def cmd_performance(args) -> int: + from quantcore.data.market_data import MarketDataService + from quantcore.measurement.performance import PerformanceService + + s = _settings(args) + md = MarketDataService(s, snapshot=args.snapshot) + table = PerformanceService(md, s.risk_free_rate).asset_performance(args.symbols, freq=args.freq, + currency=args.currency, start=args.start) + with pd.option_context("display.width", 200, "display.max_columns", 30): + print(table.round(4).T) + return 0 + + +def cmd_optimize(args) -> int: + from quantcore.data.market_data import MarketDataService + from quantcore.optimization.service import OptimizationRequest, OptimizationService + + s = _settings(args) + md = MarketDataService(s, snapshot=args.snapshot) + kwargs = {} + if args.request: + kwargs = json.loads(open(args.request).read()) + kwargs.setdefault("cash", args.cash) + kwargs.setdefault("horizon", args.horizon) + kwargs.setdefault("confidence", args.confidence) + if args.cvar_limit is not None: + kwargs["cvar_limit"] = args.cvar_limit + if args.symbols: + kwargs["symbols"] = args.symbols + kwargs.setdefault("max_assets", args.max_assets) + kwargs.setdefault("scenario_model", args.scenario_model) + if args.n_scenarios is not None: + kwargs.setdefault("n_scenarios", args.n_scenarios) + kwargs.setdefault("sampling", args.sampling) + kwargs.setdefault("importance_sampling", args.importance_sampling) + kwargs.setdefault("row_generation", args.row_generation) + kwargs.setdefault("solver", args.solver) + if args.holdings: + kwargs["holdings"] = json.loads(args.holdings) + kwargs["mode"] = "rebalance" + req = OptimizationRequest(**kwargs) + res = OptimizationService(s, md).optimize(req) + if args.json: + print(json.dumps(res.as_dict(), indent=2, default=str)) + else: + print(f"status={res.status} solver={res.solver} reproducible={res.reproducibility} scenarios={res.scenario_hash} " + f"key={res.scenario_key['id'] if res.scenario_key else None} " + f"wealth={res.wealth:,.0f} invested={res.invested:,.0f} cash_end={res.cash_end:,.0f} cost={res.total_cost:,.0f}") + print(f"E[P&L]={res.expected_pnl:,.0f} ({res.expected_return:.3%}) VaR={res.var:,.0f} CVaR={res.cvar:,.0f} " + f"limit={res.cvar_limit} @ {res.confidence:.0%} over {res.horizon} ({res.horizon_periods} periods)") + if res.positions: + df = pd.DataFrame(res.positions)[["symbol", "name", "sector", "price", "shares", "trade", "value", "weight"]] + with pd.option_context("display.width", 200, "display.max_rows", 500): + print(df.round(3).to_string(index=False)) + print("sectors:", {k: round(v, 3) for k, v in res.sector_exposure.items()}) + print("timings:", res.timings, "warnings:", res.warnings) + return 0 if res.status in ("Optimal", "FeasibleFound") else 2 + + +def cmd_serve(args) -> int: + import uvicorn + + s = _settings(args) + uvicorn.run("quantcore.api.app:app", host=args.host or s.api_host, port=args.port or s.api_port, reload=False) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="quantcore", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--data-dir", help="override QUANTCORE_DATA_DIR") + p.add_argument("--cuopt-url", help="override QUANTCORE_CUOPT_URL") + sub = p.add_subparsers(dest="command", required=True) + + sp = sub.add_parser("setup", help="build the universe and download all history (run once)") + sp.add_argument("--offline", action="store_true", help="use the static candidate lists shipped with the package") + sp.add_argument("--max-per-region", type=int, default=None, help="limit candidates per region (testing)") + sp.set_defaults(func=cmd_setup) + + sub.add_parser("reselect", help="re-run only the universe selection on stored data").set_defaults(func=cmd_reselect) + + up = sub.add_parser("update", help="fetch the newest prices and FX rates") + up.add_argument("--full", action="store_true", help="re-download the complete history") + up.set_defaults(func=cmd_update) + + sub.add_parser("clean", help="re-apply the price cleaning to the stored history").set_defaults(func=cmd_clean) + + sn = sub.add_parser("snapshot", help="freeze universe/prices/fx under snapshots/") + sn.add_argument("--name", default=None) + sn.set_defaults(func=cmd_snapshot) + + sub.add_parser("status", help="store status and cuOpt health").set_defaults(func=cmd_status) + + rk = sub.add_parser("risk", help="per-asset volatility, VaR and CVaR") + rk.add_argument("symbols", nargs="+") + rk.add_argument("--freq", default="daily", choices=["daily", "weekly", "monthly"]) + rk.add_argument("--horizon", default="day") + rk.add_argument("--window", type=int, default=200) + rk.add_argument("--lam", type=float, default=0.94) + rk.add_argument("--currency", default="base", choices=["base", "local"]) + rk.add_argument("--snapshot", default=None) + rk.set_defaults(func=cmd_risk) + + pf = sub.add_parser("performance", help="per-asset performance statistics") + pf.add_argument("symbols", nargs="+") + pf.add_argument("--freq", default="daily", choices=["daily", "weekly", "monthly"]) + pf.add_argument("--currency", default="base", choices=["base", "local"]) + pf.add_argument("--start", default=None) + pf.add_argument("--snapshot", default=None) + pf.set_defaults(func=cmd_performance) + + op = sub.add_parser("optimize", help="mean-CVaR MILP optimisation") + op.add_argument("--request", help="JSON file with an OptimizationRequest (fields override the flags)") + op.add_argument("--cash", type=float, default=1_000_000.0) + op.add_argument("--holdings", help='JSON dict {"NESN.SW": 100, ...} -> rebalancing mode') + op.add_argument("--symbols", nargs="*", default=None) + op.add_argument("--max-assets", type=int, default=100) + op.add_argument("--horizon", default="week") + op.add_argument("--confidence", type=float, default=0.95) + op.add_argument("--cvar-limit", type=float, default=None, help="max CVaR in CHF over the horizon") + op.add_argument("--scenario-model", default="empirical", choices=["gbm", "empirical", "gaussian_exact"]) + op.add_argument("--n-scenarios", type=int, default=None, + help="default: 16 per candidate within [4096, 16384]; rounded up to a power of two") + op.add_argument("--sampling", default="sobol", choices=["sobol", "mc", "antithetic", "lhs", "balanced"]) + from quantcore.optimization.service import IMPORTANCE_SAMPLING_MODES # lazy, like every command's + + op.add_argument("--importance-sampling", default="auto", + choices=list(IMPORTANCE_SAMPLING_MODES), + help="'market': tilt towards the tail of the cap-weighted portfolio; " + "'adaptive': towards a first-pass solution") + op.add_argument("--row-generation", default="auto", choices=["auto", "on", "off"]) + op.add_argument("--solver", default="auto", choices=["auto", "cuopt", "highs"]) + op.add_argument("--snapshot", default=None) + op.add_argument("--json", action="store_true", help="print the full result as JSON") + op.set_defaults(func=cmd_optimize) + + sv = sub.add_parser("serve", help="run the HTTP API and the browser test bench (/ui/)") + sv.add_argument("--host", default=None) + sv.add_argument("--port", type=int, default=None) + sv.set_defaults(func=cmd_serve) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quantcore/client.py b/quantcore/client.py new file mode 100644 index 0000000..d8e583d --- /dev/null +++ b/quantcore/client.py @@ -0,0 +1,314 @@ +"""HTTP client of the quant-core API -- the package seen from the outside. + +The services of :mod:`quantcore` can be used in two ways. In-process, by importing +``MarketDataService``, ``RiskService``, ``OptimizationService`` and friends, which needs the data +directory on the same machine. Or over HTTP, against a running ``quantcore serve`` (or the container +on the DGX Spark), which needs nothing but a URL -- this module. + +:class:`QuantcoreClient` has one method per endpoint, passes the request bodies through unchanged +(the field names are exactly those of ``quantcore.api.schemas``, so the interactive docs at +``/docs`` are its reference) and returns the parsed JSON. It adds no logic and no defaults of its +own beyond the ones the API already applies: what comes back is what the services computed. + + from quantcore.client import QuantcoreClient + + qc = QuantcoreClient("http://spark-6d4a:5001") + qc.health()["store"]["last_price_date"] + qc.asset_risk(["NESN.SW", "AAPL"], horizon="week", drift="zero") + res = qc.optimize(cash=1_000_000, cvar_limit=40_000, horizon="week", risk_drift="zero") + res["status"], res["cvar"], len(res["positions"]) + +Errors: any answer that is not 2xx raises :class:`QuantcoreError` carrying the status code and the +API's ``detail`` message; an unreachable server raises it too, naming the URL that was tried. The +methods that start background work (``start_setup``, ``start_update``) return immediately with a job +id -- :meth:`QuantcoreClient.wait_for_job` polls it to completion. + +The transport is injectable (``session=``): anything with a ``request(method, url, params=, +json=, timeout=)`` method that returns an object with ``status_code``, ``json()`` and ``text``. +``requests.Session`` is the default; ``fastapi.testclient.TestClient`` satisfies it too, which is +how the tests exercise this client against the app in-process. +""" + +from __future__ import annotations + +import os +import time +from typing import Any, Self + +import requests + +DEFAULT_URL = "http://127.0.0.1:5001" +DEFAULT_TIMEOUT = 900.0 # an optimisation may legitimately run for minutes +JOB_STATES_DONE = ("done", "failed") + + +def default_url() -> str: + """``QUANTCORE_API_URL`` if set, else ``http://127.0.0.1:5001``.""" + return os.environ.get("QUANTCORE_API_URL", DEFAULT_URL) + + +class QuantcoreError(RuntimeError): + """An API answer that is not 2xx, or a server that could not be reached. + + ``status_code`` is ``None`` when the request never got an answer (connection refused, timeout, + DNS failure) -- distinguishing "the service said no" from "there is no service there". + """ + + def __init__(self, message: str, *, status_code: int | None = None, method: str = "", path: str = ""): + super().__init__(message) + self.status_code = status_code + self.method = method + self.path = path + + @property + def unreachable(self) -> bool: + return self.status_code is None + + +def _retrying_session() -> requests.Session: + """A session that retries an idempotent request (GET/HEAD/OPTIONS) when the connection drops + before the answer arrives -- typically a kept-alive connection that the server closed between two + calls: uvicorn closes idle connections after 5 s, which is also the default job poll interval. + POSTs are never retried after they were sent: an optimisation or a job must not run twice.""" + from requests.adapters import HTTPAdapter + from urllib3.util.retry import Retry + + retry = Retry(total=2, connect=2, read=2, status=0, redirect=0, other=0, backoff_factor=0.2, + allowed_methods=frozenset({"GET", "HEAD", "OPTIONS"}), raise_on_status=False) + session = requests.Session() + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + +class QuantcoreClient: + """Client of one quant-core API instance. + + ``base_url``: e.g. ``http://127.0.0.1:5001`` or ``http://spark-6d4a:5001``; a trailing slash and + a trailing ``/ui`` are stripped, so a URL copied out of the browser works. + ``timeout``: seconds per request, overridable per call (``timeout=`` on any method). + """ + + def __init__(self, base_url: str | None = None, timeout: float = DEFAULT_TIMEOUT, session: Any = None): + self.base_url = (base_url or default_url()).rstrip("/").removesuffix("/ui") + self.timeout = float(timeout) + self._session = session + self._owns_session = session is None + + # -- transport ------------------------------------------------------------------------------ + @property + def session(self) -> Any: + if self._session is None: + self._session = _retrying_session() + return self._session + + def close(self) -> None: + if self._owns_session and self._session is not None: + self._session.close() + self._session = None + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def request(self, method: str, path: str, *, params: dict | None = None, json: Any = None, + timeout: float | None = None) -> Any: + """One call against the API. ``params``/``json`` entries that are ``None`` are dropped, so a + method can pass every optional argument and let the API apply its own defaults.""" + if params: + params = {k: v for k, v in params.items() if v is not None} + if isinstance(json, dict): + json = {k: v for k, v in json.items() if v is not None} + url = f"{self.base_url}{path}" + try: + res = self.session.request(method, url, params=params, json=json, + timeout=timeout if timeout is not None else self.timeout) + except requests.RequestException as exc: + raise QuantcoreError(f"cannot reach the quant-core API at {url}: {exc}", + method=method, path=path) from exc + if not 200 <= res.status_code < 300: + raise QuantcoreError(f"{method} {path} -> {res.status_code}: {_detail(res)}", + status_code=res.status_code, method=method, path=path) + return res.json() + + def get(self, path: str, **kw: Any) -> Any: + return self.request("GET", path, **kw) + + def post(self, path: str, **kw: Any) -> Any: + return self.request("POST", path, **kw) + + # -- system --------------------------------------------------------------------------------- + def health(self, **kw: Any) -> dict: + """Store status (symbols, days, last price date), cuOpt reachability, base currency.""" + return self.get("/health", **kw) + + def openapi(self, **kw: Any) -> dict: + """The OpenAPI schema -- the machine-readable list of every endpoint and field.""" + return self.get("/openapi.json", **kw) + + # -- data ----------------------------------------------------------------------------------- + def universe(self, region: str | None = None, sector: str | None = None, country: str | None = None, + **kw: Any) -> list[dict]: + """The investable universe, one row per stock (metadata, market cap, coverage).""" + return self.get("/universe", params={"region": region, "sector": sector, "country": country}, **kw) + + def symbol(self, symbol: str, **kw: Any) -> dict: + """One universe row.""" + return self.get(f"/universe/{symbol}", **kw) + + def symbols_table(self, region: str | None = None, sector: str | None = None, country: str | None = None, + search: str | None = None, limit: int = 200, sort: str = "market_cap", **kw: Any) -> dict: + """The universe as a table plus the filter facets (counts per region, sector, country) -- + one call for a stock picker.""" + return self.get("/data/symbols", params={"region": region, "sector": sector, "country": country, + "search": search, "limit": limit, "sort": sort}, **kw) + + def prices(self, symbols: list[str], start: str | None = None, end: str | None = None, + currency: str = "base", freq: str = "daily", **kw: Any) -> dict: + """Adjusted closes as ``{"index": [...], "columns": [...], "data": [[...]]}``.""" + return self.get("/prices", params={"symbols": ",".join(symbols), "start": start, "end": end, + "currency": currency, "freq": freq}, **kw) + + def returns(self, symbols: list[str], start: str | None = None, end: str | None = None, + currency: str = "base", freq: str = "daily", **kw: Any) -> dict: + """Log-returns in the same frame layout as :meth:`prices`.""" + return self.get("/returns", params={"symbols": ",".join(symbols), "start": start, "end": end, + "currency": currency, "freq": freq}, **kw) + + def market_index(self, start: str | None = None, end: str | None = None, freq: str = "weekly", + **kw: Any) -> dict: + """The cap-weighted broad-market index: its return series and annualised mean/volatility.""" + return self.get("/market/index", params={"start": start, "end": end, "freq": freq}, **kw) + + def market_weights(self, max_assets: int | None = None, **kw: Any) -> dict: + """Current market-capitalisation weights (the Black-Litterman equilibrium portfolio).""" + return self.get("/market/weights", params={"max_assets": max_assets}, **kw) + + # -- risk ----------------------------------------------------------------------------------- + def asset_risk(self, symbols: list[str], **body: Any) -> list[dict]: + """Per-stock volatility, VaR and CVaR (normal and historical, rolling and EWM). + + ``horizon``, ``freq``, ``window``, ``lam``, ``confidence``, ``currency``, and: + ``drift="zero"`` leaves the estimated mean out of the parametric figures (the market-risk + convention, and the only sane choice at long horizons), ``space="value"`` reports them as a + fraction of the position's value instead of as log-returns. + """ + return self.post("/risk/assets", json={"symbols": symbols, **body}) + + def risk_series(self, symbol: str, **body: Any) -> dict: + """Rolling and EWM volatility / VaR / CVaR through time for one stock.""" + return self.post("/risk/series", json={"symbol": symbol, **body}) + + def covariance(self, symbols: list[str], **body: Any) -> dict: + """Covariance matrix as measured (``method="rolling"|"ewm"``, ``window``, ``lam``, + ``shrinkage``). For the estimator the *models* use, see :meth:`model_covariance`.""" + return self.post("/risk/covariance", json={"symbols": symbols, **body}) + + def correlation(self, symbols: list[str], **body: Any) -> dict: + """Correlation matrix, same parameters as :meth:`covariance`.""" + return self.post("/risk/correlation", json={"symbols": symbols, **body}) + + def portfolio_risk(self, holdings: dict[str, float], **body: Any) -> dict: + """VaR/CVaR of a portfolio held in *share numbers* (plus ``cash``), normal or historical.""" + return self.post("/risk/portfolio", json={"holdings": holdings, **body}) + + # -- performance ---------------------------------------------------------------------------- + def asset_performance(self, symbols: list[str], **body: Any) -> list[dict]: + """Annualised return, volatility, Sharpe, Sortino, drawdown, Calmar, skew, kurtosis.""" + return self.post("/performance/assets", json={"symbols": symbols, **body}) + + def rolling_performance(self, symbol: str, **body: Any) -> dict: + """Rolling (``window``) or exponentially weighted (``lam``) performance series.""" + return self.post("/performance/rolling", json={"symbol": symbol, **body}) + + def portfolio_performance(self, holdings: dict[str, float], **body: Any) -> dict: + """Performance of a share-based portfolio, rebalanced or buy-and-hold.""" + return self.post("/performance/portfolio", json={"holdings": holdings, **body}) + + # -- models --------------------------------------------------------------------------------- + def expected_returns(self, **body: Any) -> dict: + """Black-Litterman expected returns: equilibrium ``pi``, the posterior, and every + intermediate quantity (``delta``, ``tau``, the calibrated ``omega``). ``views`` are the same + objects ``/optimize`` takes, so a view can be inspected before it is optimised on.""" + return self.post("/model/expected-returns", json=body) + + def model_covariance(self, symbols: list[str], **body: Any) -> dict: + """The covariance the scenario generator and the optimiser use, with its conditioning + (eigenvalues, applied shrinkage, effective observations).""" + return self.post("/model/covariance", json={"symbols": symbols, **body}) + + def scenarios(self, symbols: list[str], **body: Any) -> dict: + """A scenario set and its statistics: per-stock moments, the P&L distribution of a + portfolio (``holdings`` or ``weights`` + ``value``), histograms, VaR/CVaR, and the + fingerprint (``hash``, ``shock_hash``) that identifies the draws.""" + return self.post("/model/scenarios", json={"symbols": symbols, **body}) + + # -- optimisation --------------------------------------------------------------------------- + def optimize(self, **body: Any) -> dict: + """Mean-CVaR MILP in integer shares. ``mode="new"`` invests ``cash``; ``mode="rebalance"`` + starts from ``holdings`` and pays transaction ``costs``, and the result carries a ``before`` + block so the two states can be compared. + + The result names the solver, the status, the MIP gap, the reproducibility level, the + scenario key and hash, and the provenance of every model input. Can run for minutes: pass + ``time_limit`` (seconds, the budget of the whole request) and a generous ``timeout=``. + """ + return self.post("/optimize", json=body) + + # -- jobs and snapshots --------------------------------------------------------------------- + def start_setup(self, offline: bool = False, max_per_region: int | None = None, **kw: Any) -> dict: + """Build the universe and download all history (~15 min). Returns a job id at once.""" + return self.post("/jobs/setup", params={"offline": offline, "max_per_region": max_per_region}, **kw) + + def start_update(self, full: bool = False, **kw: Any) -> dict: + """Fetch the newest prices and FX rates. Returns a job id at once.""" + return self.post("/jobs/update", params={"full": full}, **kw) + + def jobs(self, **kw: Any) -> list[dict]: + return self.get("/jobs", **kw) + + def job(self, job_id: str, **kw: Any) -> dict: + """One job: ``queued`` | ``running`` | ``done`` | ``failed``, with its report or error.""" + return self.get(f"/jobs/{job_id}", **kw) + + def wait_for_job(self, job_id: str, poll: float = 5.0, deadline: float | None = None, + on_poll: Any = None) -> dict: + """Poll a job until it is ``done`` or ``failed``; returns the final job record. + + ``deadline``: seconds to wait before giving up (``None``: forever). ``on_poll``: a callable + invoked with each intermediate record, for progress output. + """ + started = time.monotonic() + while True: + record = self.job(job_id) + if record.get("status") in JOB_STATES_DONE: + return record + if deadline is not None and time.monotonic() - started > deadline: + raise QuantcoreError(f"job {job_id} still {record.get('status')} after {deadline:.0f} s", + method="GET", path=f"/jobs/{job_id}") + if on_poll is not None: + on_poll(record) + time.sleep(poll) + + def snapshots(self, **kw: Any) -> list[str]: + return self.get("/snapshots", **kw) + + def create_snapshot(self, name: str | None = None, **kw: Any) -> dict: + """Freeze universe, prices and FX under a name, for reproducible demos and grading.""" + return self.post("/snapshots", json={"name": name}, **kw) + + +def _detail(res: Any) -> str: + """The API's error message: FastAPI puts it in ``detail``, validation errors in a list.""" + try: + body = res.json() + except Exception: # noqa: BLE001 (not JSON: a proxy error page, a truncated answer) + return (getattr(res, "text", "") or "")[:500] + detail = body.get("detail", body) if isinstance(body, dict) else body + if isinstance(detail, list): # 422: [{"loc": [...], "msg": ..., "type": ...}, ...] + return "; ".join(f"{'.'.join(str(p) for p in d.get('loc', []))}: {d.get('msg')}" + for d in detail if isinstance(d, dict))[:500] + return str(detail)[:500] diff --git a/quantcore/config.py b/quantcore/config.py new file mode 100644 index 0000000..71401f0 --- /dev/null +++ b/quantcore/config.py @@ -0,0 +1,125 @@ +"""Central configuration (environment variables with the ``QUANTCORE_`` prefix, or a ``.env`` file). + +Every service receives a :class:`Settings` instance -- nothing reads the environment on its own, +which keeps the services testable (tests pass a ``Settings(data_dir=tmp_path)``). +""" + +from __future__ import annotations + +import logging +from functools import lru_cache +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Runtime settings of the quant core.""" + + model_config = SettingsConfigDict(env_prefix="QUANTCORE_", env_file=".env", extra="ignore") + + # --- storage --------------------------------------------------------------------------- + data_dir: Path = Field(default=Path("data"), description="Root of all persisted data.") + + # --- universe -------------------------------------------------------------------------- + n_us: int = Field(default=500, description="Largest N US stocks (S&P 500 candidates).") + n_europe: int = Field(default=500, description="Largest N European stocks (STOXX 600 candidates).") + history_start: str = Field(default="2004-12-01", description="First date requested from Yahoo.") + required_history_from: str = Field( + default="2005-01-31", + description="A stock is kept only if it has a price on or before this date ('data since 2005').", + ) + min_price_coverage: float = Field( + default=0.90, description="Minimum share of union-calendar days with a price since the first date." + ) + max_missing_run: int = Field( + default=20, description="Longest tolerated gap (business days) in a price series since the required date." + ) + + # --- market conventions ---------------------------------------------------------------- + base_currency: str = Field(default="CHF", description="Portfolio / reporting currency.") + home_market: str = Field(default="Switzerland", description="Home market of the investor.") + risk_free_rate: float = Field(default=0.0, description="Annual risk-free rate in the base currency.") + trading_days_per_year: int = 252 + weeks_per_year: int = 52 + + # --- Yahoo access ---------------------------------------------------------------------- + yahoo_batch_size: int = Field(default=100, description="Tickers per bulk download request.") + yahoo_timeout: float = Field(default=30.0, description="Seconds per HTTP request.") + yahoo_max_retries: int = 4 + yahoo_backoff: float = Field(default=2.0, description="Base of the exponential back-off (seconds).") + yahoo_info_pause: float = Field(default=0.15, description="Pause between ticker-info requests (s).") + + # --- cuOpt ----------------------------------------------------------------------------- + cuopt_url: str = Field( + default="http://127.0.0.1:5000", + description="cuOpt REST endpoint. The default is a cuOpt container on the same machine (the " + "standalone DGX Spark setup, deploy/apptainer/); point it elsewhere for a remote one.") + cuopt_time_limit: float = Field(default=120.0, description="MILP time limit in seconds.") + cuopt_poll_interval: float = 1.0 + cuopt_timeout: float = Field(default=900.0, description="Max seconds to wait for a solution.") + cuopt_request_timeout: float = Field(default=60.0, description="Seconds per HTTP call to cuOpt.") + cuopt_upload_seconds_per_mb: float = Field( + default=1.0, description="Added to the submit timeout per MB of request body. cuOpt accepts about " + "10 MB/s, so this is a 10x margin; a client-side timeout during the upload " + "would otherwise surface as an aborted job.") + solver_fallback: bool = Field( + default=True, description="Fall back to the local HiGHS MILP solver (scipy) when cuOpt is unreachable." + ) + cuopt_threads: int | None = Field(default=None, description="CPU threads of the cuOpt MIP solver (None: server default).") + cuopt_msgpack: bool = Field(default=True, description="Send cuOpt requests as msgpack when the package is available.") + + # --- MILP tolerances (reproducibility) --------------------------------------------------- + mip_rel_gap: float = Field(default=1e-4, description="Relative MIP gap for HiGHS and cuOpt.") + mip_abs_gap: float = Field(default=0.01, description="Absolute MIP gap in money units (cuOpt).") + + # --- misc ------------------------------------------------------------------------------ + log_level: str = "INFO" + api_host: str = Field( + default="127.0.0.1", + description="Interface `quantcore serve` binds to: localhost only by default; the Docker images " + "pass 0.0.0.0 to publish the port.") + api_port: int = 5001 + enable_ui: bool = Field( + default=True, + description="Serve the static test bench at /ui/ (and redirect / to it). Set false for a " + "deployment that only exposes the JSON API and /docs -- the same image then " + "serves a front end built by someone else.", + ) + + # derived paths ------------------------------------------------------------------------- + @property + def universe_dir(self) -> Path: + return self.data_dir / "universe" + + @property + def prices_dir(self) -> Path: + return self.data_dir / "prices" + + @property + def fx_dir(self) -> Path: + return self.data_dir / "fx" + + @property + def snapshots_dir(self) -> Path: + return self.data_dir / "snapshots" + + @property + def jobs_dir(self) -> Path: + return self.data_dir / "jobs" + + def ensure_dirs(self) -> None: + for d in (self.universe_dir, self.prices_dir, self.fx_dir, self.snapshots_dir, self.jobs_dir): + d.mkdir(parents=True, exist_ok=True) + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + """Process-wide settings (cached). Tests construct :class:`Settings` directly instead.""" + settings = Settings() + logging.basicConfig( + level=getattr(logging, settings.log_level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + return settings diff --git a/quantcore/measurement/__init__.py b/quantcore/measurement/__init__.py new file mode 100644 index 0000000..58647a5 --- /dev/null +++ b/quantcore/measurement/__init__.py @@ -0,0 +1,18 @@ +"""Risk and performance measurement on log-returns (moving windows and exponential weighting).""" + +from quantcore.measurement.performance import PerformanceService, performance_table +from quantcore.measurement.returns import ( + FREQUENCIES, + HORIZONS, + align_prices, + horizon_periods, + log_returns, + periods_per_year, + resample_prices, +) +from quantcore.measurement.risk import RiskService, asset_risk_table + +__all__ = [ + "FREQUENCIES", "HORIZONS", "align_prices", "horizon_periods", "log_returns", "periods_per_year", + "resample_prices", "RiskService", "asset_risk_table", "PerformanceService", "performance_table", +] diff --git a/quantcore/measurement/estimators.py b/quantcore/measurement/estimators.py new file mode 100644 index 0000000..a5f8146 --- /dev/null +++ b/quantcore/measurement/estimators.py @@ -0,0 +1,228 @@ +"""Moving-window and exponentially weighted estimators of mean, volatility, covariance, correlation. + +Two estimator families, both centred (the corresponding mean is subtracted): + +* **rolling** -- equally weighted window of ``window`` observations (sample covariance, ddof=1); +* **ewm** -- exponentially weighted with decay ``lam`` (RiskMetrics: weight of the + observation ``k`` periods back is proportional to ``lam**k``; ``lam=0.94`` daily, + ``lam=0.99`` for slow adaptation; effective window ~ 1/(1-lam)). + +Per-asset *time series* use pandas (``rolling`` / ``ewm``); full matrices are computed for a +single date (``*_cov``) or streamed date by date (``covariance_sequence``) because a +``T x N x N`` array does not fit in memory for 1000 assets. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import numpy as np +import pandas as pd + + +# --------------------------------------------------------------------------------------------- +# per-asset time series +# --------------------------------------------------------------------------------------------- +def rolling_mean(r: pd.DataFrame | pd.Series, window: int, min_periods: int | None = None): + return r.rolling(window, min_periods=min_periods or max(2, window // 2)).mean() + + +def rolling_vol(r, window: int, periods_per_year: int | None = None, min_periods: int | None = None): + """Rolling standard deviation, optionally annualised (x sqrt(periods per year)).""" + vol = r.rolling(window, min_periods=min_periods or max(2, window // 2)).std(ddof=1) + return vol * np.sqrt(periods_per_year) if periods_per_year else vol + + +def ewm_mean(r, lam: float, min_periods: int = 10): + return r.ewm(alpha=1.0 - lam, adjust=True, min_periods=min_periods).mean() + + +def ewm_vol(r, lam: float, periods_per_year: int | None = None, min_periods: int = 10): + """Exponentially weighted standard deviation (centred, weights normalised to one).""" + vol = np.sqrt(r.ewm(alpha=1.0 - lam, adjust=True, min_periods=min_periods).var(bias=True)) + return vol * np.sqrt(periods_per_year) if periods_per_year else vol + + +def rolling_quantile(r, window: int, q: float, min_periods: int | None = None): + return r.rolling(window, min_periods=min_periods or max(2, window // 2)).quantile(q) + + +# --------------------------------------------------------------------------------------------- +# matrices at one date +# --------------------------------------------------------------------------------------------- +def _window(r: pd.DataFrame, window: int | None, at=None) -> pd.DataFrame: + if at is not None: + r = r.loc[r.index <= pd.Timestamp(at)] + if window is not None: + r = r.iloc[-window:] + return r + + +def rolling_cov(r: pd.DataFrame, window: int, at=None, min_obs: int | None = None) -> pd.DataFrame: + """Sample covariance of the last ``window`` rows (rows with any NaN are dropped).""" + sub = _window(r, window, at).dropna() + min_obs = min_obs or max(3, window // 2) + if len(sub) < min_obs: + raise ValueError(f"only {len(sub)} complete observations in the window (need {min_obs})") + # atleast_2d: np.cov of a single column returns a 0-d array, which DataFrame rejects -- a + # one-asset portfolio is a legitimate request (and the simplest one a caller makes) + cov = np.atleast_2d(np.cov(sub.values, rowvar=False, ddof=1)) + return pd.DataFrame(cov, index=r.columns, columns=r.columns) + + +def ewm_weights(n: int, lam: float) -> np.ndarray: + """Normalised weights for ``n`` observations, oldest first: ``lam**(n-1-t) / sum``.""" + w = lam ** np.arange(n - 1, -1, -1, dtype=float) + return w / w.sum() + + +def ewm_cov(r: pd.DataFrame, lam: float, at=None, max_obs: int | None = None, min_obs: int = 20) -> pd.DataFrame: + """Exponentially weighted covariance at ``at`` (default: last date), centred. + + ``max_obs`` truncates the history (weights below ``lam**max_obs`` are negligible; the + default keeps ``log(1e-4)/log(lam)`` observations, ~1500 for lam=0.994, ~150 for 0.94). + """ + max_obs = max_obs or int(np.ceil(np.log(1e-4) / np.log(lam))) + sub = _window(r, max_obs, at).dropna() + if len(sub) < min_obs: + raise ValueError(f"only {len(sub)} complete observations (need {min_obs})") + x = sub.values + w = ewm_weights(len(x), lam) + m = w @ x + xc = x - m + cov = (xc * w[:, None]).T @ xc + return pd.DataFrame(cov, index=r.columns, columns=r.columns) + + +def cov_to_corr(cov: pd.DataFrame) -> pd.DataFrame: + sd = np.sqrt(np.diag(cov.values)) + with np.errstate(divide="ignore", invalid="ignore"): + corr = cov.values / np.outer(sd, sd) + corr = np.clip(np.nan_to_num(corr), -1.0, 1.0) + np.fill_diagonal(corr, 1.0) + return pd.DataFrame(corr, index=cov.index, columns=cov.columns) + + +def rolling_corr(r: pd.DataFrame, window: int, at=None) -> pd.DataFrame: + return cov_to_corr(rolling_cov(r, window, at)) + + +def ewm_corr(r: pd.DataFrame, lam: float, at=None) -> pd.DataFrame: + return cov_to_corr(ewm_cov(r, lam, at)) + + +def shrinkage_intensity(x: np.ndarray) -> float: + """Data-driven intensity of the shrinkage towards the diagonal (Schaefer & Strimmer 2005). + + For centred observations ``x`` (n x N) the optimal linear shrinkage of the sample covariance + towards its diagonal is ``lambda* = sum_{i!=j} Var(s_ij) / sum_{i!=j} s_ij^2`` (clipped to + [0, 1]) -- the Ledoit-Wolf estimator with the "diagonal, unequal variances" target. It grows + with N / n, i.e. exactly when the sample covariance becomes ill-conditioned. + """ + x = np.asarray(x, dtype=float) + n = len(x) + if n < 3: + return 1.0 + xc = x - x.mean(axis=0) + w_bar = xc.T @ xc / n # mean of the products w_kij + sq = (xc**2).T @ (xc**2) # sum_k w_kij^2 + var_s = n / (n - 1) ** 3 * (sq - n * w_bar**2) + s = n / (n - 1) * w_bar + off = ~np.eye(len(s), dtype=bool) + denom = float((s[off] ** 2).sum()) + if denom <= 0: + return 1.0 + return float(np.clip(var_s[off].sum() / denom, 0.0, 1.0)) + + +def effective_observations(method: str, window: int, lam: float) -> int: + return int(window) if method == "rolling" else int(round(3.0 / (1.0 - lam))) + + +def resolve_shrinkage(shrinkage, r: pd.DataFrame, method: str, window: int, lam: float, at=None) -> float: + """``'auto'`` -> Schaefer-Strimmer intensity on the (effective) estimation window; numbers pass through.""" + if shrinkage in (None, 0, 0.0, "none"): + return 0.0 + if isinstance(shrinkage, str): + if shrinkage != "auto": + raise ValueError("shrinkage must be a number in [0, 1] or 'auto'") + sub = _window(r, effective_observations(method, window, lam), at).dropna() + return shrinkage_intensity(sub.values) + return float(shrinkage) + + +def covariance(r: pd.DataFrame, method: str = "rolling", window: int = 200, lam: float = 0.94, at=None, + shrinkage=0.0) -> pd.DataFrame: + """Dispatch on ``method`` (``'rolling'`` | ``'ewm'``) with optional shrinkage to the diagonal. + + Linear shrinkage ``(1-s) * Sigma + s * diag(Sigma)`` keeps the variances and pulls the + correlations towards zero -- needed when N is not small against the effective window. + ``shrinkage='auto'`` estimates the intensity from the data (:func:`shrinkage_intensity`). + """ + if method == "rolling": + cov = rolling_cov(r, window, at) + elif method == "ewm": + cov = ewm_cov(r, lam, at) + else: + raise ValueError("method must be 'rolling' or 'ewm'") + return shrink(cov, resolve_shrinkage(shrinkage, r, method, window, lam, at)) + + +def shrink(cov: pd.DataFrame, intensity: float) -> pd.DataFrame: + if intensity <= 0: + return cov + if not 0 < intensity <= 1: + raise ValueError("shrinkage intensity must be in [0, 1]") + diag = np.diag(np.diag(cov.values)) + return pd.DataFrame((1 - intensity) * cov.values + intensity * diag, index=cov.index, columns=cov.columns) + + +# --------------------------------------------------------------------------------------------- +# streamed sequences of matrices (for the innovation filter) +# --------------------------------------------------------------------------------------------- +def covariance_sequence( + r: pd.DataFrame, method: str, window: int = 200, lam: float = 0.94, warmup: int | None = None, + shrinkage=0.0, +) -> Iterator[tuple[pd.Timestamp, np.ndarray, np.ndarray]]: + """Yield ``(date_t, Sigma_{t-1}, r_t)`` for every ``t`` after the warm-up. + + ``Sigma_{t-1}`` uses data **up to and including t-1** -- the information set for + standardising the return of day ``t``. The rolling estimator recomputes the window at + every step (O(w N^2) each); the EWM estimator uses the exact recursion of the centred + RiskMetrics filter, so the cost per step is O(N^2). + """ + x = r.dropna().values.astype(float) + dates = r.dropna().index + n_obs, n = x.shape + shrinkage = resolve_shrinkage(shrinkage, r, method, window, lam) # one intensity for the whole sequence + if method == "rolling": + warmup = warmup or window + if n_obs <= warmup: + raise ValueError("not enough observations for the warm-up window") + for t in range(warmup, n_obs): + block = x[t - window : t] + cov = np.atleast_2d(np.cov(block, rowvar=False, ddof=1)) + yield dates[t], shrink_array(cov, shrinkage), x[t] + elif method == "ewm": + warmup = warmup or min(n_obs - 1, max(50, int(2 / (1 - lam)))) + if n_obs <= warmup: + raise ValueError("not enough observations for the warm-up window") + # initialise with the normalised-weight estimate over the warm-up block, then recurse + w = ewm_weights(warmup, lam) + m = w @ x[:warmup] + xc = x[:warmup] - m + cov = (xc * w[:, None]).T @ xc + alpha = 1.0 - lam + for t in range(warmup, n_obs): + yield dates[t], shrink_array(cov, shrinkage), x[t] + d = x[t] - m + m = m + alpha * d + cov = lam * cov + alpha * np.outer(d, x[t] - m) # Welford-style centred update + else: + raise ValueError("method must be 'rolling' or 'ewm'") + + +def shrink_array(cov: np.ndarray, intensity: float) -> np.ndarray: + if intensity <= 0: + return cov + return (1 - intensity) * cov + intensity * np.diag(np.diag(cov)) diff --git a/quantcore/measurement/performance.py b/quantcore/measurement/performance.py new file mode 100644 index 0000000..6278af3 --- /dev/null +++ b/quantcore/measurement/performance.py @@ -0,0 +1,145 @@ +"""Performance measurement on log-returns: return, volatility, Sharpe, drawdown and friends. + +All statistics are computed from one-period log-returns at a given frequency and annualised +with the number of periods per year (252 / 52 / 12). ``sharpe`` uses the annual risk-free rate +of the base currency. Rolling and exponentially weighted variants give time series. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from scipy import stats + +from quantcore.measurement import estimators as est +from quantcore.measurement.returns import periods_per_year, portfolio_log_returns + + +def cumulative_return(r: pd.DataFrame | pd.Series): + """Total simple return over the sample: ``exp(sum r) - 1``.""" + return np.expm1(r.sum()) + + +def annualised_return(r, ppy: int): + """Annualised log-return (arithmetic mean of log-returns x periods per year).""" + return r.mean() * ppy + + +def annualised_volatility(r, ppy: int): + return r.std(ddof=1) * np.sqrt(ppy) + + +def sharpe_ratio(r, ppy: int, risk_free: float = 0.0): + vol = annualised_volatility(r, ppy) + return (annualised_return(r, ppy) - risk_free) / vol + + +def sortino_ratio(r, ppy: int, risk_free: float = 0.0): + downside = r.where(r < 0, 0.0) + dd = np.sqrt((downside**2).mean() * ppy) + return (annualised_return(r, ppy) - risk_free) / dd + + +def drawdown_series(r: pd.Series | pd.DataFrame): + """Drawdown (fraction below the running peak) of the cumulative value path.""" + wealth = np.exp(r.fillna(0.0).cumsum()) + return wealth / wealth.cummax() - 1.0 + + +def max_drawdown(r): + return drawdown_series(r).min() + + +def calmar_ratio(r, ppy: int): + mdd = max_drawdown(r) + ann = np.expm1(annualised_return(r, ppy)) # geometric annual return + return ann / -mdd if np.ndim(mdd) == 0 else ann / -mdd + + +def performance_table(r: pd.DataFrame, freq: str = "daily", risk_free: float = 0.0) -> pd.DataFrame: + """One row per column of ``r`` with the standard performance statistics.""" + ppy = periods_per_year(freq) + out = pd.DataFrame(index=r.columns) + out["start"] = r.apply(lambda c: c.first_valid_index()) + out["end"] = r.index[-1] + out["n_obs"] = r.notna().sum() + out["cumulative_return"] = cumulative_return(r) + out["annualised_return_log"] = annualised_return(r, ppy) + out["annualised_return"] = np.expm1(out["annualised_return_log"]) + out["annualised_volatility"] = annualised_volatility(r, ppy) + out["sharpe"] = sharpe_ratio(r, ppy, risk_free) + out["sortino"] = sortino_ratio(r, ppy, risk_free) + out["max_drawdown"] = max_drawdown(r) + out["calmar"] = np.expm1(out["annualised_return_log"]) / -out["max_drawdown"] + out["skewness"] = r.apply(lambda c: stats.skew(c.dropna())) + out["excess_kurtosis"] = r.apply(lambda c: stats.kurtosis(c.dropna())) + out["hit_ratio"] = (r > 0).sum() / r.notna().sum() + out["best_period"] = r.max() + out["worst_period"] = r.min() + return out + + +def rolling_performance(r: pd.Series, window: int, freq: str = "daily", risk_free: float = 0.0) -> pd.DataFrame: + """Rolling annualised return, volatility, Sharpe and the running drawdown.""" + ppy = periods_per_year(freq) + df = pd.DataFrame(index=r.index) + mean = est.rolling_mean(r, window) + vol = est.rolling_vol(r, window) + df["return_ann"] = mean * ppy + df["volatility_ann"] = vol * np.sqrt(ppy) + df["sharpe"] = (df["return_ann"] - risk_free) / df["volatility_ann"] + df["cumulative_window"] = np.expm1(r.rolling(window, min_periods=window // 2).sum()) + df["drawdown"] = drawdown_series(r) + return df + + +def ewm_performance(r: pd.Series, lam: float, freq: str = "daily", risk_free: float = 0.0) -> pd.DataFrame: + ppy = periods_per_year(freq) + df = pd.DataFrame(index=r.index) + df["return_ann"] = est.ewm_mean(r, lam) * ppy + df["volatility_ann"] = est.ewm_vol(r, lam) * np.sqrt(ppy) + df["sharpe"] = (df["return_ann"] - risk_free) / df["volatility_ann"] + df["drawdown"] = drawdown_series(r) + return df + + +class PerformanceService: + """Deterministic performance measurement for assets and share-based portfolios.""" + + def __init__(self, market_data, risk_free: float = 0.0): + self.md = market_data + self.risk_free = risk_free + + def asset_performance(self, symbols, freq="daily", currency="base", start=None, end=None) -> pd.DataFrame: + r = self.md.returns(symbols, start=start, end=end, currency=currency, freq=freq) + return performance_table(r, freq, self.risk_free) + + def rolling(self, symbol, window=252, freq="daily", currency="base", start=None, end=None) -> pd.DataFrame: + r = self.md.returns([symbol], start=start, end=end, currency=currency, freq=freq)[symbol].dropna() + return rolling_performance(r, window, freq, self.risk_free) + + def ewm(self, symbol, lam=0.99, freq="daily", currency="base", start=None, end=None) -> pd.DataFrame: + r = self.md.returns([symbol], start=start, end=end, currency=currency, freq=freq)[symbol].dropna() + return ewm_performance(r, lam, freq, self.risk_free) + + def portfolio_returns(self, holdings: dict[str, float], freq="daily", currency="base", start=None, end=None, + rebalanced: bool = True) -> pd.Series: + """Log-returns of a share-based portfolio. + + ``rebalanced=True`` keeps today's value weights fixed (a constant-mix benchmark); + ``False`` is buy-and-hold with the given share numbers over the whole history. + """ + symbols = list(holdings) + px = self.md.prices(symbols, start=start, end=end, currency=currency, freq=freq) + shares = pd.Series(holdings, dtype=float).reindex(symbols) + if rebalanced: + values = shares * px.ffill().iloc[-1] + return portfolio_log_returns(np.log(px).diff().iloc[1:], (values / values.sum()).values) + value = (px.ffill() * shares).sum(axis=1) + return np.log(value).diff().iloc[1:].rename("portfolio") + + def portfolio_performance(self, holdings, freq="daily", currency="base", start=None, end=None, + rebalanced=True) -> pd.Series: + rp = self.portfolio_returns(holdings, freq, currency, start, end, rebalanced) + table = performance_table(rp.to_frame("portfolio"), freq, self.risk_free) + return table.iloc[0] diff --git a/quantcore/measurement/returns.py b/quantcore/measurement/returns.py new file mode 100644 index 0000000..6bab155 --- /dev/null +++ b/quantcore/measurement/returns.py @@ -0,0 +1,116 @@ +"""Log-returns at daily / weekly / monthly frequency, calendar alignment and horizons. + +Conventions used throughout the package: + +* prices are adjusted closes, one column per symbol, indexed by calendar date (tz-naive); +* the **union calendar** of all listings is used; a holiday on one exchange is filled with + the previous close (``align_prices``), which yields a zero return on that day; +* ``log_returns`` are ``ln(P_t / P_{t-1})``; a CHF return of a foreign stock is the local + log-return plus the FX log-return (see :mod:`quantcore.data.fx`); +* weekly returns are Friday-to-Friday, monthly returns month-end to month-end; +* a *horizon* (``'day'``, ``'week'``, ``'month'`` or an integer) is expressed in periods of the + chosen frequency (``horizon_periods``), assuming i.i.d. returns that add up over time. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +FREQUENCIES = {"daily": 252, "weekly": 52, "monthly": 12} +HORIZONS_DAYS = {"day": 1, "week": 5, "month": 21, "quarter": 63, "year": 252} +HORIZONS = list(HORIZONS_DAYS) + + +def periods_per_year(freq: str) -> int: + try: + return FREQUENCIES[freq] + except KeyError as exc: + raise ValueError(f"unknown frequency {freq!r}; use one of {list(FREQUENCIES)}") from exc + + +def horizon_periods(horizon: str | int, freq: str = "daily") -> int: + """Number of ``freq`` periods in ``horizon`` (``'week'`` at daily frequency -> 5).""" + if isinstance(horizon, (int, np.integer)): + if horizon < 1: + raise ValueError("horizon must be >= 1 period") + return int(horizon) + try: + days = HORIZONS_DAYS[horizon] + except KeyError as exc: + raise ValueError(f"unknown horizon {horizon!r}; use one of {HORIZONS} or an integer") from exc + periods = days * periods_per_year(freq) / FREQUENCIES["daily"] + if periods < 1: + raise ValueError(f"horizon {horizon!r} is shorter than one {freq} period") + return int(round(periods)) + + +def align_prices(prices: pd.DataFrame, max_ffill: int | None = 5) -> pd.DataFrame: + """Union calendar with limited forward-fill of exchange holidays; leading NaNs stay NaN.""" + prices = prices.sort_index() + # keep business days only (Yahoo occasionally returns a weekend row for some exchanges) + prices = prices[prices.index.dayofweek < 5] + return prices.ffill(limit=max_ffill) if max_ffill else prices + + +def _month_end_alias() -> str: + """``'ME'`` from pandas 2.2 on, ``'M'`` before it (pandas 3 dropped the old spelling).""" + for alias in ("ME", "M"): + try: + pd.tseries.frequencies.to_offset(alias) + except ValueError: + continue + return alias + raise RuntimeError("no month-end frequency alias accepted by this pandas") + + +MONTH_END = _month_end_alias() + + +def resample_prices(prices: pd.DataFrame, freq: str) -> pd.DataFrame: + """Daily prices -> the last observation of each week (Fri) or month.""" + if freq == "daily": + return prices + if freq == "weekly": + return prices.resample("W-FRI").last() + if freq == "monthly": + return prices.resample(MONTH_END).last() + raise ValueError(f"unknown frequency {freq!r}") + + +def log_returns(prices: pd.DataFrame | pd.Series) -> pd.DataFrame | pd.Series: + """``ln(P_t / P_{t-1})``; the first row (all NaN) is dropped.""" + r = np.log(prices).diff() + return r.iloc[1:] if len(r) else r + + +def simple_returns(prices: pd.DataFrame | pd.Series) -> pd.DataFrame | pd.Series: + return prices.pct_change().iloc[1:] + + +def horizon_returns(r: pd.DataFrame | pd.Series, periods: int, overlapping: bool = True): + """Aggregate one-period log-returns to ``periods``-period log-returns (sums). + + Overlapping windows keep ``T - periods + 1`` observations (more data, autocorrelated); + non-overlapping blocks are independent under the i.i.d. assumption but fewer. + """ + if periods == 1: + return r + if overlapping: + return r.rolling(periods).sum().iloc[periods - 1 :] + blocks = np.arange(len(r))[::-1] // periods # block id counted from the end + grouped = r.groupby(blocks[::-1]).sum() + counts = r.groupby(blocks[::-1]).size() + return grouped[counts == periods] + + +def portfolio_log_returns(r: pd.DataFrame, weights: pd.Series | np.ndarray) -> pd.Series: + """Log-return of a portfolio with weights fixed at the start of each period. + + Exact aggregation: the simple portfolio return is the weighted sum of simple returns, + ``ln(1 + sum_i w_i (exp(r_i) - 1))``. Weights are normalised to the invested fraction; a + cash remainder (``1 - sum w``) earns zero. + """ + w = np.asarray(weights, dtype=float) + simple = np.expm1(r.fillna(0.0).values) @ w + return pd.Series(np.log1p(simple), index=r.index, name="portfolio") diff --git a/quantcore/measurement/risk.py b/quantcore/measurement/risk.py new file mode 100644 index 0000000..d099441 --- /dev/null +++ b/quantcore/measurement/risk.py @@ -0,0 +1,401 @@ +"""Value-at-Risk and Conditional Value-at-Risk (expected shortfall) on log-returns. + +Conventions +----------- +* Losses are positive numbers: for a log-return ``r`` over the horizon, the loss in + return space is ``-r``; in money it is ``V0 * (1 - exp(r))``. +* ``VaR_beta`` is the ``beta``-quantile of the loss distribution, ``CVaR_beta`` the expected + loss beyond it (``beta`` = 0.95 or 0.99). +* Horizons: i.i.d. returns scale the mean with ``h`` and the volatility with ``sqrt(h)`` + (parametric); the historical method aggregates ``h`` consecutive returns. + +Methods +------- +* ``normal`` -- Gaussian (GBM) with the chosen volatility estimator; +* ``historical`` -- empirical quantiles of the (window of) realised returns; +* time series of both, driven by moving-window or exponentially weighted volatilities. + +Money amounts ("an absolute level for a period") use the exact mapping of a log-return to a +P&L, so a 99 % CVaR of CHF 12 000 over a week reads: *in the worst 1 % of weeks the average +loss is CHF 12 000*. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +from scipy import stats + +from quantcore.measurement import estimators as est +from quantcore.measurement.returns import horizon_periods, horizon_returns, periods_per_year, portfolio_log_returns + +DEFAULT_CONFIDENCE = (0.95, 0.99) + + +# --------------------------------------------------------------------------------------------- +# closed forms and empirical estimators (return space, positive losses) +# --------------------------------------------------------------------------------------------- +def normal_var(mu: float | np.ndarray, sigma: float | np.ndarray, beta: float): + """Gaussian VaR of the loss ``-r`` with ``r ~ N(mu, sigma^2)``.""" + return -mu + stats.norm.ppf(beta) * sigma + + +def normal_cvar(mu, sigma, beta: float): + """Gaussian expected shortfall: ``-mu + sigma * phi(z_beta) / (1 - beta)``.""" + z = stats.norm.ppf(beta) + return -mu + sigma * stats.norm.pdf(z) / (1.0 - beta) + + +def normal_value_var(mu, sigma, beta: float): + """VaR as a *fraction of the position's value* when the log-return is ``N(mu, sigma^2)``. + + A quantile transforms exactly under the monotone map ``r -> 1 - e^r``, so this is simply + ``1 - exp(-VaR_log)`` -- no Ito term is involved. + """ + return 1.0 - np.exp(-normal_var(mu, sigma, beta)) + + +def normal_value_cvar(mu, sigma, beta: float): + """CVaR as a fraction of the position's value for a lognormal price. + + A CVaR is a tail *mean*, and ``1 - e^{-x}`` is concave, so ``1 - exp(-CVaR_log)`` is not the + expected value loss -- it overstates it (0.7 % relative at sigma = 0.30). The exact figure uses + the truncated lognormal moment ``E[e^r 1{r <= q}] = e^{mu + sigma^2/2} Phi(z_{1-beta} - sigma)``:: + + CVaR_value = 1 - e^{mu + sigma^2/2} Phi(z_{1-beta} - sigma) / (1 - beta) + + which is the same ``sigma^2/2`` correction that turns a log drift into an arithmetic one. + """ + mu = np.asarray(mu, dtype=float) + sigma = np.asarray(sigma, dtype=float) + z = stats.norm.ppf(1.0 - beta) + return 1.0 - np.exp(mu + 0.5 * sigma ** 2) * stats.norm.cdf(z - sigma) / (1.0 - beta) + + +def historical_var(r: pd.Series | pd.DataFrame | np.ndarray, beta: float): + """Empirical VaR: minus the ``(1 - beta)``-quantile of the returns.""" + if isinstance(r, (pd.Series, pd.DataFrame)): + return -r.quantile(1.0 - beta) + return -np.quantile(np.asarray(r), 1.0 - beta, axis=0) + + +def historical_cvar(r: pd.Series | pd.DataFrame | np.ndarray, beta: float): + """Empirical CVaR: minus the mean of the returns at or below the ``(1 - beta)``-quantile.""" + if isinstance(r, pd.DataFrame): + return pd.Series({c: historical_cvar(r[c].dropna(), beta) for c in r.columns}) + x = np.asarray(r.dropna() if isinstance(r, pd.Series) else r, dtype=float) + if x.ndim == 2: + return np.array([historical_cvar(x[:, j], beta) for j in range(x.shape[1])]) + if len(x) == 0: + return np.nan + q = np.quantile(x, 1.0 - beta) + tail = x[x <= q] + return -tail.mean() if len(tail) else -q + + +def scenario_cvar(losses: np.ndarray, beta: float, probabilities: np.ndarray | None = None) -> tuple[float, float]: + """(VaR, CVaR) of a discrete loss sample, consistent with the Rockafellar-Uryasev LP. + + With ``S`` equiprobable scenarios the CVaR is the mean of the ``(1-beta)S`` largest losses + (fractional tail weight handled exactly), which the LP formulation reproduces. With scenario + ``probabilities`` (importance sampling) the tail is the set of largest losses whose probability + adds up to ``1-beta``, the last one taken fractionally. + """ + losses = np.asarray(losses, dtype=float) + if probabilities is None: + losses = np.sort(losses)[::-1] + s = len(losses) + k = (1.0 - beta) * s + n_full = int(np.floor(k + 1e-9)) # tolerance: (1 - 0.95) * 1000 is 50.00000000000004 in floating point + frac = k - n_full if k - n_full > 1e-9 else 0.0 + var = losses[min(n_full, s - 1)] if frac > 0 or n_full == 0 else losses[n_full - 1] + tail = losses[:n_full].sum() + (frac * losses[n_full] if n_full < s else 0.0) + return float(var), float(tail / k) if k > 0 else float(losses[0]) + p = np.asarray(probabilities, dtype=float) + p = p / p.sum() + order = np.argsort(losses)[::-1] + sorted_l, w = losses[order], p[order] + tail_mass = 1.0 - beta + cum = np.cumsum(w) + k = int(np.searchsorted(cum, tail_mass - 1e-12, side="left")) + k = min(k, len(sorted_l) - 1) + var = float(sorted_l[k]) + full_mass = cum[k - 1] if k > 0 else 0.0 + frac = max(tail_mass - full_mass, 0.0) + cvar = (sorted_l[:k] @ w[:k] + frac * sorted_l[k]) / tail_mass + return var, float(cvar) + + +def weighted_quantile(x, q, weights: np.ndarray | None = None) -> np.ndarray: + """Quantiles of a sample that carries probability ``weights`` (importance sampling). + + Column-wise for a 2-D ``x``. Without weights this is ``np.quantile``; with them the weighted + CDF is interpolated at ``c_i = (sum_{j<=i} w_j - w_i / 2) / sum w``, the usual convention that + reproduces the unweighted result for equal weights. + """ + x = np.asarray(x, dtype=float) + q = np.atleast_1d(np.asarray(q, dtype=float)) + flat = x.ndim == 1 + xx = x[:, None] if flat else x + if weights is None: + out = np.quantile(xx, q, axis=0) + else: + w = np.asarray(weights, dtype=float) + order = np.argsort(xx, axis=0) + sorted_x = np.take_along_axis(xx, order, axis=0) + sorted_w = w[order] + cum = np.cumsum(sorted_w, axis=0) - 0.5 * sorted_w + cum = cum / sorted_w.sum(axis=0) + out = np.stack([[np.interp(qq, cum[:, j], sorted_x[:, j]) for j in range(xx.shape[1])] for qq in q]) + return out[:, 0] if flat else out + + +def loss_to_money(loss_return, value: float): + """Exact money loss for a loss in log-return space: ``V0 * (1 - exp(-loss))``.""" + return value * (1.0 - np.exp(-loss_return)) + + +def value_loss(loss_return): + """A loss in log-return space as a fraction of the position's value: ``1 - exp(-x)``. + + A VaR of 0.30 in log terms is a loss of 25.9 % of the value, never 30 %; the difference is + negligible for a day and material for a quarter. It can never reach 100 %. + """ + return 1.0 - np.exp(-np.asarray(loss_return, dtype=float)) + + +RETURN_SPACES = ("log", "value") +DRIFT_MODES = ("estimated", "zero") + + +# --------------------------------------------------------------------------------------------- +# per-asset table at one date +# --------------------------------------------------------------------------------------------- +def asset_risk_table( + r: pd.DataFrame, + freq: str = "daily", + window: int = 200, + lam: float = 0.94, + confidence=DEFAULT_CONFIDENCE, + horizon: str | int = "day", + hist_window: int | None = None, + at=None, + space: str = "log", + drift: str = "estimated", +) -> pd.DataFrame: + """Volatility, VaR and CVaR per asset for one horizon (rows: assets). + + Columns: ``vol_rolling_ann``, ``vol_ewm_ann`` (annualised), then for every confidence + level ``var{cc}_normal_rolling``, ``cvar{cc}_normal_rolling``, ``var{cc}_normal_ewm``, + ``cvar{cc}_normal_ewm``, ``var{cc}_hist``, ``cvar{cc}_hist`` (positive = loss). + + ``space='log'`` reports them in log-return units (the modelling space); ``'value'`` as a + fraction of the position's value, which cannot reach 100 %. The value figures are *computed* + in that space -- the normal ones from the lognormal closed forms, the historical ones from the + simple returns of the same window -- because only the VaR survives the map ``1 - e^{-x}`` + unchanged; a CVaR is a tail mean and would be overstated by converting it afterwards. + + ``drift='estimated'`` carries the window's mean into the parametric VaR/CVaR (``-mu h``); + harmless for a day, dominant for a quarter, where the mean is estimated far less precisely + than the volatility -- a stock in a strong rally then shows a *smaller* VaR than a quiet one. + ``'zero'`` is the usual market-risk convention: volatility only. The historical columns always + carry the drift of their window. + """ + if space not in RETURN_SPACES: + raise ValueError(f"space must be one of {RETURN_SPACES}") + if drift not in DRIFT_MODES: + raise ValueError(f"drift must be one of {DRIFT_MODES}") + ppy = periods_per_year(freq) + h = horizon_periods(horizon, freq) + if at is not None: + r = r.loc[r.index <= pd.Timestamp(at)] + hist_window = hist_window or max(window * 5, 750) + out = pd.DataFrame(index=r.columns) + roll_sd = r.rolling(window, min_periods=window // 2).std(ddof=1).iloc[-1] + roll_mu = r.rolling(window, min_periods=window // 2).mean().iloc[-1] + ewm_sd = est.ewm_vol(r, lam).iloc[-1] + ewm_mu = est.ewm_mean(r, lam).iloc[-1] + out["mean_rolling_ann"] = roll_mu * ppy + out["vol_rolling_ann"] = roll_sd * np.sqrt(ppy) + out["vol_ewm_ann"] = ewm_sd * np.sqrt(ppy) + out["horizon_periods"] = h + hist = horizon_returns(r.iloc[-hist_window:], h, overlapping=True) + var_fn, cvar_fn = (normal_var, normal_cvar) if space == "log" else (normal_value_var, normal_value_cvar) + if drift == "zero": + roll_mu, ewm_mu = roll_mu * 0.0, ewm_mu * 0.0 + hist_sample = hist if space == "log" else np.expm1(hist) # monotone: the same tail, in value units + for beta in confidence: + cc = f"{int(round(beta * 100)):02d}" + out[f"var{cc}_normal_rolling"] = var_fn(roll_mu * h, roll_sd * np.sqrt(h), beta) + out[f"cvar{cc}_normal_rolling"] = cvar_fn(roll_mu * h, roll_sd * np.sqrt(h), beta) + out[f"var{cc}_normal_ewm"] = var_fn(ewm_mu * h, ewm_sd * np.sqrt(h), beta) + out[f"cvar{cc}_normal_ewm"] = cvar_fn(ewm_mu * h, ewm_sd * np.sqrt(h), beta) + out[f"var{cc}_hist"] = historical_var(hist_sample, beta) + out[f"cvar{cc}_hist"] = historical_cvar(hist_sample, beta) + out["n_obs"] = r.notna().sum() + out["as_of"] = r.index[-1] + return out + + +def risk_series( + r: pd.Series, freq: str = "daily", window: int = 200, lam: float = 0.94, confidence=DEFAULT_CONFIDENCE, + horizon: str | int = "day", +) -> pd.DataFrame: + """Time series of rolling / EWM volatility and Gaussian VaR/CVaR of one return series. + + Historical VaR/CVaR use rolling quantiles / tail means over ``window`` (1-period horizon, + scaled with sqrt(h) as an approximation for longer horizons). + """ + ppy = periods_per_year(freq) + h = horizon_periods(horizon, freq) + df = pd.DataFrame(index=r.index) + roll_sd = est.rolling_vol(r, window) + roll_mu = est.rolling_mean(r, window) + e_sd = est.ewm_vol(r, lam) + e_mu = est.ewm_mean(r, lam) + df["vol_rolling_ann"] = roll_sd * np.sqrt(ppy) + df["vol_ewm_ann"] = e_sd * np.sqrt(ppy) + for beta in confidence: + cc = f"{int(round(beta * 100)):02d}" + df[f"var{cc}_normal_rolling"] = normal_var(roll_mu * h, roll_sd * np.sqrt(h), beta) + df[f"cvar{cc}_normal_rolling"] = normal_cvar(roll_mu * h, roll_sd * np.sqrt(h), beta) + df[f"var{cc}_normal_ewm"] = normal_var(e_mu * h, e_sd * np.sqrt(h), beta) + df[f"cvar{cc}_normal_ewm"] = normal_cvar(e_mu * h, e_sd * np.sqrt(h), beta) + q = est.rolling_quantile(r, window, 1.0 - beta) + df[f"var{cc}_hist"] = -q * np.sqrt(h) + tail_mean = r.rolling(window, min_periods=window // 2).apply( + lambda x, b=beta: -x[x <= np.quantile(x, 1.0 - b)].mean(), raw=True + ) + df[f"cvar{cc}_hist"] = tail_mean * np.sqrt(h) + return df + + +# --------------------------------------------------------------------------------------------- +# portfolio risk +# --------------------------------------------------------------------------------------------- +@dataclass +class PortfolioRisk: + """Risk of a share-based portfolio over one horizon, in return space and in money.""" + + as_of: str + value: float + horizon: str | int + horizon_periods: int + freq: str + method: str + confidence: dict # {"0.95": {"var": .., "cvar": .., "var_money": .., "cvar_money": ..}, ...} + volatility_ann: float + expected_return_horizon: float + weights: dict + n_obs: int + covariance_model: dict + + def as_dict(self) -> dict: + return self.__dict__.copy() + + +def portfolio_risk_from_returns( + r: pd.DataFrame, weights: pd.Series, value: float, freq: str, horizon, confidence, method: str, + window: int, lam: float, cov_method: str = "rolling", shrinkage: float = 0.0, scenarios: np.ndarray | None = None, +) -> PortfolioRisk: + """Portfolio VaR/CVaR from asset returns and weights (fraction of ``value`` per asset). + + ``method``: ``'normal'`` (Gaussian with the rolling/EWM covariance), ``'historical'`` + (realised portfolio returns over the last ``window*5`` periods, aggregated to the horizon) + or ``'scenarios'`` (a ``S x N`` array of simulated horizon log-returns, e.g. from + :mod:`quantcore.models.simulation`). + """ + ppy = periods_per_year(freq) + h = horizon_periods(horizon, freq) + w = weights.reindex(r.columns).fillna(0.0) + cov = est.covariance(r, cov_method, window=window, lam=lam, shrinkage=shrinkage) + var_p = float(w.values @ cov.values @ w.values) + mu_p = float(w.values @ (r.iloc[-window:].mean().fillna(0.0).values)) + sd_h, mu_h = np.sqrt(var_p * h), mu_p * h + levels: dict = {} + if method == "normal": + for beta in confidence: + var_r, cvar_r = normal_var(mu_h, sd_h, beta), normal_cvar(mu_h, sd_h, beta) + levels[f"{beta:.2f}"] = _level(var_r, cvar_r, value) + n_obs = int(r.iloc[-window:].dropna().shape[0]) + elif method == "historical": + rp = portfolio_log_returns(r.iloc[-max(window * 5, 750):], w.values) + rp_h = horizon_returns(rp, h, overlapping=True) + for beta in confidence: + levels[f"{beta:.2f}"] = _level(float(historical_var(rp_h, beta)), float(historical_cvar(rp_h, beta)), value) + n_obs = int(len(rp_h)) + elif method == "scenarios": + if scenarios is None: + raise ValueError("method='scenarios' needs a scenario array") + pnl = value * (np.expm1(scenarios) @ w.values) # money P&L per scenario + for beta in confidence: + var_m, cvar_m = scenario_cvar(-pnl, beta) + levels[f"{beta:.2f}"] = { + "var": float(-np.log1p(-var_m / value)) if var_m < value else np.inf, + "cvar": float(-np.log1p(-cvar_m / value)) if cvar_m < value else np.inf, + "var_money": var_m, "cvar_money": cvar_m, + } + n_obs = int(len(scenarios)) + else: + raise ValueError("method must be 'normal', 'historical' or 'scenarios'") + return PortfolioRisk( + as_of=str(r.index[-1].date()), value=float(value), horizon=horizon, horizon_periods=h, freq=freq, + method=method, confidence=levels, volatility_ann=float(np.sqrt(var_p * ppy)), + expected_return_horizon=mu_h, weights=w.round(6).to_dict(), n_obs=n_obs, + covariance_model={"method": cov_method, "window": window, "lambda": lam, "shrinkage": shrinkage}, + ) + + +def _level(var_r: float, cvar_r: float, value: float) -> dict: + return { + "var": float(var_r), "cvar": float(cvar_r), + "var_money": float(loss_to_money(var_r, value)), "cvar_money": float(loss_to_money(cvar_r, value)), + } + + +# --------------------------------------------------------------------------------------------- +# service facade +# --------------------------------------------------------------------------------------------- +class RiskService: + """Deterministic risk measurement on top of :class:`~quantcore.data.market_data.MarketDataService`.""" + + def __init__(self, market_data): + self.md = market_data + + def asset_risk(self, symbols, freq="daily", currency="base", window=200, lam=0.94, + confidence=DEFAULT_CONFIDENCE, horizon="day", start=None, end=None, + space: str = "log", drift: str = "estimated") -> pd.DataFrame: + r = self.md.returns(symbols, start=start, end=end, currency=currency, freq=freq) + return asset_risk_table(r, freq, window, lam, confidence, horizon, space=space, drift=drift) + + def risk_series(self, symbol, freq="daily", currency="base", window=200, lam=0.94, + confidence=DEFAULT_CONFIDENCE, horizon="day", start=None, end=None) -> pd.DataFrame: + r = self.md.returns([symbol], start=start, end=end, currency=currency, freq=freq)[symbol].dropna() + return risk_series(r, freq, window, lam, confidence, horizon) + + def covariance(self, symbols, freq="daily", currency="base", method="rolling", window=200, lam=0.94, + shrinkage=0.0, start=None, end=None) -> pd.DataFrame: + r = self.md.returns(symbols, start=start, end=end, currency=currency, freq=freq) + return est.covariance(r, method, window=window, lam=lam, shrinkage=shrinkage) + + def correlation(self, symbols, freq="daily", currency="base", method="rolling", window=200, lam=0.94, + start=None, end=None) -> pd.DataFrame: + return est.cov_to_corr(self.covariance(symbols, freq, currency, method, window, lam, 0.0, start, end)) + + def portfolio_risk(self, holdings: dict[str, float], cash: float = 0.0, freq="daily", currency="base", + horizon="day", confidence=DEFAULT_CONFIDENCE, method="normal", cov_method="rolling", + window=200, lam=0.94, shrinkage=0.0, start=None, end=None, + scenarios: np.ndarray | None = None) -> PortfolioRisk: + """VaR / CVaR of a portfolio given in **numbers of shares** (plus cash), in money terms.""" + symbols = list(holdings) + px = self.md.latest_prices(symbols, currency=currency) + values = pd.Series(holdings, dtype=float) * px + total = float(values.sum() + cash) + if total <= 0: + raise ValueError("portfolio value must be positive") + weights = values / total + r = self.md.returns(symbols, start=start, end=end, currency=currency, freq=freq) + return portfolio_risk_from_returns(r, weights, total, freq, horizon, confidence, method, window, lam, + cov_method, shrinkage, scenarios) diff --git a/quantcore/models/__init__.py b/quantcore/models/__init__.py new file mode 100644 index 0000000..3886825 --- /dev/null +++ b/quantcore/models/__init__.py @@ -0,0 +1,10 @@ +"""Stochastic models: covariance estimation, price-scenario simulation, Black-Litterman returns.""" + +from quantcore.models.black_litterman import BlackLittermanModel, BlackLittermanResult, View, view_from_sentiment +from quantcore.models.covariance import CovarianceModel +from quantcore.models.simulation import EmpiricalInnovationSimulator, GBMSimulator, ScenarioSet + +__all__ = [ + "BlackLittermanModel", "BlackLittermanResult", "View", "view_from_sentiment", "CovarianceModel", + "EmpiricalInnovationSimulator", "GBMSimulator", "ScenarioSet", +] diff --git a/quantcore/models/black_litterman.py b/quantcore/models/black_litterman.py new file mode 100644 index 0000000..28c3b9f --- /dev/null +++ b/quantcore/models/black_litterman.py @@ -0,0 +1,231 @@ +"""Black-Litterman expected returns with Idzorek's confidence-based view calibration. + +Model (per period, excess returns over the risk-free rate): + +* **Equilibrium**: the market portfolio (capitalisation weights ``w``) is mean-variance + optimal for a risk aversion ``delta`` = (mu_M - r_f) / (w' Sigma w), where ``mu_M`` is the + average return of the capitalisation-weighted broad market index. The implied equilibrium + excess returns are ``pi = delta * Sigma * w``. +* **Views**: ``P mu = Q + e``, ``e ~ N(0, Omega)`` with a diagonal ``Omega``. Each view ``k`` has a + *confidence* ``C_k`` in (0, 1] (Idzorek 2005): the view's uncertainty ``omega_k`` is the one + for which the posterior portfolio moves the fraction ``C_k`` of the way from the market + weights to the weights the view would imply at 100 % confidence. For a single view that + calibration has the closed form ``omega_k = (1 - C_k) / C_k * p_k (tau Sigma) p_k'`` + (Walters 2014), which is what :func:`idzorek_omega` uses -- the tilt-matching minimisation + of the paper is available as :func:`idzorek_omega_numeric` for verification. +* **Posterior**: ``mu_BL = pi + tau Sigma P' (P tau Sigma P' + Omega)^-1 (Q - P pi)`` and + ``Sigma_BL = Sigma + tau Sigma - tau Sigma P' (P tau Sigma P' + Omega)^-1 P tau Sigma``. + +Units: the covariance is per period (weekly here), ``q`` of a view is stated as an **annual +total** return and converted internally. Results are given per period and annualised. + +Reference: T. Idzorek, *A step-by-step guide to the Black-Litterman model, incorporating +user-specified confidence levels*, 2005. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd +from scipy.optimize import minimize_scalar + + +@dataclass +class View: + """A (possibly relative) view: ``sum_i assets[i] * mu_i = q`` with Idzorek confidence.""" + + assets: dict[str, float] + q: float # annual total return of the view portfolio (log-return units) + confidence: float = 0.5 # (0, 1] + label: str = "" + source: str = "" # provenance, e.g. the insight id that produced the view + + def __post_init__(self) -> None: + if not self.assets: + raise ValueError("a view needs at least one asset") + if not 0 < self.confidence <= 1: + raise ValueError("confidence must be in (0, 1]") + + def row(self, symbols: list[str]) -> np.ndarray: + p = np.zeros(len(symbols)) + index = {s: i for i, s in enumerate(symbols)} + for asset, weight in self.assets.items(): + if asset not in index: + raise KeyError(f"view asset {asset!r} is not in the model universe") + p[index[asset]] = weight + return p + + @property + def is_relative(self) -> bool: + return abs(sum(self.assets.values())) < 1e-12 + + +@dataclass +class BlackLittermanResult: + symbols: list[str] + periods_per_year: int + tau: float + delta: float + market_return_ann: float + market_volatility_ann: float + risk_free_ann: float + market_weights: pd.Series + prior: pd.Series # equilibrium excess returns per period + posterior: pd.Series # posterior excess returns per period + posterior_cov: pd.DataFrame # per period + omega: np.ndarray + views: list[dict] = field(default_factory=list) + + @property + def prior_ann(self) -> pd.Series: + return self.prior * self.periods_per_year + self.risk_free_ann + + @property + def posterior_ann(self) -> pd.Series: + return self.posterior * self.periods_per_year + self.risk_free_ann + + @property + def posterior_excess_ann(self) -> pd.Series: + return self.posterior * self.periods_per_year + + def expected_return_per_period(self, freq_periods_per_year: int) -> pd.Series: + """Posterior total expected log-return per period of another frequency (e.g. daily).""" + return self.posterior_ann / freq_periods_per_year + + def unconstrained_weights(self) -> pd.Series: + """Mean-variance weights ``(delta Sigma_BL)^-1 mu_BL`` implied by the posterior.""" + w = np.linalg.solve(self.delta * self.posterior_cov.values, self.posterior.values) + return pd.Series(w, index=self.symbols) + + def as_dict(self) -> dict: + return { + "periods_per_year": self.periods_per_year, "tau": self.tau, "delta": self.delta, + "market_return_ann": self.market_return_ann, "market_volatility_ann": self.market_volatility_ann, + "risk_free_ann": self.risk_free_ann, "omega": self.omega.tolist(), "views": self.views, + "prior_ann": self.prior_ann.round(6).to_dict(), "posterior_ann": self.posterior_ann.round(6).to_dict(), + "market_weights": self.market_weights.round(6).to_dict(), + } + + +def idzorek_omega(p: np.ndarray, tau_sigma: np.ndarray, confidence: float) -> float: + """Closed-form Idzorek uncertainty: ``(1 - C) / C * p (tau Sigma) p'`` (C=1 -> 0).""" + c = float(np.clip(confidence, 1e-9, 1.0)) + base = float(p @ tau_sigma @ p) + return (1.0 - c) / c * base + + +def idzorek_omega_numeric(p: np.ndarray, q: float, pi: np.ndarray, sigma: np.ndarray, tau: float, delta: float, + confidence: float) -> float: + """Idzorek's original tilt-matching calibration of one view (for verification). + + Finds ``omega`` minimising ``|| w(omega) - (w_mkt + C * (w_100 - w_mkt)) ||`` where + ``w(omega) = (delta Sigma)^-1 mu_BL(omega)`` and ``w_100`` is the weight vector at 100 % + confidence (``omega = 0``). + """ + ts = tau * sigma + w_mkt = np.linalg.solve(delta * sigma, pi) + denom_100 = float(p @ ts @ p) + mu_100 = pi + ts @ p * (q - p @ pi) / denom_100 + w_100 = np.linalg.solve(delta * sigma, mu_100) + target = w_mkt + confidence * (w_100 - w_mkt) + + def objective(log_omega: float) -> float: + omega = np.exp(log_omega) + mu = pi + ts @ p * (q - p @ pi) / (denom_100 + omega) + w = np.linalg.solve(delta * sigma, mu) + return float(np.sum((w - target) ** 2)) + + res = minimize_scalar(objective, bounds=(np.log(denom_100) - 25, np.log(denom_100) + 25), method="bounded", + options={"xatol": 1e-10}) + return float(np.exp(res.x)) + + +class BlackLittermanModel: + """Equilibrium returns from market weights and a broad-market average, plus views.""" + + def __init__(self, cov: pd.DataFrame, market_weights: pd.Series, market_return: float, risk_free: float = 0.0, + tau: float = 0.05, periods_per_year: int = 52): + self.symbols = list(cov.index) + self.sigma = cov.values.astype(float) + w = market_weights.reindex(self.symbols).fillna(0.0).astype(float) + if w.sum() <= 0: + raise ValueError("market weights must have a positive sum") + self.w = (w / w.sum()).values + self.market_return = float(market_return) # per period, total + self.risk_free = float(risk_free) # per period + self.tau = float(tau) + self.ppy = int(periods_per_year) + self.market_var = float(self.w @ self.sigma @ self.w) + self.delta = (self.market_return - self.risk_free) / self.market_var + self.pi = self.delta * self.sigma @ self.w # equilibrium excess returns per period + + # -- helpers --------------------------------------------------------------------------------- + @property + def prior(self) -> pd.Series: + return pd.Series(self.pi, index=self.symbols) + + @property + def prior_ann(self) -> pd.Series: + return self.prior * self.ppy + self.risk_free * self.ppy + + def volatility_ann(self) -> pd.Series: + return pd.Series(np.sqrt(np.diag(self.sigma) * self.ppy), index=self.symbols) + + def _views_to_matrices(self, views: list[View], omega_method: str): + k, n = len(views), len(self.symbols) + P = np.zeros((k, n)) + Q = np.zeros(k) + omega = np.zeros(k) + ts = self.tau * self.sigma + rf_ann = self.risk_free * self.ppy + for i, v in enumerate(views): + P[i] = v.row(self.symbols) + # annual total -> per-period excess (the risk-free part cancels for relative views) + Q[i] = (v.q - rf_ann * P[i].sum()) / self.ppy + if omega_method == "idzorek": + omega[i] = idzorek_omega(P[i], ts, v.confidence) + elif omega_method == "he_litterman": + omega[i] = float(P[i] @ ts @ P[i]) + else: + raise ValueError("omega_method must be 'idzorek' or 'he_litterman'") + return P, Q, omega + + # -- posterior ------------------------------------------------------------------------------- + def posterior(self, views: list[View] | None = None, omega_method: str = "idzorek") -> BlackLittermanResult: + views = list(views or []) + ts = self.tau * self.sigma + if not views: + mu, cov_post, omega = self.pi.copy(), self.sigma + ts, np.zeros(0) + else: + P, Q, omega = self._views_to_matrices(views, omega_method) + M = P @ ts @ P.T + np.diag(omega) + adj = np.linalg.solve(M, Q - P @ self.pi) + mu = self.pi + ts @ P.T @ adj + cov_post = self.sigma + ts - ts @ P.T @ np.linalg.solve(M, P @ ts) + return BlackLittermanResult( + symbols=self.symbols, periods_per_year=self.ppy, tau=self.tau, delta=self.delta, + market_return_ann=self.market_return * self.ppy, market_volatility_ann=float(np.sqrt(self.market_var * self.ppy)), + risk_free_ann=self.risk_free * self.ppy, market_weights=pd.Series(self.w, index=self.symbols), + prior=pd.Series(self.pi, index=self.symbols), posterior=pd.Series(mu, index=self.symbols), + posterior_cov=pd.DataFrame(cov_post, index=self.symbols, columns=self.symbols), omega=omega, + views=[{"label": v.label, "assets": v.assets, "q_ann": v.q, "confidence": v.confidence, "source": v.source, + "omega": float(omega[i])} for i, v in enumerate(views)], + ) + + +def view_from_sentiment(model: BlackLittermanModel, symbol: str, sentiment: float, confidence: float, + strength: float = 0.25, label: str = "", source: str = "") -> View: + """Translate a signed assessment into an absolute view on one stock. + + The AI layer delivers *positive / negative* assessments with a degree of belief; the view + return is the equilibrium return shifted by ``sentiment * strength`` annual volatilities: + ``q = pi_ann + r_f + sentiment * strength * sigma_ann``. ``sentiment`` in [-1, 1], + ``confidence`` in (0, 1] (Idzorek), ``strength`` is a modelling parameter (0.25 by default). + """ + if not -1.0 <= sentiment <= 1.0: + raise ValueError("sentiment must be in [-1, 1]") + q = float(model.prior_ann[symbol] + sentiment * strength * model.volatility_ann()[symbol]) + return View({symbol: 1.0}, q=q, confidence=confidence, label=label or f"{symbol} sentiment {sentiment:+.2f}", + source=source) diff --git a/quantcore/models/covariance.py b/quantcore/models/covariance.py new file mode 100644 index 0000000..39c65c9 --- /dev/null +++ b/quantcore/models/covariance.py @@ -0,0 +1,145 @@ +"""Covariance model specification and matrix square roots. + +:class:`CovarianceModel` is the *named* estimator the other services refer to, e.g. +``CovarianceModel("rolling", window=200)`` (200-day equally weighted) or +``CovarianceModel("ewm", lam=0.99)``; it delegates to :mod:`quantcore.measurement.estimators`. + +Square roots: the **symmetric** root ``Sigma^(1/2) = V diag(sqrt(l)) V'`` (from the +eigendecomposition; used for the innovation filter, as in the model specification) and the +**Cholesky** factor (faster, order dependent). Both satisfy ``A A' = Sigma``. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass + +import numpy as np +import pandas as pd + +from quantcore.measurement import estimators as est + + +@dataclass(frozen=True) +class CovarianceModel: + method: str = "rolling" # 'rolling' | 'ewm' + window: int = 200 # rolling window length (observations) + lam: float = 0.94 # EWM decay + shrinkage: float | str = "auto" # linear shrinkage towards the diagonal: number in [0, 1] or 'auto' + + def __post_init__(self) -> None: + if self.method not in ("rolling", "ewm"): + raise ValueError("method must be 'rolling' or 'ewm'") + if not 0 < self.lam < 1: + raise ValueError("lam must be in (0, 1)") + if isinstance(self.shrinkage, str): + if self.shrinkage != "auto": + raise ValueError("shrinkage must be a number in [0, 1] or 'auto'") + elif not 0 <= self.shrinkage <= 1: + raise ValueError("shrinkage must be in [0, 1]") + + def estimate(self, r: pd.DataFrame, at=None) -> pd.DataFrame: + return est.covariance(r, self.method, window=self.window, lam=self.lam, at=at, shrinkage=self.shrinkage) + + def intensity(self, r: pd.DataFrame, at=None) -> float: + """The shrinkage intensity actually applied for ``r`` (resolves ``'auto'``).""" + return est.resolve_shrinkage(self.shrinkage, r, self.method, self.window, self.lam, at) + + def sequence(self, r: pd.DataFrame, warmup: int | None = None): + return est.covariance_sequence(r, self.method, window=self.window, lam=self.lam, warmup=warmup, + shrinkage=self.shrinkage) + + @property + def effective_window(self) -> float: + return float(self.window) if self.method == "rolling" else 1.0 / (1.0 - self.lam) + + @property + def effective_observations(self) -> int: + """Observations the estimator effectively averages over (``3/(1-lambda)`` for EWM).""" + return est.effective_observations(self.method, self.window, self.lam) + + def conditioning(self, r: pd.DataFrame, at=None) -> dict: + """Is this estimator able to support ``N`` assets? + + With fewer effective observations than assets the sample covariance is singular (a + portfolio in its null space looks riskless, which an optimiser will happily exploit). + Shrinkage towards the diagonal repairs it; this report says by how much. ``ratio`` is the + smallest eigenvalue as a fraction of the average variance -- below ~1e-6 the matrix is + numerically singular, the automatic shrinkage keeps it above 2 %. + """ + cov = self.estimate(r, at=at).values + vals = np.linalg.eigvalsh(cov) + scale = float(np.mean(np.diag(cov))) or 1.0 + n = cov.shape[0] + t_eff = self.effective_observations + return { + "n_assets": n, "effective_observations": t_eff, "observations": int(len(r.dropna())), + "shrinkage_applied": round(self.intensity(r, at), 4), + "min_eigenvalue_ratio": float(vals.min() / scale), + "condition_number": float(vals.max() / vals.min()) if vals.min() > 0 else float("inf"), + "rank_deficient": bool(vals.min() <= 1e-10 * scale), + "under_determined": bool(t_eff < n), + } + + def describe(self) -> dict: + d = asdict(self) + d["effective_window"] = round(self.effective_window, 4) # rounded: this dict is part of the scenario key + return d + + @classmethod + def from_dict(cls, d: dict | None) -> "CovarianceModel": + if d is None: + return cls() + if isinstance(d, CovarianceModel): + return d + return cls(**{k: v for k, v in d.items() if k in ("method", "window", "lam", "shrinkage")}) + + +# --------------------------------------------------------------------------------------------- +# matrix roots +# --------------------------------------------------------------------------------------------- +def _as_array(cov) -> np.ndarray: + return cov.values if isinstance(cov, pd.DataFrame) else np.asarray(cov, dtype=float) + + +def symmetric_sqrt(cov, eps_rel: float = 1e-10) -> np.ndarray: + """``Sigma^(1/2)`` via eigendecomposition; tiny/negative eigenvalues are floored.""" + a = _as_array(cov) + vals, vecs = np.linalg.eigh((a + a.T) / 2) + floor = eps_rel * max(vals.max(), 1e-300) + vals = np.maximum(vals, floor) + return (vecs * np.sqrt(vals)) @ vecs.T + + +def symmetric_inv_sqrt(cov, eps_rel: float = 1e-10) -> np.ndarray: + """``Sigma^(-1/2)`` (pseudo-inverse for directions with negligible variance).""" + a = _as_array(cov) + vals, vecs = np.linalg.eigh((a + a.T) / 2) + floor = eps_rel * max(vals.max(), 1e-300) + inv = np.where(vals > floor, 1.0 / np.sqrt(np.maximum(vals, floor)), 0.0) + return (vecs * inv) @ vecs.T + + +def cholesky_factor(cov, jitter_rel: float = 1e-10) -> np.ndarray: + """Lower-triangular ``L`` with ``L L' = Sigma`` (adds a relative jitter if needed).""" + a = _as_array(cov) + a = (a + a.T) / 2 + scale = np.trace(a) / len(a) + for k in range(6): + try: + return np.linalg.cholesky(a + np.eye(len(a)) * scale * jitter_rel * (10**k) * (k > 0)) + except np.linalg.LinAlgError: + continue + raise np.linalg.LinAlgError("covariance matrix is not positive definite even after jitter") + + +def matrix_root(cov, kind: str = "symmetric") -> np.ndarray: + if kind == "symmetric": + return symmetric_sqrt(cov) + if kind == "cholesky": + return cholesky_factor(cov) + raise ValueError("kind must be 'symmetric' or 'cholesky'") + + +def is_positive_definite(cov, tol: float = 0.0) -> bool: + vals = np.linalg.eigvalsh(_as_array(cov)) + return bool(vals.min() > tol) diff --git a/quantcore/models/sampling.py b/quantcore/models/sampling.py new file mode 100644 index 0000000..1ad3f2e --- /dev/null +++ b/quantcore/models/sampling.py @@ -0,0 +1,255 @@ +"""Scenario sampling: quasi-Monte-Carlo draws, moment matching, importance sampling, resampling. + +Why this module exists. The mean-CVaR optimiser is a sample-average approximation: it sees the +model only through the scenario set. With a plain Monte-Carlo set of ``S`` scenarios the *sample +mean* of every asset is off by ``sigma / sqrt(S)`` -- with ``S = 2000`` and weekly volatilities of +3-11 % that is 3-13 % per year, the same size as the expected returns themselves -- and the +optimiser maximises that noise. Measured on the real universe (30 stocks, 8 seeds): two seeds gave +portfolios differing by 100 % of wealth, an in-sample expected return 26 % too high and a true +expected return 10 % below the optimum. The remedies implemented here remove that noise almost +entirely: + +* **quasi-Monte-Carlo** (scrambled Sobol' points, fixed seed) instead of pseudo-random draws -- + ``S`` is rounded up to a power of two, the first Sobol' coordinates are mapped to the largest + principal components (:func:`pca_root`); +* **moment matching** -- the (weighted) sample is centred and whitened so that its first two + moments are *exactly* the model's; only the tail shape remains sampled; +* **importance sampling** -- an exponential tilt of the sampling distribution along a loss + direction (a defensive mixture with the nominal density, so the likelihood ratios stay bounded + by ``1/(1-sum(mixture))``) puts a quarter of the scenarios in the tail instead of ``1-beta`` of + them; scenarios then carry probabilities ``p_s`` that the Rockafellar-Uryasev rows use instead + of ``1/S``. Several shifts can be mixed in one set (e.g. one block at ``z_{0.95}`` and one at + ``z_{0.99}``), which makes the set usable at any confidence level; +* **resampling schemes** for the empirical-innovation model: plain bootstrap, balanced bootstrap + and Sobol' draws on the ranks of the market-factor innovation. + +Everything is deterministic given the seed; :func:`scenario_hash` fingerprints a scenario set so +that two runs can be verified to have used the same one. +""" + +from __future__ import annotations + +import hashlib + +import numpy as np +from scipy.optimize import brentq +from scipy.stats import norm, qmc + +SAMPLING_METHODS = ("sobol", "mc", "antithetic", "lhs") +RESAMPLING_METHODS = ("sobol", "mc", "balanced") + + +def next_power_of_two(n: int) -> int: + return 1 << max(int(n) - 1, 0).bit_length() if n > 1 else 1 + + +# --------------------------------------------------------------------------------------------- +# base samples +# --------------------------------------------------------------------------------------------- +def standard_normal_sample(n: int, dim: int, method: str = "sobol", seed: int | None = 42) -> np.ndarray: + """``n x dim`` standard-normal draws. + + ``'sobol'``: scrambled Sobol' points through the inverse normal CDF (``n`` must be a power of + two -- use :func:`next_power_of_two`); ``'lhs'``: Latin hypercube; ``'antithetic'``: ``n/2`` + pseudo-random draws and their negatives; ``'mc'``: pseudo-random (PCG64). + """ + if method == "sobol": + if n & (n - 1): + raise ValueError("Sobol' sampling needs a power-of-two sample size") + u = qmc.Sobol(d=dim, scramble=True, seed=seed).random(n) + return norm.ppf(np.clip(u, 1e-12, 1 - 1e-12)) + if method == "lhs": + u = qmc.LatinHypercube(d=dim, scramble=True, seed=seed).random(n) + return norm.ppf(np.clip(u, 1e-12, 1 - 1e-12)) + rng = np.random.default_rng(seed) + if method == "mc": + return rng.standard_normal((n, dim)) + if method == "antithetic": + half = rng.standard_normal(((n + 1) // 2, dim)) + return np.vstack([half, -half])[:n] + raise ValueError(f"sampling must be one of {SAMPLING_METHODS}") + + +def pca_root(cov: np.ndarray) -> np.ndarray: + """``A = V diag(sqrt(l))`` with eigenvalues descending, so ``A A' = cov`` and the first + coordinate of a draw maps to the largest principal component (best use of the first Sobol' + dimensions).""" + vals, vecs = np.linalg.eigh(np.asarray(cov, dtype=float)) + order = np.argsort(vals)[::-1] + vals = np.maximum(vals[order], 0.0) + return vecs[:, order] * np.sqrt(vals) + + +# --------------------------------------------------------------------------------------------- +# moment matching +# --------------------------------------------------------------------------------------------- +def moment_match(z: np.ndarray, probabilities: np.ndarray | None = None, target_variance: float = 1.0) -> np.ndarray: + """Affine transform of the rows of ``z`` so that the (weighted) mean is exactly 0 and the + (weighted) covariance exactly ``target_variance * I``. Needs more rows than columns.""" + z = np.asarray(z, dtype=float) + n, d = z.shape + if n <= d: + raise ValueError(f"moment matching needs more scenarios ({n}) than dimensions ({d})") + p = np.full(n, 1.0 / n) if probabilities is None else np.asarray(probabilities, dtype=float) / np.sum(probabilities) + zc = z - p @ z + cov = (zc * p[:, None]).T @ zc + vals, vecs = np.linalg.eigh(cov) + vals = np.maximum(vals, 1e-14 * vals.max()) + inv_root = (vecs / np.sqrt(vals)) @ vecs.T + return zc @ inv_root * np.sqrt(target_variance) + + +def effective_sample_size(probabilities: np.ndarray) -> float: + p = np.asarray(probabilities, dtype=float) + p = p / p.sum() + return float(1.0 / np.sum(p * p)) + + +# --------------------------------------------------------------------------------------------- +# importance sampling (exponential tilt along a loss direction) +# --------------------------------------------------------------------------------------------- +def loss_direction(root: np.ndarray, exposure: np.ndarray) -> np.ndarray: + """Unit vector in draw space along which the loss of the portfolio with money exposure + ``exposure`` (shares x price) increases fastest: ``-(A' v) / ||A' v||`` for ``r = mu + A z``.""" + a = np.asarray(root, dtype=float).T @ np.asarray(exposure, dtype=float) + norm_a = np.linalg.norm(a) + if norm_a <= 0: + raise ValueError("exposure has no risk - cannot tilt") + return -a / norm_a + + +def _as_blocks(shift, mixture) -> list[tuple[float, float]]: + """``[(shift, mixture), ...]`` from scalars or sequences; validates the total mixture.""" + shifts = [float(shift)] if np.isscalar(shift) else [float(x) for x in shift] + mixtures = [float(mixture)] * len(shifts) if np.isscalar(mixture) else [float(x) for x in mixture] + if len(mixtures) != len(shifts): + raise ValueError("shift and mixture must have the same length") + if not shifts or sum(mixtures) <= 0 or sum(mixtures) >= 1 or min(mixtures) <= 0: + raise ValueError("mixtures must be positive and sum to less than 1") + return list(zip(shifts, mixtures)) + + +def tilt_normal_sample(z: np.ndarray, direction: np.ndarray, shift, mixture=0.5 + ) -> tuple[np.ndarray, np.ndarray]: + """Defensive-mixture importance sampling for standard-normal draws. + + A share ``mixture`` of the rows is shifted by ``theta = shift * direction`` (the loss of the + reference portfolio is then centred at its VaR when ``shift = z_beta``); the rest stay + nominal. ``shift`` and ``mixture`` may be sequences, in which case one block of rows is + shifted per entry -- a set tilted at ``z_{0.95}`` and ``z_{0.99}`` at once serves both + confidence levels. Returns the shifted draws and the self-normalised probabilities + ``p_s ~ f(z_s) / g(z_s)`` with ``g = (1-sum e_k) f + sum_k e_k f(. - theta_k)``, whose + likelihood ratios are bounded by ``1 / (1 - sum e_k)``. + """ + blocks = _as_blocks(shift, mixture) + z = np.array(z, dtype=float, copy=True) + n = len(z) + direction = np.asarray(direction, dtype=float) + thetas = [sh * direction for sh, _ in blocks] + start = 0 + for theta, (_, eps) in zip(thetas, blocks): + m = int(round(eps * n)) + z[start : start + m] += theta + start += m + density = (1.0 - sum(e for _, e in blocks)) * np.ones(n) + for theta, (_, eps) in zip(thetas, blocks): + density = density + eps * np.exp(np.clip(z @ theta - theta @ theta / 2.0, -700, 700)) + weights = 1.0 / density + return z, weights / weights.sum() + + +def tilted_day_probabilities(scores: np.ndarray, target_mean: float) -> tuple[np.ndarray, float]: + """Exponential tilt of the empirical distribution of ``scores`` (one per observation) so that + the tilted mean equals ``target_mean``: ``q_t ~ exp(theta * score_t)``. Returns ``(q, theta)``.""" + s = np.asarray(scores, dtype=float) + s0 = s - s.mean() + if target_mean <= s.mean(): + return np.full(len(s), 1.0 / len(s)), 0.0 + if target_mean >= s.max(): + raise ValueError("target mean beyond the largest score - tilt not feasible") + + def tilted_mean(theta): + w = np.exp(theta * (s0 - s0.max())) + return float((w * s).sum() / w.sum()) - target_mean + + hi = 1.0 + while tilted_mean(hi) < 0: + hi *= 2.0 + if hi > 1e6: + raise ValueError("tilt did not converge") + theta = brentq(tilted_mean, 0.0, hi, xtol=1e-10) + w = np.exp(theta * (s0 - s0.max())) + return w / w.sum(), float(theta) + + +# --------------------------------------------------------------------------------------------- +# resampling of historical innovations +# --------------------------------------------------------------------------------------------- +def resample_indices(n_obs: int, n_scenarios: int, horizon: int, method: str = "sobol", seed: int | None = 42, + order: np.ndarray | None = None) -> np.ndarray: + """``n_scenarios x horizon`` observation indices. + + ``'mc'``: uniform bootstrap; ``'balanced'``: every observation used equally often (random + permutation of the tiled index set); ``'sobol'``: scrambled Sobol' points in ``horizon`` + dimensions mapped to ranks -- with ``order`` the observations sorted by, e.g., the market-factor + innovation, so that every day of the horizon covers the loss distribution evenly. + """ + S, h = int(n_scenarios), int(horizon) + if method == "mc": + return np.random.default_rng(seed).integers(0, n_obs, size=(S, h)) + if method == "balanced": + rng = np.random.default_rng(seed) + full, rem = divmod(S * h, n_obs) + pool = [rng.permutation(n_obs) for _ in range(full)] + [rng.permutation(n_obs)[:rem]] + return rng.permutation(np.concatenate(pool)).reshape(S, h) # every observation full or full + 1 times + if method == "sobol": + if S & (S - 1): + raise ValueError("Sobol' resampling needs a power-of-two sample size") + u = qmc.Sobol(d=h, scramble=True, seed=seed).random(S) + ranks = np.minimum((u * n_obs).astype(int), n_obs - 1) + return ranks if order is None else np.asarray(order)[ranks] + raise ValueError(f"resampling must be one of {RESAMPLING_METHODS}") + + +def tilted_resample(scores: np.ndarray, n_scenarios: int, horizon: int, target_mean, seed: int | None = 42, + mixture=0.5) -> tuple[np.ndarray, np.ndarray, list[float]]: + """Importance-sampled bootstrap: one block of scenarios per ``target_mean`` draws its + ``horizon`` observations from the correspondingly tilted day distribution ``q`` + (:func:`tilted_day_probabilities` on ``scores`` = loss-direction innovations), the remaining + scenarios uniformly. ``target_mean`` and ``mixture`` may be sequences (one block each). + Returns ``(indices, probabilities, thetas)`` with self-normalised likelihood ratios + ``1 / ((1-sum e_k) + sum_k e_k prod_j T q^k_{t_j})``.""" + blocks = _as_blocks(target_mean, mixture) + T = len(scores) + rng = np.random.default_rng(seed) + S, h = int(n_scenarios), int(horizon) + qs, thetas = [], [] + for target, _ in blocks: + q, theta = tilted_day_probabilities(scores, target) + qs.append(q) + thetas.append(theta) + idx = np.empty((S, h), dtype=int) + start = 0 + for q, (_, eps) in zip(qs, blocks): + m = int(round(eps * S)) + idx[start : start + m] = rng.choice(T, size=(m, h), p=q) + start += m + idx[start:] = rng.integers(0, T, size=(S - start, h)) + density = (1.0 - sum(e for _, e in blocks)) * np.ones(S) + for q, (_, eps) in zip(qs, blocks): + density = density + eps * np.exp(np.clip(np.log(T * q[idx]).sum(axis=1), -700, 700)) + weights = 1.0 / density + return idx, weights / weights.sum(), thetas + + +# --------------------------------------------------------------------------------------------- +# fingerprint +# --------------------------------------------------------------------------------------------- +def scenario_hash(scenarios: np.ndarray, probabilities: np.ndarray | None = None) -> str: + """SHA-256 of the scenario matrix (rounded to 1e-12) and its probabilities -- two optimisations + with the same hash used identical inputs.""" + h = hashlib.sha256() + h.update(np.ascontiguousarray(np.round(np.asarray(scenarios, dtype=float), 12)).tobytes()) + if probabilities is not None: + h.update(np.ascontiguousarray(np.round(np.asarray(probabilities, dtype=float), 15)).tobytes()) + return h.hexdigest()[:16] diff --git a/quantcore/models/simulation.py b/quantcore/models/simulation.py new file mode 100644 index 0000000..5ecaccc --- /dev/null +++ b/quantcore/models/simulation.py @@ -0,0 +1,403 @@ +"""Stock-price models: forward simulation of horizon log-returns. + +Both models assume i.i.d. (no autocorrelation) daily or weekly log-returns, so an ``h``-period +return is the sum of ``h`` one-period draws. + +**Geometric Brownian motion** (multivariate normal):: + + r_h ~ N(h * m, h * Sigma_T) Sigma_T from the chosen covariance model + +**Empirical innovations** (filtered historical simulation):: + + eps_t = Sigma_{t-1}^(-1/2) r_t standardised with the covariance known at t-1 + r(j | T) = h * m + Sigma_T^(1/2) sum_{k=1..h} eps_{j_k}, j_k drawn at random from history + +The innovations keep the empirical shape of the return distribution (fat tails, skew, +co-movements beyond correlation) while the scale comes from the *latest* covariance. + +**Drift convention.** The expected-return model (Black-Litterman) delivers *arithmetic* expected +returns ``mu`` (mean-variance and reverse optimisation are statements about simple returns). The +log-drift is therefore ``m = ln(1 + mu_h) / h - diag(Sigma) / 2`` (Ito correction), so that the +expected simple return over the horizon is exactly ``mu_h = mu * h``; with the ``'log'`` convention +``mu`` is used as the log-drift directly and the expected simple return carries the ``sigma^2/2`` +bonus (up to +30 % p.a. for the most volatile stocks -- not what a mean-variance model means). + +**Sampling.** Draws come from :mod:`quantcore.models.sampling`: scrambled Sobol' points by +default, moment matching so that the scenario mean and covariance are exactly the model's, an +optional importance-sampling tilt along the loss direction of a reference portfolio (scenarios +then carry probabilities), and a fingerprint of the final set. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field, replace + +import numpy as np +import pandas as pd +from scipy.stats import norm + +from quantcore.measurement.risk import scenario_cvar, weighted_quantile +from quantcore.models import sampling as smp +from quantcore.models.covariance import CovarianceModel, matrix_root, symmetric_inv_sqrt + +log = logging.getLogger(__name__) + +DRIFT_CONVENTIONS = ("arithmetic", "log") + + +@dataclass +class TiltSpec: + """Importance-sampling tilt along the loss direction of the portfolio with money ``exposure`` + (shares x price per asset). + + ``shift`` is given in standard deviations of that portfolio's loss: a block of ``mixture`` + scenarios is moved by that much, so a shift of ``z_beta`` centres the loss at its VaR. Both + may be sequences -- the default tilts one block at ``z_{0.95}`` and one at ``z_{0.99}``, which + makes one scenario set serve both confidence levels, so that the set does not depend on the + ``confidence`` of a particular optimisation run. + """ + + exposure: np.ndarray + shift: tuple[float, ...] = (1.6448536269514722, 2.3263478740408408) # z_0.95, z_0.99 + mixture: tuple[float, ...] = (0.25, 0.25) + label: str = "market" # provenance: which portfolio defined the direction + + @classmethod + def at_confidence(cls, exposure, confidence: float, mixture: float = 0.5, label: str = "adaptive") -> "TiltSpec": + """A single block shifted to the VaR of ``confidence`` (depends on that confidence).""" + return cls(exposure=exposure, shift=(float(norm.ppf(confidence)),), mixture=(float(mixture),), label=label) + + def describe(self) -> dict: + return {"shift_sd": list(self.shift), "mixture": list(self.mixture), "direction": self.label} + + +@dataclass +class ScenarioSet: + """``S x N`` matrix of simulated horizon log-returns plus its provenance.""" + + log_returns: np.ndarray + symbols: list[str] + horizon_periods: int + freq: str + model: str + covariance_model: dict + as_of: str + seed: int | None = None + drift: np.ndarray | None = None # horizon log-drift per asset + expected_simple: np.ndarray | None = None # model expectation of the horizon simple return + probabilities: np.ndarray | None = None # None = equiprobable + horizon_variance: np.ndarray | None = None # diag of h * Sigma_T (for :meth:`with_drift`) + extra: dict = field(default_factory=dict) + + @property + def n_scenarios(self) -> int: + return int(self.log_returns.shape[0]) + + @property + def weights(self) -> np.ndarray: + if self.probabilities is None: + return np.full(self.n_scenarios, 1.0 / self.n_scenarios) + return self.probabilities + + @property + def hash(self) -> str: + """Fingerprint of the scenario matrix and its probabilities.""" + return smp.scenario_hash(self.log_returns, self.probabilities) + + @property + def shock_hash(self) -> str: + """Fingerprint of the *shocks* only (the drift removed). Two sets with the same shock hash + differ at most in the expected-return model, so portfolios optimised on them see identical + risk; a changed view or risk-free rate shifts the scenarios but not this hash.""" + drift = np.zeros(self.log_returns.shape[1]) if self.drift is None else self.drift + return smp.scenario_hash(self.log_returns - drift, self.probabilities) + + @property + def shocks(self) -> np.ndarray: + """The scenario matrix with the drift removed: the random part of the model.""" + return self.log_returns if self.drift is None else self.log_returns - self.drift + + def with_drift(self, mu_period, convention: str = "arithmetic") -> "ScenarioSet": + """The same shocks with a different expected-return model. + + Changing views, the risk-free rate or the expected-return model shifts every scenario by a + constant per asset; the random part is untouched. This lets the service keep one scenario + set for a whole family of runs (and skips the expensive innovation filter). + """ + if self.horizon_variance is None: + raise ValueError("this ScenarioSet has no horizon variance - cannot re-drift") + log_drift, expected_simple = horizon_drift( + _mu_vector(mu_period, self.symbols), np.diag(self.horizon_variance) / self.horizon_periods, + self.horizon_periods, convention) + return replace(self, log_returns=self.shocks + log_drift, drift=log_drift, + expected_simple=expected_simple, extra={**self.extra, "drift_convention": convention}) + + def simple_returns(self) -> np.ndarray: + return np.expm1(self.log_returns) + + def mean_simple_returns(self) -> np.ndarray: + """Probability-weighted scenario mean of the simple returns (the sample counterpart of + :attr:`expected_simple`).""" + return self.weights @ self.simple_returns() + + def summary(self) -> pd.DataFrame: + """Per-asset moments and quantiles, in both spaces. + + The unprefixed columns are **log**-returns (the space the model works in, where an ``h``-period + return is a sum of ``h`` draws); the ``_simple`` columns are the same scenarios as price + returns, ``e^x - 1``, which is what a position actually gains or loses and cannot fall below + -100 %. A quarterly log-return of -2.0 is a price return of -86 %, so the two must never be + confused when the horizon is long. + """ + r = self.log_returns + w = self.weights + mean = w @ r + std = np.sqrt(w @ (r - mean) ** 2) + qs = weighted_quantile(r, (0.01, 0.05, 0.50, 0.95), self.probabilities) + simple = np.expm1(r) + mean_s = w @ simple + std_s = np.sqrt(w @ (simple - mean_s) ** 2) + qs_s = np.expm1(qs) # a monotone map, so the quantiles carry over exactly + return pd.DataFrame( + { + "mean": mean, "std": std, + "q01": qs[0], "q05": qs[1], "q50": qs[2], "q95": qs[3], + "min": r.min(axis=0), "max": r.max(axis=0), + "mean_simple": mean_s, "std_simple": std_s, + "q01_simple": qs_s[0], "q05_simple": qs_s[1], "q50_simple": qs_s[2], "q95_simple": qs_s[3], + "min_simple": np.expm1(r.min(axis=0)), "max_simple": np.expm1(r.max(axis=0)), + "expected_simple": self.expected_simple if self.expected_simple is not None else np.expm1(mean), + }, + index=self.symbols, + ) + + def portfolio_pnl(self, exposure) -> np.ndarray: + """Money P&L per scenario of a portfolio held with money ``exposure`` per asset.""" + return self.simple_returns() @ np.asarray(exposure, dtype=float) + + def distribution(self, exposure, bins: int = 60, confidence=(0.95, 0.99), + quantiles=(0.01, 0.05, 0.25, 0.50, 0.75, 0.95, 0.99)) -> dict: + """The horizon P&L distribution of a portfolio: moments, quantiles, VaR/CVaR, histogram. + + Everything is probability-weighted, so an importance-sampled set (whose raw sample is *not* + the model distribution) reports the figures of the model. + """ + pnl = self.portfolio_pnl(exposure) + w = self.weights + mean = float(w @ pnl) + var = float(w @ (pnl - mean) ** 2) + sd = float(np.sqrt(max(var, 0.0))) + skew = float(w @ (pnl - mean) ** 3 / sd ** 3) if sd > 0 else 0.0 + kurt = float(w @ (pnl - mean) ** 4 / sd ** 4) if sd > 0 else 0.0 + mass, edges = np.histogram(pnl, bins=bins, weights=w) + risk = {} + for beta in confidence: + var_m, cvar_m = scenario_cvar(-pnl, float(beta), self.probabilities) + risk[f"{float(beta):g}"] = {"var": float(var_m), "cvar": float(cvar_m)} + return { + "n_scenarios": self.n_scenarios, "weighted": self.probabilities is not None, + "effective_sample_size": round(smp.effective_sample_size(w), 1), + "expected_pnl": mean, "volatility": sd, "skewness": skew, "kurtosis": kurt, + "min": float(pnl.min()), "max": float(pnl.max()), + "quantiles": {f"{q:g}": float(v) for q, v in + zip(quantiles, weighted_quantile(pnl, quantiles, self.probabilities))}, + "risk": risk, + "histogram": {"edges": edges.tolist(), "mass": mass.tolist()}, + } + + def describe(self) -> dict: + return { + "model": self.model, "n_scenarios": self.n_scenarios, "n_assets": len(self.symbols), + "horizon_periods": self.horizon_periods, "freq": self.freq, "as_of": self.as_of, "seed": self.seed, + "weighted": self.probabilities is not None, + "effective_sample_size": round(smp.effective_sample_size(self.weights), 1), + "hash": self.hash, "shock_hash": self.shock_hash, "covariance_model": self.covariance_model, + **self.extra, + } + + +# --------------------------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------------------------- +def _mu_vector(mu, symbols) -> np.ndarray: + if mu is None: + return np.zeros(len(symbols)) + if isinstance(mu, pd.Series): + mu = mu.reindex(symbols).fillna(0.0).values + return np.asarray(mu, dtype=float) + + +def horizon_drift(mu_period, cov_period, horizon_periods: int, convention: str = "arithmetic" + ) -> tuple[np.ndarray, np.ndarray]: + """``(log_drift, expected_simple)`` over the horizon for per-period expected returns ``mu``. + + ``'arithmetic'``: ``mu`` is the expected simple return per period; the log-drift gets the + Ito correction so that ``E[exp(r_h) - 1] = h mu`` exactly under normality. + ``'log'``: ``mu`` is the log-drift; the expected simple return is ``exp(h mu + h sigma^2/2) - 1``. + """ + if convention not in DRIFT_CONVENTIONS: + raise ValueError(f"drift_convention must be one of {DRIFT_CONVENTIONS}") + mu = np.asarray(mu_period, dtype=float) + var_h = np.diag(np.asarray(cov_period, dtype=float)) * horizon_periods + mu_h = mu * horizon_periods + if convention == "arithmetic": + mu_h = np.maximum(mu_h, -0.99) + return np.log1p(mu_h) - 0.5 * var_h, mu_h + return mu_h, np.expm1(mu_h + 0.5 * var_h) + + +def _finalise(z_scaled: np.ndarray, root: np.ndarray, log_drift: np.ndarray) -> np.ndarray: + return log_drift + z_scaled @ root.T + + +# --------------------------------------------------------------------------------------------- +# GBM +# --------------------------------------------------------------------------------------------- +class GBMSimulator: + """Multivariate normal horizon returns with the latest covariance of ``cov_model``. + + ``root``: ``'pca'`` (eigen-root, largest component first -- the right choice for Sobol' + points), ``'symmetric'`` or ``'cholesky'``; the distribution is the same for all three. + """ + + def __init__(self, cov_model: CovarianceModel | None = None, root: str = "pca"): + self.cov_model = cov_model or CovarianceModel() + self.root = root + + def _root(self, cov) -> np.ndarray: + return smp.pca_root(cov) if self.root == "pca" else matrix_root(cov, self.root) + + def simulate(self, r: pd.DataFrame, n_scenarios: int = 4096, horizon_periods: int = 1, mu=None, + seed: int | None = 42, freq: str = "daily", sampling: str = "sobol", moment_match: bool = True, + tilt: TiltSpec | None = None, drift_convention: str = "arithmetic") -> ScenarioSet: + symbols = list(r.columns) + cov = self.cov_model.estimate(r).values + a_h = self._root(cov) * np.sqrt(horizon_periods) + log_drift, expected_simple = horizon_drift(_mu_vector(mu, symbols), cov, horizon_periods, drift_convention) + n_req = int(n_scenarios) + S = smp.next_power_of_two(n_req) if sampling == "sobol" else n_req + z = smp.standard_normal_sample(S, len(symbols), sampling, seed) + probabilities = None + info: dict = {"sampling": sampling, "moment_matched": bool(moment_match), "n_requested": n_req, + "drift_convention": drift_convention, "root": self.root} + if tilt is not None: + direction = smp.loss_direction(a_h, tilt.exposure) + z, probabilities = smp.tilt_normal_sample(z, direction, tilt.shift, tilt.mixture) + info["importance_sampling"] = tilt.describe() + if moment_match: + z = smp.moment_match(z, probabilities) + scen = _finalise(z, a_h, log_drift) + desc = self.cov_model.describe() + desc["shrinkage_applied"] = self.cov_model.intensity(r) + return ScenarioSet(scen, symbols, horizon_periods, freq, "gbm", desc, str(r.index[-1].date()), seed, log_drift, + expected_simple, probabilities, np.diag(cov) * horizon_periods, info) + + +# --------------------------------------------------------------------------------------------- +# empirical innovations +# --------------------------------------------------------------------------------------------- +class EmpiricalInnovationSimulator: + """Bootstrap of standardised historical innovations rescaled with the latest covariance.""" + + def __init__(self, cov_model: CovarianceModel | None = None, root: str = "symmetric", center: bool = True, + max_history: int | None = None, clip: float | None = 8.0, standardize: str | None = "diagonal"): + self.cov_model = cov_model or CovarianceModel() + self.root = root + self.center = center + self.max_history = max_history + self.clip = clip # winsorise standardised innovations at +/- clip (a guard, not a model choice) + # re-standardisation of the innovation sample before resampling: 'diagonal' (unit variance per + # asset), 'full' (whitening with the sample covariance of the innovations, needs T >> N) or None. + # A lagged volatility filter leaves the innovations with variance > 1 on average (the forecast + # is too low after every shock); rescaling makes the scenario covariance reproduce Sigma_T. + if standardize not in (None, "diagonal", "full"): + raise ValueError("standardize must be None, 'diagonal' or 'full'") + self.standardize = standardize + + def innovations(self, r: pd.DataFrame) -> pd.DataFrame: + """``eps_t = Sigma_{t-1}^(-1/2) r_t`` for every date after the estimator's warm-up.""" + r = r.dropna() + if self.max_history: + r = r.iloc[-self.max_history :] + rows, dates = [], [] + for date, cov_prev, ret in self.cov_model.sequence(r): + if self.root == "cholesky": + from scipy.linalg import solve_triangular + + from quantcore.models.covariance import cholesky_factor + + eps = solve_triangular(cholesky_factor(cov_prev), ret, lower=True) + else: + eps = symmetric_inv_sqrt(cov_prev) @ ret + rows.append(eps) + dates.append(date) + if not rows: + raise ValueError("no innovations - history shorter than the estimator warm-up") + eps = np.vstack(rows) + if self.clip: + n_clipped = int((np.abs(eps) > self.clip).sum()) + if n_clipped: + log.info("innovations: %d values winsorised at +/-%.0f", n_clipped, self.clip) + eps = np.clip(eps, -self.clip, self.clip) + return pd.DataFrame(eps, index=pd.DatetimeIndex(dates), columns=r.columns) + + def simulate(self, r: pd.DataFrame, n_scenarios: int = 4096, horizon_periods: int = 1, mu=None, + seed: int | None = 42, freq: str = "daily", innovations: pd.DataFrame | None = None, + sampling: str = "sobol", moment_match: bool = True, tilt: TiltSpec | None = None, + drift_convention: str = "arithmetic") -> ScenarioSet: + symbols = list(r.columns) + eps = innovations if innovations is not None else self.innovations(r) + e = eps.values - (eps.values.mean(axis=0) if self.center else 0.0) + e, scale_info = self._restandardize(e) + cov = self.cov_model.estimate(r).values + a = matrix_root(cov, self.root) + log_drift, expected_simple = horizon_drift(_mu_vector(mu, symbols), cov, horizon_periods, drift_convention) + h = int(horizon_periods) + n_req = int(n_scenarios) + S = smp.next_power_of_two(n_req) if sampling == "sobol" else n_req + # market-factor ordering of the observations (projection on the dominant direction of Sigma_T) + w1 = np.linalg.eigh(cov)[1][:, -1] + order = np.argsort(e @ (a @ w1)) + probabilities = None + info: dict = {"n_innovations": len(e), "root": self.root, "standardize": self.standardize, + "sampling": sampling, "moment_matched": bool(moment_match), "n_requested": n_req, + "drift_convention": drift_convention, **scale_info} + if tilt is not None: + direction = smp.loss_direction(a, tilt.exposure) # in innovation space + scores = e @ direction + targets = tuple(sh / np.sqrt(h) for sh in tilt.shift) # per-day mean of the tilted score + idx, probabilities, thetas = smp.tilted_resample(scores, S, h, targets, seed, tilt.mixture) + info["importance_sampling"] = {**tilt.describe(), "theta": [round(t, 4) for t in thetas]} + else: + idx = smp.resample_indices(len(e), S, h, sampling, seed, order) + summed = e[idx].sum(axis=1) # S x N sums of h innovations, covariance ~ h I + if moment_match: + summed = smp.moment_match(summed, probabilities, target_variance=float(h)) + scen = _finalise(summed, a, log_drift) + desc = self.cov_model.describe() + desc["shrinkage_applied"] = self.cov_model.intensity(r) + return ScenarioSet(scen, symbols, horizon_periods, freq, "empirical", desc, str(r.index[-1].date()), seed, + log_drift, expected_simple, probabilities, np.diag(cov) * horizon_periods, info) + + def _restandardize(self, e: np.ndarray) -> tuple[np.ndarray, dict]: + """Rescale the centred innovation sample to unit variance (diagonal) or identity covariance (full).""" + if self.standardize is None: + return e, {} + var = e.var(axis=0, ddof=1) + info = {"innovation_variance_median": float(np.median(var))} + if self.standardize == "diagonal": + return e / np.sqrt(np.where(var > 0, var, 1.0)), info + n_obs, n = e.shape + if n_obs < 3 * n: + log.warning("full whitening needs T >= 3N (T=%d, N=%d) - using diagonal rescaling", n_obs, n) + return e / np.sqrt(np.where(var > 0, var, 1.0)), info + return e @ symmetric_inv_sqrt(np.cov(e, rowvar=False)), info + + +def make_simulator(model: str, cov_model: CovarianceModel | None = None, **kwargs): + if model == "gbm": + return GBMSimulator(cov_model, **kwargs) + if model in ("empirical", "innovations", "fhs"): + return EmpiricalInnovationSimulator(cov_model, **kwargs) + raise ValueError("model must be 'gbm' or 'empirical'") diff --git a/quantcore/optimization/__init__.py b/quantcore/optimization/__init__.py new file mode 100644 index 0000000..2c6bdfb --- /dev/null +++ b/quantcore/optimization/__init__.py @@ -0,0 +1,17 @@ +"""Mean-CVaR portfolio optimisation in integer share numbers (MILP on cuOpt, HiGHS fallback).""" + +from quantcore.optimization.problem import ( + MeanCVaRProblem, + MILPModel, + OptimizationConstraints, + ProblemSpec, + TransactionCosts, +) +from quantcore.optimization.service import OptimizationRequest, OptimizationResult, OptimizationService +from quantcore.optimization.solvers import CuOptSolver, HighsSolver, SolverResult, solve_with_fallback + +__all__ = [ + "MeanCVaRProblem", "MILPModel", "OptimizationConstraints", "ProblemSpec", "TransactionCosts", + "OptimizationRequest", "OptimizationResult", "OptimizationService", "CuOptSolver", "HighsSolver", + "SolverResult", "solve_with_fallback", +] diff --git a/quantcore/optimization/gaussian_exact.py b/quantcore/optimization/gaussian_exact.py new file mode 100644 index 0000000..01b7d0c --- /dev/null +++ b/quantcore/optimization/gaussian_exact.py @@ -0,0 +1,95 @@ +"""Exact mean-CVaR optimisation under the Gaussian model by outer approximation. + +If the horizon returns are normal, ``R ~ N(m, C)``, the money P&L of ``v = P * n`` is normal and + + CVaR_beta(n) = TC(n) - m'v + k_beta * sqrt(v' C v), k_beta = phi(z_beta) / (1 - beta). + +This is the mean-variance problem in disguise (for elliptical returns every portfolio on the +mean-CVaR frontier is on the mean-variance frontier), but in integer shares it is a convex +mixed-integer second-order-cone programme, which neither cuOpt nor HiGHS solves directly. The +convex function is instead outer-approximated by tangent cuts + + t >= TC(n) + (k_beta C v_k / sigma_k - m)' v (sigma_k = sqrt(v_k' C v_k)) + +on the same MILP as the scenario problem (:class:`MeanCVaRProblem` in Gaussian mode). Starting +from a cut at a reference portfolio, the MILP is solved, the true CVaR of its solution is +compared with the epigraph value ``t``, and a new cut is added at the solution until they agree. +Because the integer feasible set is finite and each cut removes the current infeasible point, +the loop terminates at the exact optimum (within the solver's MIP gap); measured: 20-70 cuts, +identical to Gurobi's direct MISOCP solution to the cent. No scenarios, no sampling error, fully +deterministic. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from dataclasses import dataclass, field, replace + +from quantcore.optimization.problem import MeanCVaRProblem, MILPModel, ProblemSpec +from quantcore.optimization.rowgen import reference_exposure +from quantcore.optimization.solvers import SolverResult + +log = logging.getLogger(__name__) + +SolveFn = Callable[[MILPModel], SolverResult] + + +@dataclass +class OuterApproximationInfo: + iterations: int = 0 + cuts: int = 0 + gaps: list[float] = field(default_factory=list) # true CVaR - epigraph value per iteration + solve_times: list[float] = field(default_factory=list) + total_time: float = 0.0 + converged: bool = False + + def as_dict(self) -> dict: + return {"iterations": self.iterations, "cuts": self.cuts, "gaps": [round(g, 4) for g in self.gaps], + "solve_times_s": [round(t, 3) for t in self.solve_times], "total_time_s": round(self.total_time, 3), + "converged": self.converged} + + +def solve_gaussian_exact(spec: ProblemSpec, solve: SolveFn, tol: float = 1e-6, max_iter: int = 200 + ) -> tuple[SolverResult, MeanCVaRProblem, OuterApproximationInfo]: + """Outer-approximation loop for a Gaussian-mode :class:`ProblemSpec`. + + ``tol`` is relative to the CVaR scale (the limit, or the CVaR itself). Returns the last + solver result (its ``x`` decodes with the returned problem), the problem with all cuts, and + the iteration log. + """ + if not spec.is_gaussian: + raise ValueError("solve_gaussian_exact needs gaussian_mean and gaussian_cov") + t0 = time.monotonic() + info = OuterApproximationInfo() + problem = MeanCVaRProblem(spec) + problem.add_cut(reference_exposure(spec)) + result: SolverResult | None = None + for it in range(1, max_iter + 1): + t1 = time.monotonic() + result = solve(problem.build()) + info.solve_times.append(time.monotonic() - t1) + info.iterations = it + info.cuts = len(problem.cuts) + if not result.ok: + log.warning("outer approximation: solver returned %s at iteration %d", result.status, it) + break + d = problem.decode(result.x) + gap = d["cvar"] - d["cvar_model"] # true CVaR of the solution minus the epigraph value + info.gaps.append(float(gap)) + scale = max(abs(spec.cvar_limit or 0.0), abs(d["cvar"]), spec.wealth * 1e-4) + if gap <= tol * scale: + info.converged = True + result = replace(result, message=(result.message + f" (outer approximation: {it} iterations, " + f"{len(problem.cuts)} cuts)").strip()) + break + if d["sigma"] <= 0: # riskless solution whose cut cannot be formed: the model is exact there + info.converged = True + break + problem.add_cut(d["value"]) + else: + log.warning("outer approximation did not converge in %d iterations", max_iter) + result = replace(result, status="FeasibleFound", message=(result.message + " (outer approximation: iteration limit)").strip()) + info.total_time = time.monotonic() - t0 + return result, problem, info diff --git a/quantcore/optimization/problem.py b/quantcore/optimization/problem.py new file mode 100644 index 0000000..4edb049 --- /dev/null +++ b/quantcore/optimization/problem.py @@ -0,0 +1,544 @@ +"""Mean-CVaR optimisation as a mixed-integer linear programme. + +Decision variables (all share numbers are integers -- a stock can only be bought whole): + +* ``n_i`` final number of shares of stock ``i`` (integer, >= 0: no short positions) +* ``b_i``, ``s_i`` shares bought / sold (integers, rebalancing mode only): ``n_i = n0_i + b_i - s_i`` +* ``alpha`` the VaR level (free) and ``u_s >= 0`` the scenario excess losses of the + Rockafellar-Uryasev linearisation of CVaR (scenario mode), or the epigraph variable ``t`` + of the CVaR (Gaussian mode, see below) +* ``y_i`` binary "position open" indicators (only with a cardinality / minimum-position rule) + +**Scenario mode.** With ``S`` scenarios of horizon returns ``R_{s,i}`` (simple returns in the +base currency) with probabilities ``p_s`` (``1/S`` unless importance sampling was used), current +prices ``P_i`` and transaction costs ``TC``, the loss of scenario ``s`` is +``L_s = TC - sum_i P_i R_{s,i} n_i`` and + + CVaR_beta = min_alpha alpha + 1/(1-beta) sum_s p_s max(L_s - alpha, 0) + +which is linear once ``u_s >= L_s - alpha`` is added. The expected P&L in the objective uses the +*model* expected returns ``mu_i`` (``sum_i P_i mu_i n_i - TC``) when the spec provides them, +otherwise the probability-weighted scenario mean -- the scenarios then only shape the tail. + +**Gaussian mode** (``gaussian_mean``/``gaussian_cov`` given, no scenarios): horizon returns are +``R ~ N(m, C)``, the P&L is normal and ``CVaR_beta = TC - m'v + k_beta sqrt(v'Cv)`` with +``v = P * n`` and ``k_beta = phi(z_beta)/(1-beta)``. The convex function is represented by an +epigraph variable ``t`` and tangent cuts at exposures ``v_k`` (``t >= TC + (k_beta C v_k / sigma_k +- m)' v``), added by an outer-approximation loop (:mod:`quantcore.optimization.gaussian_exact`). +No scenarios, no sampling error: this is the mean-variance-equivalent exact solution. + +Objectives: + +* ``max_return``: maximise expected P&L subject to ``CVaR_beta <= cvar_limit`` (money); +* ``min_cvar``: minimise ``CVaR_beta`` subject to expected P&L ``>= target_pnl``; +* ``utility``: maximise ``expected P&L - risk_aversion * CVaR_beta``. + +Constraints: budget (cash cannot go negative), maximum weight per stock and per sector +(fractions of total wealth), optional transaction-cost and turnover limits, optional +cardinality / minimum position size. Weight limits are tight *variable bounds* +(``n_i <= floor(w_max W / P_i)``), which helps the MILP solver considerably; ``alpha`` and the +``u_s`` are bounded by the worst gain and loss attainable within those caps (see +``_loss_bounds``) -- free variables make the CVaR objective numerically hard. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import scipy.sparse as sp +from scipy.stats import norm + +from quantcore.measurement.risk import scenario_cvar + + +@dataclass +class OptimizationConstraints: + max_weight: float = 0.10 # per stock, fraction of total wealth + max_sector_weight: float | dict[str, float] = 0.25 # per GICS sector (scalar = same for all) + max_positions: int | None = None # cardinality limit (adds binaries) + min_position_value: float = 0.0 # minimum value of an open position (adds binaries) + max_turnover: float | None = None # fraction of wealth traded (buys + sells) + max_transaction_cost: float | None = None # absolute money amount + min_invested: float = 0.0 # minimum fraction of wealth invested in stocks + max_invested: float = 1.0 # maximum fraction invested (1.0 = fully, remainder is cash) + + def sector_limit(self, sector: str) -> float | None: + if isinstance(self.max_sector_weight, dict): + return self.max_sector_weight.get(sector, self.max_sector_weight.get("*")) + return self.max_sector_weight + + def uses_binaries(self) -> bool: + return self.max_positions is not None or self.min_position_value > 0 + + +@dataclass +class TransactionCosts: + proportional_buy: float = 0.001 # 10 bp of the traded value + proportional_sell: float = 0.001 + fixed_per_trade: float = 0.0 # money per stock traded (adds trade binaries) + + def uses_binaries(self) -> bool: + return self.fixed_per_trade > 0 + + +def cvar_multiplier(beta: float) -> float: + """``k_beta = phi(z_beta) / (1 - beta)``: Gaussian CVaR in standard deviations.""" + return float(norm.pdf(norm.ppf(beta)) / (1.0 - beta)) + + +@dataclass +class ProblemSpec: + """Everything the builder needs, as plain arrays (built by the service, tested directly).""" + + symbols: list[str] + prices: np.ndarray # current price per share, base currency + scenarios: np.ndarray | None # S x N simple returns over the horizon (None in Gaussian mode) + sectors: list[str] + cash: float # initial cash + holdings: np.ndarray | None = None # initial shares (None or zeros: new portfolio) + confidence: float = 0.95 + cvar_limit: float | None = None # money + objective: str = "max_return" # 'max_return' | 'min_cvar' | 'utility' + risk_aversion: float = 1.0 # for 'utility' + target_pnl: float | None = None # for 'min_cvar' (money over the horizon) + constraints: OptimizationConstraints = field(default_factory=OptimizationConstraints) + costs: TransactionCosts = field(default_factory=TransactionCosts) + probabilities: np.ndarray | None = None # scenario probabilities (None = 1/S); may sum to < 1 for a subset + expected_returns: np.ndarray | None = None # model expected simple return per asset over the horizon + gaussian_mean: np.ndarray | None = None # Gaussian mode: mean of the horizon simple returns + gaussian_cov: np.ndarray | None = None # Gaussian mode: covariance of the horizon simple returns + + def __post_init__(self) -> None: + self.prices = np.asarray(self.prices, dtype=float) + n = len(self.symbols) + if self.prices.shape != (n,): + raise ValueError("prices must have one entry per symbol") + if len(self.sectors) != n: + raise ValueError("sectors must have one entry per symbol") + if self.holdings is None: + self.holdings = np.zeros(n) + self.holdings = np.asarray(self.holdings, dtype=float) + if np.any(self.holdings < 0): + raise ValueError("initial holdings cannot be negative") + if not 0 < self.confidence < 1: + raise ValueError("confidence must be in (0, 1)") + if self.objective == "max_return" and self.cvar_limit is None: + raise ValueError("objective 'max_return' needs a cvar_limit") + if self.objective == "min_cvar" and self.target_pnl is None: + raise ValueError("objective 'min_cvar' needs a target_pnl") + if self.objective not in ("max_return", "min_cvar", "utility"): + raise ValueError("objective must be 'max_return', 'min_cvar' or 'utility'") + if self.is_gaussian: + self.gaussian_mean = np.asarray(self.gaussian_mean, dtype=float) + self.gaussian_cov = np.asarray(self.gaussian_cov, dtype=float) + if self.gaussian_mean.shape != (n,) or self.gaussian_cov.shape != (n, n): + raise ValueError("gaussian_mean must be N and gaussian_cov N x N") + self.scenarios = np.zeros((0, n)) + if self.expected_returns is None: + self.expected_returns = self.gaussian_mean + else: + if self.scenarios is None: + raise ValueError("scenarios are required unless gaussian_mean/gaussian_cov are given") + self.scenarios = np.asarray(self.scenarios, dtype=float) + if self.scenarios.ndim != 2 or self.scenarios.shape[1] != n or self.scenarios.shape[0] == 0: + raise ValueError("scenarios must be S x N with S >= 1") + if self.probabilities is not None: + self.probabilities = np.asarray(self.probabilities, dtype=float) + if self.probabilities.shape != (self.scenarios.shape[0],): + raise ValueError("probabilities must have one entry per scenario") + if np.any(self.probabilities < 0) or self.probabilities.sum() > 1.0 + 1e-9: + raise ValueError("probabilities must be non-negative and sum to at most 1") + if self.expected_returns is not None: + self.expected_returns = np.asarray(self.expected_returns, dtype=float) + if self.expected_returns.shape != (n,): + raise ValueError("expected_returns must have one entry per symbol") + + @property + def is_gaussian(self) -> bool: + return self.gaussian_mean is not None and self.gaussian_cov is not None + + @property + def n_scenarios(self) -> int: + return int(self.scenarios.shape[0]) + + @property + def weights(self) -> np.ndarray: + """Scenario probabilities (``1/S`` when none were given).""" + if self.probabilities is None: + return np.full(self.n_scenarios, 1.0 / max(self.n_scenarios, 1)) + return self.probabilities + + @property + def mean_returns(self) -> np.ndarray: + """Expected simple return per asset used in the objective.""" + if self.expected_returns is not None: + return self.expected_returns + w = self.weights + return (w @ self.scenarios) / w.sum() + + @property + def rebalancing(self) -> bool: + return bool(np.any(self.holdings > 0)) + + @property + def wealth(self) -> float: + return float(self.cash + self.holdings @ self.prices) + + +@dataclass +class MILPModel: + """Solver-agnostic MILP: ``min/max c'x s.t. row_lb <= A x <= row_ub, var_lb <= x <= var_ub``.""" + + c: np.ndarray + A: sp.csr_matrix + row_lb: np.ndarray + row_ub: np.ndarray + var_lb: np.ndarray + var_ub: np.ndarray + var_types: list[str] # 'I' | 'C' (binaries are 'I' with bounds [0, 1]) + var_names: list[str] + row_names: list[str] + maximize: bool + offset: float = 0.0 + + @property + def n_vars(self) -> int: + return len(self.c) + + @property + def n_rows(self) -> int: + return self.A.shape[0] + + @property + def nnz(self) -> int: + return int(self.A.nnz) + + def describe(self) -> dict: + return {"n_vars": self.n_vars, "n_int": sum(t == "I" for t in self.var_types), "n_rows": self.n_rows, + "nnz": self.nnz, "maximize": self.maximize} + + +class MeanCVaRProblem: + """Builds the MILP for a :class:`ProblemSpec` and decodes solver vectors.""" + + def __init__(self, spec: ProblemSpec): + self.spec = spec + self.n = len(spec.symbols) + self.gaussian = spec.is_gaussian + self.S = 0 if self.gaussian else spec.n_scenarios + self.rebalancing = spec.rebalancing + self.use_pos_bin = spec.constraints.uses_binaries() + self.use_trade_bin = spec.costs.uses_binaries() + self.cuts: list[np.ndarray] = [] # Gaussian mode: exposures v_k of the tangent cuts + self._layout() + + # -- variable layout ----------------------------------------------------------------------- + def _layout(self) -> None: + n = self.n + self.i_n = np.arange(0, n) + pos = n + if self.rebalancing: + self.i_b = np.arange(pos, pos + n) + pos += n + self.i_s = np.arange(pos, pos + n) + pos += n + else: + self.i_b = self.i_s = None + if self.gaussian: + self.i_alpha = None + self.i_u = np.arange(0) + self.i_cv = pos # epigraph variable of the CVaR + pos += 1 + else: + self.i_alpha = pos + pos += 1 + self.i_u = np.arange(pos, pos + self.S) + pos += self.S + self.i_cv = None + if self.use_pos_bin: + self.i_y = np.arange(pos, pos + n) + pos += n + else: + self.i_y = None + if self.use_trade_bin: + self.i_t = np.arange(pos, pos + n) + pos += n + else: + self.i_t = None + self.n_vars = pos + + # -- helpers -------------------------------------------------------------------------------- + def _max_shares(self) -> np.ndarray: + """Tight upper bound per stock: ``floor(min(w_max, f_max) * W / P_i)`` (the weight limit, or the + invested fraction if that is smaller). A legacy holding above the cap has to be sold down.""" + w = self.spec.wealth + lim = min(self.spec.constraints.max_weight, self.spec.constraints.max_invested) * w + ub = np.floor(lim / self.spec.prices + 1e-9) + return np.maximum(ub, 0.0) + + def _loss_bounds(self, ub_shares: np.ndarray) -> tuple[float, float]: + """Valid finite bounds ``(max_gain, max_loss)`` over the box ``0 <= n_i <= ub_shares_i``. + + Any feasible portfolio's money P&L lies in ``[-max_loss, max_gain]``, so ``alpha`` and the + ``u_s`` can be bounded: free variables with infinite bounds make the CVaR objective + numerically hard for the solvers (cuOpt's dual simplex aborts on such models). + """ + p = self.spec.prices + cost_ub = float(self.spec.costs.proportional_buy + self.spec.costs.proportional_sell) * float( + (ub_shares * p).sum()) + float(self.spec.costs.fixed_per_trade) * self.n + if self.gaussian: + v_max = ub_shares * p + mean = np.abs(self.spec.gaussian_mean) @ v_max + sigma = float(np.sqrt(max(v_max @ self.spec.gaussian_cov @ v_max, 0.0))) + spread = mean + cvar_multiplier(self.spec.confidence) * sigma + cost_ub + return spread, spread + gain = self.spec.scenarios * p[None, :] # money gain per share, per scenario + max_gain = float(np.maximum(gain.max(axis=0), 0.0) @ ub_shares) + max_loss = float(np.maximum(-gain.min(axis=0), 0.0) @ ub_shares) + cost_ub + return max_gain, max_loss + + def _cost_terms(self): + """Coefficients of the transaction cost ``TC`` per variable (b, s, t or n).""" + p, c = self.spec.prices, self.spec.costs + terms = [] # (indices, coefficients) + if self.rebalancing: + terms.append((self.i_b, c.proportional_buy * p)) + terms.append((self.i_s, c.proportional_sell * p)) + else: + terms.append((self.i_n, c.proportional_buy * p)) + if self.use_trade_bin: + terms.append((self.i_t, np.full(self.n, c.fixed_per_trade))) + return terms + + def _cvar_expr(self) -> tuple[np.ndarray, np.ndarray]: + """Indices and coefficients of the linear expression equal to ``CVaR_beta``.""" + if self.gaussian: + return np.array([self.i_cv]), np.array([1.0]) + tail = self.spec.weights / (1.0 - self.spec.confidence) + return np.concatenate([[self.i_alpha], self.i_u]), np.concatenate([[1.0], tail]) + + # -- Gaussian mode ----------------------------------------------------------------------- + def gaussian_risk(self, exposure: np.ndarray, total_cost: float = 0.0) -> tuple[float, float, float]: + """``(VaR, CVaR, sigma)`` of the money P&L for exposure ``v`` under the Gaussian model.""" + v = np.asarray(exposure, dtype=float) + sigma = float(np.sqrt(max(v @ self.spec.gaussian_cov @ v, 0.0))) + mean = float(self.spec.gaussian_mean @ v) - total_cost + z = float(norm.ppf(self.spec.confidence)) + return -mean + z * sigma, -mean + cvar_multiplier(self.spec.confidence) * sigma, sigma + + def add_cut(self, exposure: np.ndarray) -> None: + """Add a tangent cut of the Gaussian CVaR at money exposure ``v_k`` (``v_k`` must carry risk).""" + if not self.gaussian: + raise ValueError("cuts only exist in Gaussian mode") + v = np.asarray(exposure, dtype=float) + if v @ self.spec.gaussian_cov @ v <= 0: + raise ValueError("cut exposure has no risk") + self.cuts.append(v) + + # -- build ------------------------------------------------------------------------------------ + def build(self) -> MILPModel: + sp_ = self.spec + n, S = self.n, self.S + p = sp_.prices + W = sp_.wealth + mean_gain = sp_.mean_returns * p # expected money gain per share + cost_terms = self._cost_terms() + ub_shares = self._max_shares() + + rows, cols, vals = [], [], [] + row_lb, row_ub, row_names = [], [], [] + r = 0 + + def add_row(idx, coef, lb, ub, name): + nonlocal r + idx = np.asarray(idx) + coef = np.asarray(coef, dtype=float) + keep = coef != 0 + rows.extend([r] * int(keep.sum())) + cols.extend(idx[keep].tolist()) + vals.extend(coef[keep].tolist()) + row_lb.append(lb) + row_ub.append(ub) + row_names.append(name) + r += 1 + + cost_idx = np.concatenate([i for i, _ in cost_terms]) + cost_coef = np.concatenate([c for _, c in cost_terms]) + + if self.gaussian: + # 1g. tangent cuts: TC + (k C v_k / sigma_k - m)' (P n) - t <= 0 + if not self.cuts: + raise ValueError("Gaussian mode needs at least one cut (use gaussian_exact.solve)") + k = cvar_multiplier(sp_.confidence) + for j, v_k in enumerate(self.cuts): + sigma_k = np.sqrt(v_k @ sp_.gaussian_cov @ v_k) + grad = k * (sp_.gaussian_cov @ v_k) / sigma_k - sp_.gaussian_mean + idx = np.concatenate([self.i_n, cost_idx, [self.i_cv]]) + coef = np.concatenate([grad * p, cost_coef, [-1.0]]) + add_row(idx, coef, -np.inf, 0.0, f"cut_{j}") + else: + # 1. CVaR scenario rows: sum_i gain_si n_i - TC + alpha + u_s >= 0 + gain = sp_.scenarios * p[None, :] # money gain per share in each scenario + for s in range(S): + idx = np.concatenate([self.i_n, cost_idx, [self.i_alpha, self.i_u[s]]]) + coef = np.concatenate([gain[s], -cost_coef, [1.0, 1.0]]) + add_row(idx, coef, 0.0, np.inf, f"cvar_scen_{s}") + + # 2. CVaR expression rows (limit for max_return, objective otherwise) + cvar_idx, cvar_coef = self._cvar_expr() + if sp_.objective == "max_return" or sp_.cvar_limit is not None: + add_row(cvar_idx, cvar_coef, -np.inf, float(sp_.cvar_limit), "cvar_limit") + + # 3. budget: cash_end = cash0 + sum p s - sum p b - TC >= 0 (duplicate indices are summed) + if self.rebalancing: + add_row(np.concatenate([self.i_b, self.i_s, cost_idx]), np.concatenate([p, -p, cost_coef]), -np.inf, + sp_.cash, "budget") + # 4. linking n - b + s = n0 + for i in range(n): + add_row([self.i_n[i], self.i_b[i], self.i_s[i]], [1.0, -1.0, 1.0], sp_.holdings[i], sp_.holdings[i], + f"link_{i}") + else: + add_row(np.concatenate([self.i_n, cost_idx]), np.concatenate([p, cost_coef]), -np.inf, sp_.cash, "budget") + + # 5. invested fraction bounds (min / max) + cons = sp_.constraints + lo = cons.min_invested * W if cons.min_invested > 0 else -np.inf + hi = cons.max_invested * W if cons.max_invested < 1.0 else np.inf + if np.isfinite(lo) or np.isfinite(hi): + add_row(self.i_n, p, lo, hi, "invested") + + # 6. sector limits + sectors = np.asarray(sp_.sectors) + for sec in sorted(set(sectors)): + lim = cons.sector_limit(sec) + if lim is None or lim >= cons.max_invested: + continue + members = np.where(sectors == sec)[0] + add_row(self.i_n[members], p[members], -np.inf, lim * W, f"sector_{sec}") + + # 7. transaction cost limit and 8. turnover + if cons.max_transaction_cost is not None: + add_row(cost_idx, cost_coef, -np.inf, cons.max_transaction_cost, "max_cost") + if cons.max_turnover is not None and self.rebalancing: + add_row(np.concatenate([self.i_b, self.i_s]), np.concatenate([p, p]), -np.inf, cons.max_turnover * W, + "turnover") + + # 9. cardinality / minimum position size via binaries y_i + if self.use_pos_bin: + for i in range(n): + add_row([self.i_n[i], self.i_y[i]], [1.0, -max(ub_shares[i], 1.0)], -np.inf, 0.0, f"open_{i}") + if cons.min_position_value > 0: + add_row([self.i_n[i], self.i_y[i]], [p[i], -cons.min_position_value], 0.0, np.inf, f"minpos_{i}") + if cons.max_positions is not None: + add_row(self.i_y, np.ones(n), -np.inf, float(cons.max_positions), "cardinality") + + # 10. trade indicators for fixed costs: (b_i + s_i) <= M t_i (new mode: n_i <= M t_i) + if self.use_trade_bin: + big_m = np.maximum(ub_shares, sp_.holdings) + 1.0 + for i in range(n): + if self.rebalancing: + add_row([self.i_b[i], self.i_s[i], self.i_t[i]], [1.0, 1.0, -big_m[i]], -np.inf, 0.0, f"trade_{i}") + else: + add_row([self.i_n[i], self.i_t[i]], [1.0, -big_m[i]], -np.inf, 0.0, f"trade_{i}") + + # 11. target expected P&L for min_cvar + pnl_idx = np.concatenate([self.i_n, cost_idx]) + pnl_coef = np.concatenate([mean_gain, -cost_coef]) + if sp_.objective == "min_cvar": + add_row(pnl_idx, pnl_coef, float(sp_.target_pnl), np.inf, "target_pnl") + + # objective + c = np.zeros(self.n_vars) + if sp_.objective == "max_return": + np.add.at(c, pnl_idx, pnl_coef) + maximize = True + elif sp_.objective == "min_cvar": + np.add.at(c, cvar_idx, cvar_coef) + maximize = False + else: # utility + np.add.at(c, pnl_idx, pnl_coef) + np.add.at(c, cvar_idx, -sp_.risk_aversion * cvar_coef) + maximize = True + + # variable bounds and types + var_lb = np.zeros(self.n_vars) + var_ub = np.full(self.n_vars, np.inf) + types = ["C"] * self.n_vars + names = [""] * self.n_vars + for i in range(n): + types[self.i_n[i]] = "I" + names[self.i_n[i]] = f"n[{sp_.symbols[i]}]" + var_ub[self.i_n[i]] = ub_shares[i] + if self.rebalancing: + for i in range(n): + types[self.i_b[i]] = types[self.i_s[i]] = "I" + names[self.i_b[i]] = f"buy[{sp_.symbols[i]}]" + names[self.i_s[i]] = f"sell[{sp_.symbols[i]}]" + var_ub[self.i_b[i]] = ub_shares[i] + var_ub[self.i_s[i]] = sp_.holdings[i] + max_gain, max_loss = self._loss_bounds(ub_shares) + if self.gaussian: + var_lb[self.i_cv], var_ub[self.i_cv] = -max_gain, max_loss + names[self.i_cv] = "cvar" + else: + var_lb[self.i_alpha], var_ub[self.i_alpha] = -max_gain, max_loss + names[self.i_alpha] = "alpha" + for s in range(S): + var_ub[self.i_u[s]] = max_loss + max_gain # u_s <= L_s - alpha over the box + names[self.i_u[s]] = f"u[{s}]" + if self.use_pos_bin: + for i in range(n): + types[self.i_y[i]] = "I" + var_ub[self.i_y[i]] = 1.0 + names[self.i_y[i]] = f"y[{sp_.symbols[i]}]" + if self.use_trade_bin: + for i in range(n): + types[self.i_t[i]] = "I" + var_ub[self.i_t[i]] = 1.0 + names[self.i_t[i]] = f"t[{sp_.symbols[i]}]" + + A = sp.csr_matrix((vals, (rows, cols)), shape=(r, self.n_vars)) + A.sum_duplicates() + A.sort_indices() + return MILPModel(c=c, A=A, row_lb=np.array(row_lb, dtype=float), row_ub=np.array(row_ub, dtype=float), + var_lb=var_lb, var_ub=var_ub, var_types=types, var_names=names, row_names=row_names, + maximize=maximize) + + # -- decode ----------------------------------------------------------------------------------- + def decode(self, x: np.ndarray) -> dict: + """Turn a solution vector into shares, trades, cash, costs and the realised CVaR.""" + sp_ = self.spec + p = sp_.prices + shares = np.rint(x[self.i_n]).astype(int) + if self.rebalancing: + buys = np.rint(x[self.i_b]).astype(int) + sells = np.rint(x[self.i_s]).astype(int) + else: + buys, sells = shares.copy(), np.zeros(self.n, dtype=int) + trade_value_buy = buys * p + trade_value_sell = sells * p + cost = (sp_.costs.proportional_buy * trade_value_buy + sp_.costs.proportional_sell * trade_value_sell) + if self.use_trade_bin: + cost = cost + sp_.costs.fixed_per_trade * ((buys + sells) > 0) + total_cost = float(cost.sum()) + cash_end = float(sp_.cash + trade_value_sell.sum() - trade_value_buy.sum() - total_cost) + value = shares * p + expected_pnl = float(sp_.mean_returns @ value - total_cost) # the objective's expectation + out = { + "shares": shares, "buys": buys, "sells": sells, "trades": buys - sells, "value": value, + "cost": cost, "total_cost": total_cost, "cash_end": cash_end, "invested": float(value.sum()), + "expected_pnl": expected_pnl, "weights": value / sp_.wealth, + "turnover": float((trade_value_buy.sum() + trade_value_sell.sum()) / sp_.wealth), + } + if self.gaussian: + var_m, cvar_m, sigma = self.gaussian_risk(value, total_cost) + out.update({"var": var_m, "cvar": cvar_m, "sigma": sigma, "alpha": var_m, + "cvar_model": float(x[self.i_cv]), "pnl_scenarios": None}) + else: + pnl = sp_.scenarios @ value - total_cost # money P&L per scenario + var_m, cvar_m = scenario_cvar(-pnl, sp_.confidence, sp_.probabilities) + w = sp_.weights + out.update({"var": var_m, "cvar": cvar_m, "alpha": float(x[self.i_alpha]), "pnl_scenarios": pnl, + "expected_pnl_scenarios": float((w @ pnl) / w.sum())}) + return out diff --git a/quantcore/optimization/rowgen.py b/quantcore/optimization/rowgen.py new file mode 100644 index 0000000..db0348c --- /dev/null +++ b/quantcore/optimization/rowgen.py @@ -0,0 +1,151 @@ +"""Tail-row generation for large scenario sets (Kuenzi-Bay & Mayer, 2006). + +In the Rockafellar-Uryasev MILP only the scenarios in the tail have ``u_s > 0`` at the optimum; +every other scenario row is slack. So the problem is solved on a *working set* of scenarios +(initially the worst ``initial_multiple * (1 - beta) * S`` scenarios of a reference portfolio), +the losses of all ``S`` scenarios are evaluated for the solution, the scenarios whose loss exceeds +the solution's ``alpha`` are added, and the loop repeats. The restricted problem is a relaxation +(dropped rows, ``u_s = 0``), so its objective bounds the full one; when no omitted scenario is +violated the solution is optimal for the full scenario set (within the MIP gap). + +Measured on the real universe (30 stocks, beta = 0.95): working sets of ~11 % of ``S``, two to +three iterations, 6-7x faster than the full MILP at 32 768-100 000 scenarios -- and only the +working-set rows are sent to the solver. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from dataclasses import dataclass, field, replace + +import numpy as np + +from quantcore.optimization.problem import MeanCVaRProblem, MILPModel, ProblemSpec +from quantcore.optimization.solvers import SolverResult + +log = logging.getLogger(__name__) + +SolveFn = Callable[[MILPModel], SolverResult] + + +@dataclass +class RowGenerationInfo: + iterations: int = 0 + working_set_sizes: list[int] = field(default_factory=list) + violated: list[int] = field(default_factory=list) + solve_times: list[float] = field(default_factory=list) + total_time: float = 0.0 + converged: bool = False + + def as_dict(self) -> dict: + return {"iterations": self.iterations, "working_set_sizes": self.working_set_sizes, "violated": self.violated, + "rows_final": self.working_set_sizes[-1] if self.working_set_sizes else 0, + "solve_times_s": [round(t, 3) for t in self.solve_times], "total_time_s": round(self.total_time, 3), + "converged": self.converged} + + +def reference_exposure(spec: ProblemSpec) -> np.ndarray: + """Money exposure of the reference portfolio: the current holdings, or (new portfolio) an + equal-weight allocation within the per-stock cap.""" + if spec.rebalancing: + return spec.holdings * spec.prices + n = len(spec.symbols) + w = min(spec.constraints.max_weight, spec.constraints.max_invested, 1.0 / n) + return np.full(n, w * spec.wealth) + + +def scenario_losses(spec: ProblemSpec, exposure: np.ndarray, total_cost: float = 0.0) -> np.ndarray: + """Money loss of every scenario for a given exposure (``TC - sum_i R_si v_i``).""" + return total_cost - spec.scenarios @ np.asarray(exposure, dtype=float) + + +def restricted_spec(spec: ProblemSpec, work: np.ndarray) -> ProblemSpec: + """The spec on a subset of scenarios: probabilities keep their *full-set* values (the omitted + scenarios contribute ``u_s = 0``), the objective keeps the full-set expected returns.""" + return replace(spec, scenarios=spec.scenarios[work], probabilities=spec.weights[work], + expected_returns=spec.mean_returns) + + +def expand_solution(problem_r: MeanCVaRProblem, x_r: np.ndarray, problem_full: MeanCVaRProblem) -> np.ndarray: + """Map a restricted solution to the full layout: ``u_s = max(L_s - alpha, 0)`` for every scenario.""" + x = np.zeros(problem_full.n_vars) + x[problem_full.i_n] = x_r[problem_r.i_n] + if problem_full.rebalancing: + x[problem_full.i_b] = x_r[problem_r.i_b] + x[problem_full.i_s] = x_r[problem_r.i_s] + alpha = float(x_r[problem_r.i_alpha]) + x[problem_full.i_alpha] = alpha + d = problem_r.decode(x_r) + losses = scenario_losses(problem_full.spec, d["value"], d["total_cost"]) + x[problem_full.i_u] = np.maximum(losses - alpha, 0.0) + if problem_full.use_pos_bin: + x[problem_full.i_y] = x_r[problem_r.i_y] + if problem_full.use_trade_bin: + x[problem_full.i_t] = x_r[problem_r.i_t] + return x + + +def solve_by_row_generation(spec: ProblemSpec, solve: SolveFn, initial_multiple: float = 2.0, max_iter: int = 30, + tol: float = 1e-7, deadline: float | None = None + ) -> tuple[SolverResult, MeanCVaRProblem, RowGenerationInfo]: + """Solve the scenario MILP of ``spec`` by tail-row generation. + + ``solve`` maps a :class:`MILPModel` to a :class:`SolverResult`. Returns the result in the + *full* problem's layout (so :meth:`MeanCVaRProblem.decode` of the returned problem works), the + full problem and the iteration log. + + ``deadline`` is a :func:`time.monotonic` timestamp: the *request's* time budget. Each pass is a + MILP of its own, so without it a limit of ``T`` seconds would allow ``T`` per pass instead of + ``T`` in total. Once it passes, the current (feasible, not proven optimal) solution is returned. + """ + if spec.is_gaussian: + raise ValueError("row generation applies to scenario problems") + t0 = time.monotonic() + info = RowGenerationInfo() + S = spec.n_scenarios + full = MeanCVaRProblem(spec) + losses0 = scenario_losses(spec, reference_exposure(spec)) + k0 = int(min(S, max(np.ceil(initial_multiple * (1.0 - spec.confidence) * S), 2))) + work = np.sort(np.argsort(losses0)[-k0:]) + scale = max(abs(spec.cvar_limit or 0.0), spec.wealth * 1e-3, 1.0) + result: SolverResult | None = None + for it in range(1, max_iter + 1): + problem_r = MeanCVaRProblem(restricted_spec(spec, work)) + t1 = time.monotonic() + result = solve(problem_r.build()) + info.solve_times.append(time.monotonic() - t1) + info.working_set_sizes.append(len(work)) + info.iterations = it + if not result.ok: + log.warning("row generation: solver returned %s at iteration %d", result.status, it) + info.total_time = time.monotonic() - t0 + return result, full, info + d = problem_r.decode(result.x) + alpha = float(result.x[problem_r.i_alpha]) + losses = scenario_losses(spec, d["value"], d["total_cost"]) + violated = np.setdiff1d(np.where(losses > alpha + tol * scale)[0], work) + info.violated.append(len(violated)) + log.info("row generation: iteration %d, working set %d of %d, %d violated", it, len(work), S, len(violated)) + if len(violated) > 0 and deadline is not None and time.monotonic() >= deadline: + log.warning("row generation: time budget spent after %d iterations (%d rows still violated)", + it, len(violated)) + x_full = expand_solution(problem_r, result.x, full) + info.total_time = time.monotonic() - t0 + return replace(result, x=x_full, status="FeasibleFound", + message=(result.message + f" (row generation: time limit after {it} iterations, " + f"{len(work)} of {S} rows)").strip()), full, info + if len(violated) == 0: + info.converged = True + x_full = expand_solution(problem_r, result.x, full) + result = replace(result, x=x_full, message=(result.message + f" (row generation: {it} iterations, " + f"{len(work)} of {S} rows)").strip()) + info.total_time = time.monotonic() - t0 + return result, full, info + work = np.union1d(work, violated) + log.warning("row generation did not converge in %d iterations", max_iter) + x_full = expand_solution(problem_r, result.x, full) + info.total_time = time.monotonic() - t0 + return replace(result, x=x_full, status="FeasibleFound", + message=(result.message + " (row generation: iteration limit)").strip()), full, info diff --git a/quantcore/optimization/service.py b/quantcore/optimization/service.py new file mode 100644 index 0000000..35395a1 --- /dev/null +++ b/quantcore/optimization/service.py @@ -0,0 +1,763 @@ +"""``OptimizationService`` -- from a request to an integer-share portfolio proposal. + +Pipeline (deterministic given the seed; the result carries the scenario hash and a +``reproducible`` flag): + +1. candidate stocks (a list or the largest ``max_assets`` of the universe by market cap); +2. expected returns: Black-Litterman on 10 years of weekly CHF returns with market-cap + weights, the broad-market average as the equilibrium return, and Idzorek-confidence views + (or plain historical means / zero drift) -- *arithmetic* annual expected returns; +3. horizon scenarios from the chosen stock-price model (GBM or empirical innovations) with + the chosen covariance estimator: scrambled Sobol' draws, moment matching (exact mean and + covariance), optionally an importance-sampling tilt along the loss direction of a first-pass + solution (``importance_sampling``), or -- ``scenario_model='gaussian_exact'`` -- no scenarios + at all but the exact Gaussian (mean-variance-equivalent) problem by outer approximation; +4. the mean-CVaR MILP (:mod:`quantcore.optimization.problem`) in "new" or "rebalancing" mode, + for large scenario sets by tail-row generation (:mod:`quantcore.optimization.rowgen`); +5. solved on cuOpt (HiGHS fallback), decoded into shares, trades, cash, costs, risk figures. +""" + +from __future__ import annotations + +import json +import logging +import time +from collections import OrderedDict +from dataclasses import asdict, dataclass, field + +import numpy as np +import pandas as pd +from scipy.stats import norm + +from quantcore.config import Settings +from quantcore.data.market_data import MarketDataService +from quantcore.measurement.returns import horizon_periods, periods_per_year +from quantcore.measurement.risk import scenario_cvar +from quantcore.models.black_litterman import ( + BlackLittermanModel, + BlackLittermanResult, + View, + view_from_sentiment, +) +from quantcore.models.covariance import CovarianceModel +from quantcore.models.sampling import next_power_of_two, scenario_hash +from quantcore.models.simulation import ScenarioSet, TiltSpec, make_simulator +from quantcore.optimization.gaussian_exact import solve_gaussian_exact +from quantcore.optimization.problem import ( + MeanCVaRProblem, + OptimizationConstraints, + ProblemSpec, + TransactionCosts, + cvar_multiplier, +) +from quantcore.optimization.rowgen import solve_by_row_generation +from quantcore.optimization.solvers import HighsSolver, SolverResult, solve_with_fallback + +log = logging.getLogger(__name__) + +SCENARIO_MODELS = ("gbm", "empirical", "gaussian_exact") +IMPORTANCE_SAMPLING_MODES = ("auto", "market", "adaptive", "off") +ROW_GENERATION_MIN_SCENARIOS = 8192 # 'auto': row generation from this size on +IMPORTANCE_SAMPLING_MIN_TAIL = 128 # 'auto': tilt when fewer than this many scenarios would be in the tail +STAGE1_SCENARIOS = 2048 # 'adaptive': first-pass solve that provides the tilt direction +TILT_ALIGNMENT_WARN = 0.3 # warn below this correlation between the solution's loss and the tilt's +SCENARIO_CACHE_SIZE = 8 # scenario sets kept per service instance +# The sample-average approximation searches an N-dimensional space, so the scenario count has to +# grow with the number of candidates. Measured on the real universe (GBM model against the exact +# optimum): the *reported figures* are stable at any size (the standard deviation of the expected +# P&L is below 0.1 % from 2 048 scenarios on), but the *composition* keeps moving between +# near-equivalent portfolios -- at 2 048 scenarios two seeds differ by 3.5 % of wealth for 30 +# candidates and 12.5 % for 100. About 64 scenarios per candidate brings that near 5 %, which is +# too expensive as a default (N = 300 would mean 32 768 scenarios and minutes per solve), so the +# default is 16 per candidate and the result says when the ratio is low. +SCENARIOS_PER_ASSET = 16 +MIN_DEFAULT_SCENARIOS = 4096 +MAX_DEFAULT_SCENARIOS = 16384 # beyond this prefer 'gaussian_exact' (no scenarios) or set it explicitly +SCENARIOS_PER_ASSET_WARN = 8 # warn below this ratio + + +def _view_fingerprint(views) -> list: + """Views as plain sorted data (for the adaptive-tilt part of the scenario key).""" + out = [] + for v in views: + d = v if isinstance(v, dict) else {"assets": dict(v.assets), "q": v.q, "confidence": v.confidence} + out.append({k: d[k] for k in sorted(d) if k not in ("label", "source")}) + return out + + +@dataclass +class OptimizationRequest: + mode: str = "new" # 'new' (all from cash) | 'rebalance' (existing holdings + cash) + cash: float = 1_000_000.0 + holdings: dict[str, int] = field(default_factory=dict) + symbols: list[str] | None = None # candidate stocks; default: largest `max_assets` by market cap + max_assets: int = 100 + horizon: str | int = "week" + freq: str = "daily" + confidence: float = 0.95 + cvar_limit: float | None = None # money over the horizon + cvar_limit_fraction: float | None = None # alternative: fraction of total wealth + objective: str = "max_return" # 'max_return' | 'min_cvar' | 'utility' + risk_aversion: float = 1.0 + target_return: float | None = None # fraction of wealth over the horizon (objective 'min_cvar') + scenario_model: str = "empirical" # 'gbm' | 'empirical' | 'gaussian_exact' + n_scenarios: int | None = None # None: 16 per candidate, within [4096, 16384] (see scenario_count) + seed: int = 42 + sampling: str = "sobol" # 'sobol' | 'mc' | 'antithetic' | 'lhs' (gbm); 'sobol' | 'mc' | 'balanced' (empirical) + moment_match: bool = True # exact scenario mean and covariance + # 'market': tilt along the loss direction of the cap-weighted candidate portfolio, in blocks at + # z_0.95 and z_0.99 -- independent of the risk limit, the constraints and the confidence, so a + # family of runs shares one scenario set; 'adaptive': tilt towards the tail of a first-pass + # solution (slightly sharper, but the set then depends on those parameters); 'auto': 'market' + # when the tail would otherwise be thin; 'off': no tilt. + importance_sampling: str = "auto" + is_mixture: float = 0.25 # share of scenarios per tilted block ('adaptive': of the single block) + drift_convention: str = "arithmetic" # 'arithmetic' (Ito-corrected log-drift) | 'log' + # 'model': the CVaR is measured on the scenarios as they are, drift included -- over a long + # horizon a large expected return then shifts the whole tail into profit and the risk limit + # stops binding. 'zero': the risk rows use the centred (driftless) scenarios while the + # objective keeps the expected return, which is the usual market-risk convention. + risk_drift: str = "model" # 'model' | 'zero' + row_generation: str = "auto" # 'auto' | 'on' | 'off': solve on the tail rows only + covariance: dict = field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) + expected_returns: str = "black_litterman" # 'black_litterman' | 'historical' | 'zero' + views: list[dict] = field(default_factory=list) + bl_tau: float = 0.05 + bl_years: int = 10 + bl_omega_method: str = "idzorek" # 'idzorek' | 'he_litterman' + constraints: OptimizationConstraints = field(default_factory=OptimizationConstraints) + costs: TransactionCosts = field(default_factory=TransactionCosts) + solver: str = "auto" # 'auto' | 'cuopt' | 'highs' + time_limit: float | None = None + innovation_history: int | None = 2500 # observations used for the empirical innovations + innovation_standardize: str | None = "diagonal" # 'diagonal' | 'full' | None (see simulation module) + + def __post_init__(self) -> None: + if self.mode not in ("new", "rebalance"): + raise ValueError("mode must be 'new' or 'rebalance'") + if self.mode == "new" and self.holdings: + raise ValueError("mode 'new' starts from cash only - pass no holdings") + if self.scenario_model not in SCENARIO_MODELS: + raise ValueError(f"scenario_model must be one of {SCENARIO_MODELS}") + if self.importance_sampling not in IMPORTANCE_SAMPLING_MODES: + raise ValueError(f"importance_sampling must be one of {IMPORTANCE_SAMPLING_MODES}") + if self.row_generation not in ("auto", "on", "off"): + raise ValueError("row_generation must be 'auto', 'on' or 'off'") + if self.risk_drift not in ("model", "zero"): + raise ValueError("risk_drift must be 'model' or 'zero'") + if isinstance(self.constraints, dict): + self.constraints = OptimizationConstraints(**self.constraints) + if isinstance(self.costs, dict): + self.costs = TransactionCosts(**self.costs) + if self.cvar_limit is None and self.cvar_limit_fraction is None and self.objective == "max_return": + self.cvar_limit_fraction = 0.05 + log.info("no CVaR limit given - using 5%% of wealth over the horizon") + + @property + def horizon_periods(self) -> int: + return horizon_periods(self.horizon, self.freq) + + def scenario_count(self, n_assets: int) -> int: + """Scenarios to draw for ``n_assets`` candidates (the request's value, or the default rule).""" + if self.n_scenarios is not None: + return int(self.n_scenarios) + target = SCENARIOS_PER_ASSET * max(int(n_assets), 1) + return int(min(max(next_power_of_two(target), MIN_DEFAULT_SCENARIOS), MAX_DEFAULT_SCENARIOS)) + + def uses_row_generation(self, n_scenarios: int) -> bool: + if self.row_generation == "on": + return True + return self.row_generation == "auto" and n_scenarios >= ROW_GENERATION_MIN_SCENARIOS + + def tilt_mode(self, n_scenarios: int) -> str: + """``'market'``, ``'adaptive'`` or ``'off'`` for this request. + + ``'auto'`` is ``'market'``: the tilt blocks cover both usual confidence levels, it costs + nothing in accuracy at 95 % and roughly halves the tail noise at 99 % -- and, unlike a + rule that switches on the tail count, it keeps the scenario set independent of + ``confidence``, so runs at different risk levels stay comparable. + """ + if self.importance_sampling in ("market", "adaptive", "off"): + return self.importance_sampling + return "market" + + +@dataclass +class OptimizationResult: + status: str + solver: str + mode: str + as_of: str + wealth: float + cash_start: float + cash_end: float + invested: float + total_cost: float + turnover: float + expected_pnl: float + expected_return: float # fraction of wealth over the horizon + var: float # money + cvar: float # money + cvar_limit: float | None + confidence: float + horizon: str | int + horizon_periods: int + n_positions: int + n_candidates: int + positions: list[dict] + sector_exposure: dict + model: dict + solver_info: dict + timings: dict + reproducible: bool = False # optimality proven -> same inputs give the same figures + reproducibility: str = "none" # 'exact' (HiGHS: identical shares) | 'within_gap' (cuOpt: same value, ties may differ) + scenario_hash: str | None = None + scenario_key: dict | None = None # the inputs the scenario set depends on (see OptimizationService.scenario_key) + before: dict | None = None # rebalancing: the starting portfolio measured on the same scenarios + warnings: list[str] = field(default_factory=list) + + def as_dict(self) -> dict: + return asdict(self) + + def positions_frame(self) -> pd.DataFrame: + return pd.DataFrame(self.positions).set_index("symbol") if self.positions else pd.DataFrame() + + +class OptimizationService: + def __init__(self, settings: Settings, market_data: MarketDataService | None = None): + self.settings = settings + self.md = market_data or MarketDataService(settings) + self._scenario_cache: OrderedDict[str, ScenarioSet] = OrderedDict() + + # -- building blocks -------------------------------------------------------------------------- + def candidate_symbols(self, req: OptimizationRequest) -> list[str]: + if req.symbols: + symbols = list(dict.fromkeys(req.symbols)) + else: + caps = self.md.market_caps().sort_values(ascending=False) + symbols = caps.index[: req.max_assets].tolist() + for s in req.holdings: # existing holdings must be in the candidate set (they can be sold) + if s not in symbols: + symbols.append(s) + for v in req.views: # so must the assets a view refers to + assets = [v["symbol"]] if isinstance(v, dict) and "symbol" in v else ( + list(v["assets"]) if isinstance(v, dict) else list(v.assets)) + for s in assets: + if s not in symbols: + symbols.append(s) + unknown = [s for s in symbols if s not in self.md.universe.index] + if unknown: + raise KeyError(f"symbols not in the universe: {unknown}") + return symbols + + def expected_returns(self, req: OptimizationRequest, symbols: list[str]) -> tuple[pd.Series, dict, BlackLittermanResult | None]: + """Annual (arithmetic) expected returns per symbol and a description of the model.""" + last = self.md.last_date() + start = (last - pd.DateOffset(years=req.bl_years)).date().isoformat() + if req.expected_returns == "zero": + return pd.Series(0.0, index=symbols), {"type": "zero"}, None + weekly = self.md.returns(symbols, start=start, currency="base", freq="weekly").dropna() + if req.expected_returns == "historical": + mu = weekly.mean() * 52 + 0.5 * weekly.var() * 52 # arithmetic: log mean plus sigma^2/2 + return mu, {"type": "historical", "years": req.bl_years, "n_weeks": len(weekly)}, None + if req.expected_returns != "black_litterman": + raise ValueError("expected_returns must be 'black_litterman', 'historical' or 'zero'") + cov_w = weekly.cov() + market_w = self.md.market_weights(symbols) + index_r = self.md.cap_weighted_index(start=start, freq="weekly") + market_return = float(index_r.mean() + 0.5 * index_r.var()) # arithmetic weekly market return + bl = BlackLittermanModel(cov_w, market_w, market_return, risk_free=self.settings.risk_free_rate / 52, + tau=req.bl_tau, periods_per_year=52) + views = self._views(req, bl) + result = bl.posterior(views, omega_method=req.bl_omega_method) + desc = { + "type": "black_litterman", "years": req.bl_years, "n_weeks": len(weekly), + "market_return_ann": result.market_return_ann, "market_volatility_ann": result.market_volatility_ann, + "delta": result.delta, "tau": result.tau, "views": result.views, "convention": "arithmetic", + "omega_method": req.bl_omega_method, + } + return result.posterior_ann, desc, result + + @staticmethod + def _views(req: OptimizationRequest, bl: BlackLittermanModel) -> list[View]: + views = [] + for v in req.views: + if isinstance(v, View): + views.append(v) + elif "sentiment" in v: + views.append(view_from_sentiment(bl, v["symbol"], float(v["sentiment"]), float(v.get("confidence", 0.5)), + float(v.get("strength", 0.25)), v.get("label", ""), v.get("source", ""))) + else: + views.append(View(assets=v["assets"], q=float(v["q"]), confidence=float(v.get("confidence", 0.5)), + label=v.get("label", ""), source=v.get("source", ""))) + return views + + def _returns_and_cov_model(self, req: OptimizationRequest, symbols: list[str]) -> tuple[pd.DataFrame, CovarianceModel]: + r = self.md.returns(symbols, currency="base", freq=req.freq) + return r.dropna(), CovarianceModel.from_dict(req.covariance) # complete rows only for the estimators + + def market_exposure(self, symbols: list[str]) -> np.ndarray: + """Money exposure of the cap-weighted portfolio of the candidates (scale is irrelevant: only + its *direction* is used). This is the reference portfolio of the ``'market'`` tilt -- it + depends on the universe and the market caps, never on the request's risk parameters.""" + w = self.md.market_weights(symbols).reindex(symbols).fillna(0.0).values + if w.sum() <= 0: + w = np.ones(len(symbols)) + return w / w.sum() + + def scenario_key(self, req: OptimizationRequest, symbols: list[str], tilt_mode: str | None = None, + n_scenarios: int | None = None) -> dict: + """Exactly the inputs the scenario *shocks* depend on. + + Two requests with the same key share their scenario set: the risk limit, the confidence + level, the constraints, the transaction costs, the objective, the cash and the holdings do + **not** appear here, so a whole family of runs is compared on identical scenarios. Neither + does the expected-return model: the drift is applied afterwards + (:meth:`ScenarioSet.with_drift`), so changing views shifts the scenarios without touching + their random part. The one exception is ``importance_sampling="adaptive"``, whose tilt + direction comes from a first-pass solution; the key then records the parameters of that + solve and runs with different risk parameters no longer share a set. + """ + S = int(n_scenarios or req.scenario_count(len(symbols))) + mode = tilt_mode if tilt_mode is not None else req.tilt_mode(S) + key = { + "symbols": list(symbols), "freq": req.freq, "horizon_periods": req.horizon_periods, + "model": req.scenario_model, "n_scenarios": S, "seed": req.seed, + "sampling": req.sampling, "moment_match": bool(req.moment_match), + "covariance": CovarianceModel.from_dict(req.covariance).describe(), + "importance_sampling": mode, "as_of": str(self.md.last_date().date()), + } + if mode != "off": + key["is_mixture"] = req.is_mixture + if mode == "adaptive": # the tilt depends on the optimisation parameters of the first pass + key["adaptive_tilt"] = {"confidence": req.confidence, "cvar_limit": req.cvar_limit, + "cvar_limit_fraction": req.cvar_limit_fraction, "objective": req.objective, + "cash": req.cash, "holdings": dict(sorted(req.holdings.items())), + "constraints": asdict(req.constraints), "costs": asdict(req.costs), + "expected_returns": req.expected_returns, "views": _view_fingerprint(req.views)} + if req.scenario_model in ("empirical", "innovations", "fhs"): + key["innovations"] = {"history": req.innovation_history, "standardize": req.innovation_standardize} + key["id"] = scenario_hash(np.frombuffer(json.dumps(key, sort_keys=True, default=str).encode(), dtype=np.uint8)) + return key + + def _generate_scenarios(self, req: OptimizationRequest, symbols: list[str], tilt: TiltSpec | None, + n_scenarios: int) -> ScenarioSet: + """A drift-free scenario set (the expensive part); the drift is applied by the caller.""" + r, cov_model = self._returns_and_cov_model(req, symbols) + kwargs = {} + if req.scenario_model in ("empirical", "innovations", "fhs"): + kwargs = {"max_history": req.innovation_history, "root": "symmetric" if len(symbols) <= 250 else "cholesky", + "standardize": req.innovation_standardize} + sim = make_simulator(req.scenario_model, cov_model, **kwargs) + scen = sim.simulate(r, n_scenarios=n_scenarios, horizon_periods=req.horizon_periods, mu=None, seed=req.seed, + freq=req.freq, sampling=req.sampling, moment_match=req.moment_match, tilt=tilt, + drift_convention=req.drift_convention) + scen.covariance_model["conditioning"] = cov_model.conditioning(r) + return scen + + def scenarios(self, req: OptimizationRequest, symbols: list[str], mu_ann: pd.Series, tilt: TiltSpec | None = None, + n_scenarios: int | None = None, tilt_mode: str | None = None) -> ScenarioSet: + """The scenario set for this request, with the expected returns applied as the drift. + + The shocks are cached per :meth:`scenario_key`, so a series of runs that varies risk limits, + constraints, costs or views reuses one set (and pays the innovation filter only once). + """ + if req.scenario_model == "gaussian_exact": + raise ValueError("scenario_model 'gaussian_exact' does not generate scenarios") + S = int(n_scenarios or req.scenario_count(len(symbols))) + mode = tilt_mode if tilt_mode is not None else ("adaptive" if tilt is not None else req.tilt_mode(S)) + if mode == "market" and tilt is None: + tilt = TiltSpec(exposure=self.market_exposure(symbols), mixture=(req.is_mixture,) * 2, label="market") + key = self.scenario_key(req, symbols, mode, S) + base = self._scenario_cache.get(key["id"]) + hit = base is not None + if hit: + log.info("scenario cache hit (%s, %d scenarios)", key["id"], base.n_scenarios) + else: + base = self._generate_scenarios(req, symbols, tilt, S) + self._cache_scenarios(key["id"], base) + scen = base.with_drift(mu_ann / periods_per_year(req.freq), req.drift_convention) + scen.extra["key"] = key + scen.extra["cached"] = hit + return scen + + @staticmethod + def _sampling_warnings(n_assets: int, n_scenarios: int, cov_desc: dict) -> list[str]: + """Scenario count and covariance conditioning against the size of the candidate set.""" + out = [] + if n_scenarios < SCENARIOS_PER_ASSET_WARN * n_assets: + out.append( + f"{n_scenarios} scenarios for {n_assets} candidates ({n_scenarios / n_assets:.0f} per asset): " + f"the expected P&L and CVaR are still reliable, but the composition will move between " + f"near-equivalent portfolios when the seed changes - raise n_scenarios (about 64 per " + f"candidate), reduce max_assets, or use scenario_model='gaussian_exact' for an exact answer") + cond = cov_desc.get("conditioning") or {} + if cond.get("rank_deficient"): + out.append( + f"the covariance estimate is singular ({cond['effective_observations']} effective " + f"observations for {cond['n_assets']} assets, shrinkage {cond['shrinkage_applied']}): some " + f"portfolios appear riskless to the optimiser - use shrinkage='auto' or a longer window") + elif cond.get("under_determined") and cond.get("min_eigenvalue_ratio", 1.0) < 1e-3: + out.append( + f"the covariance estimate is ill-conditioned (smallest eigenvalue " + f"{cond['min_eigenvalue_ratio']:.1e} of the average variance, {cond['effective_observations']} " + f"effective observations for {cond['n_assets']} assets) - consider shrinkage='auto'") + return out + + def _cache_scenarios(self, key_id: str, scen: ScenarioSet) -> None: + self._scenario_cache[key_id] = scen + self._scenario_cache.move_to_end(key_id) + while len(self._scenario_cache) > SCENARIO_CACHE_SIZE: + self._scenario_cache.popitem(last=False) + + def clear_scenario_cache(self) -> None: + self._scenario_cache.clear() + + def _spec_common(self, req: OptimizationRequest, symbols: list[str]) -> dict: + prices = self.md.latest_prices(symbols, currency="base").reindex(symbols) + if prices.isna().any(): + raise ValueError(f"no current price for {prices.index[prices.isna()].tolist()}") + holdings = np.array([float(req.holdings.get(s, 0)) for s in symbols]) + wealth = req.cash + float(holdings @ prices.values) + cvar_limit = req.cvar_limit + if cvar_limit is None and req.cvar_limit_fraction is not None: + cvar_limit = req.cvar_limit_fraction * wealth + target_pnl = None if req.target_return is None else req.target_return * wealth + sectors = self.md.sectors(symbols).fillna("Unknown").tolist() + return dict(symbols=symbols, prices=prices.values, sectors=sectors, cash=req.cash, holdings=holdings, + confidence=req.confidence, cvar_limit=cvar_limit, objective=req.objective, + risk_aversion=req.risk_aversion, target_pnl=target_pnl, constraints=req.constraints, costs=req.costs) + + def build_spec(self, req: OptimizationRequest, symbols: list[str], scen: ScenarioSet) -> ProblemSpec: + # risk_drift='zero': the CVaR rows see the shocks without the drift; the objective still + # uses the model's expected returns, so the forecast can never relax the risk limit + scenarios = np.expm1(scen.shocks) if req.risk_drift == "zero" else scen.simple_returns() + return ProblemSpec(scenarios=scenarios, probabilities=scen.probabilities, + expected_returns=scen.expected_simple, **self._spec_common(req, symbols)) + + def build_gaussian_spec(self, req: OptimizationRequest, symbols: list[str], mu_ann: pd.Series) -> tuple[ProblemSpec, dict]: + """Gaussian mode: horizon simple returns ``N(mu * h / ppy, h * Sigma_T)``.""" + ppy = periods_per_year(req.freq) + h = req.horizon_periods + r, cov_model = self._returns_and_cov_model(req, symbols) + cov = cov_model.estimate(r).values * h + mean = mu_ann.reindex(symbols).fillna(0.0).values / ppy * h + cov_desc = cov_model.describe() + cov_desc["conditioning"] = cov_model.conditioning(r) + desc = {"model": "gaussian_exact", "n_scenarios": 0, "n_assets": len(symbols), "horizon_periods": h, + "freq": req.freq, "as_of": str(r.index[-1].date()), "covariance_model": cov_desc, + "shrinkage_applied": cov_model.intensity(r), "drift_convention": "arithmetic"} + risk_mean = np.zeros_like(mean) if req.risk_drift == "zero" else mean + return ProblemSpec(scenarios=None, gaussian_mean=risk_mean, gaussian_cov=cov, expected_returns=mean, + **self._spec_common(req, symbols)), desc + + def _solve_fn(self, req: OptimizationRequest, prefer_local: bool = False, deadline: float | None = None): + """A solve function that spends at most the request's remaining time budget. + + ``time_limit`` is the budget of the *whole* request. Row generation and the Gaussian outer + approximation call the solver repeatedly, so each call gets what is left of it (at least a + second, since a solver cannot be given zero). + """ + def limit() -> float | None: + if deadline is None: + return req.time_limit + return max(1.0, deadline - time.monotonic()) + + # `prefer_local` covers the models cuOpt cannot be trusted with; see _solve_scenarios for + # the CVaR-objective case and `optimize` for the Gaussian cut MILPs. An explicit + # solver='cuopt' is still honoured -- the caller asked for it and gets cuOpt's own answer. + if prefer_local and req.solver in ("auto", "highs"): + return lambda model: HighsSolver(self.settings).solve(model, limit()) + return lambda model: solve_with_fallback(model, self.settings, solver=req.solver, time_limit=limit()) + + def _deadline(self, req: OptimizationRequest) -> float | None: + return None if req.time_limit is None else time.monotonic() + float(req.time_limit) + + def _solve_scenarios(self, req: OptimizationRequest, spec: ProblemSpec) -> tuple[SolverResult, MeanCVaRProblem, dict]: + deadline = self._deadline(req) + info: dict = {} + # cuOpt 26.08 aborts when the CVaR sits in the *objective* and the scenarios carry + # probabilities: the objective coefficients are then p_s/(1-beta), which importance sampling + # spreads over about six orders of magnitude (measured 1.1e-06 .. 1.0, ratio 8.8e5), and the + # LP relaxation aborts within five seconds -- with the integers relaxed as well, so it is the + # LP and not the branch and bound. The same model with uniform weights (ratio 41) solves in + # 3.8 s, five times out of five, matching HiGHS to 7e-5. With CVaR as a *constraint* the + # weights sit in a matrix row instead, where cuOpt's scaling copes, and it solves either way. + weighted_cvar_objective = req.objective == "min_cvar" and spec.probabilities is not None + if weighted_cvar_objective and req.solver == "auto": + info["solver_routing"] = "highs: CVaR objective with probability-weighted scenarios" + solve = self._solve_fn(req, prefer_local=weighted_cvar_objective, deadline=deadline) + if req.uses_row_generation(spec.n_scenarios): + sol, problem, rg = solve_by_row_generation(spec, solve, deadline=deadline) + return sol, problem, {**info, "row_generation": rg.as_dict()} + problem = MeanCVaRProblem(spec) + return solve(problem.build()), problem, info + + # -- main entry point ------------------------------------------------------------------------- + def optimize(self, req: OptimizationRequest) -> OptimizationResult: + timings: dict = {} + t0 = time.monotonic() + symbols = self.candidate_symbols(req) + mu_ann, er_desc, _bl = self.expected_returns(req, symbols) + timings["expected_returns_s"] = round(time.monotonic() - t0, 3) + extra_info: dict = {} + warnings: list[str] = [] + + if req.scenario_model == "gaussian_exact": + t1 = time.monotonic() + spec, scen_desc = self.build_gaussian_spec(req, symbols, mu_ann) + warnings.extend(self._sampling_warnings(len(symbols), 10 ** 9, scen_desc["covariance_model"])) + timings["model_s"] = round(time.monotonic() - t1, 3) + t3 = time.monotonic() + # the cut MILPs are tiny (N integers, a few dozen rows): HiGHS locally unless cuOpt is requested + sol, problem, oa = solve_gaussian_exact(spec, self._solve_fn(req, prefer_local=True, + deadline=self._deadline(req))) + extra_info["outer_approximation"] = oa.as_dict() + timings["solve_s"] = round(time.monotonic() - t3, 3) + exp_simple = spec.gaussian_mean + scen_hash = None + else: + t1 = time.monotonic() + S = req.scenario_count(len(symbols)) + mode = req.tilt_mode(S) + tilt = None + if mode == "adaptive": + # stage 1: an untilted (moment-matched) set gives the loss direction for the tilt + scen1 = self.scenarios(req, symbols, mu_ann, n_scenarios=min(S, STAGE1_SCENARIOS), tilt_mode="off") + spec1 = self.build_spec(req, symbols, scen1) + sol1, problem1, _ = self._solve_scenarios(req, spec1) + if sol1.ok: + exposure = problem1.decode(sol1.x)["value"] + if exposure @ spec1.prices > 0: + tilt = TiltSpec.at_confidence(exposure, req.confidence, req.is_mixture * 2) + extra_info["importance_sampling"] = {"stage1_scenarios": scen1.n_scenarios, + "stage1_status": sol1.status} + if tilt is None: + mode = "off" + warnings.append("importance sampling skipped: no first-pass portfolio") + scen = self.scenarios(req, symbols, mu_ann, tilt=tilt, tilt_mode=mode) + warnings.extend(self._sampling_warnings(len(symbols), scen.n_scenarios, scen.covariance_model)) + timings["scenarios_s"] = round(time.monotonic() - t1, 3) + t2 = time.monotonic() + spec = self.build_spec(req, symbols, scen) + timings["build_s"] = round(time.monotonic() - t2, 3) + t3 = time.monotonic() + sol, problem, rg = self._solve_scenarios(req, spec) + extra_info.update(rg) + timings["solve_s"] = round(time.monotonic() - t3, 3) + scen_desc = scen.describe() + exp_simple = scen.expected_simple + scen_hash = scen.hash + extra_info["scenario_cache"] = {"key": scen.extra["key"]["id"], "reused": bool(scen.extra.get("cached"))} + timings["total_s"] = round(time.monotonic() - t0, 3) + key = None if req.scenario_model == "gaussian_exact" else scen.extra["key"] + return self._result(req, symbols, spec, problem, sol, scen_desc, scen_hash, exp_simple, er_desc, mu_ann, + timings, extra_info, warnings, key) + + def _result(self, req, symbols, spec, problem, sol, scen_desc, scen_hash, exp_simple, er_desc, mu_ann, timings, + extra_info, warnings, scenario_key=None) -> OptimizationResult: + h = req.horizon_periods + warnings = list(warnings) + if sol.status == "FeasibleFound": + # FeasibleFound can come from the solver's own limit *or* from an iteration cap in the + # outer approximation / row generation, which is not a time limit at all + oa, rg = extra_info.get("outer_approximation") or {}, extra_info.get("row_generation") or {} + if oa and not oa.get("converged"): + why = f"the Gaussian outer approximation stopped after {oa.get('iterations')} cuts" + elif rg and not rg.get("converged"): + why = f"the tail-row generation stopped after {rg.get('iterations')} passes" + else: + why = "the solver stopped at its limit" + warnings.append(f"{why}; solution feasible but not proven optimal (gap {sol.mip_gap})") + if "fallback" in sol.message: + warnings.append(sol.message) + if extra_info.get("solver_routing"): + warnings.append( + "solved on HiGHS (CPU) although cuOpt is reachable: cuOpt 26.08 aborts when the CVaR " + "is the objective *and* the scenarios are probability-weighted, because importance " + "sampling spreads the objective coefficients over six orders of magnitude. Set " + "importance_sampling='off' to use the GPU with this objective, or keep the weights " + "and the CPU - HiGHS solves it in a couple of seconds either way") + milp_desc = {"n_vars": problem.n_vars, "n_scenarios": spec.n_scenarios, "gaussian": problem.gaussian, + "weighted_scenarios": spec.probabilities is not None} + base_kwargs = dict( + status=sol.status, solver=sol.solver, mode=req.mode, as_of=scen_desc.get("as_of", ""), wealth=spec.wealth, + cash_start=req.cash, cvar_limit=spec.cvar_limit, confidence=req.confidence, horizon=req.horizon, + horizon_periods=h, n_candidates=len(symbols), + model={"expected_returns": er_desc, "scenarios": scen_desc, "milp": milp_desc, + "constraints": asdict(req.constraints), "costs": asdict(req.costs), "objective": req.objective, + "sampling": {"sampling": req.sampling, "moment_match": req.moment_match, + "importance_sampling": req.importance_sampling, "row_generation": req.row_generation, + "drift_convention": req.drift_convention, "risk_drift": req.risk_drift}}, + solver_info={"status": sol.status, "solver": sol.solver, "solve_time_s": sol.solve_time, + "mip_gap": sol.mip_gap, "objective": sol.objective, "message": sol.message, **sol.raw, + **extra_info}, + timings=timings, reproducible=sol.reproducible, scenario_hash=scen_hash, scenario_key=scenario_key, + before=(self.portfolio_figures(spec, spec.holdings, req.cash) if spec.rebalancing else None), + warnings=warnings, + # HiGHS' branch-and-bound is deterministic: identical inputs give identical shares. cuOpt (26.08) + # ignores gap settings over REST and its GPU heuristics stop within ~1e-4 of the optimum: identical + # objective, CVaR and expected P&L up to that gap, but near-tied share counts can differ. + reproducibility=("none" if not sol.reproducible else "exact" if sol.solver == "highs" else "within_gap"), + ) + if not sol.ok: + warnings.append(f"no solution: {sol.status} {sol.message}".strip()) + return OptimizationResult(cash_end=req.cash, invested=0.0, total_cost=0.0, turnover=0.0, expected_pnl=0.0, + expected_return=0.0, var=float("nan"), cvar=float("nan"), n_positions=0, + positions=[], sector_exposure={}, **base_kwargs) + d = problem.decode(sol.x) + if not problem.gaussian and spec.probabilities is not None and d["invested"] > 0: + alignment = self.tilt_alignment(spec, d["value"], self.market_exposure(symbols)) + base_kwargs["solver_info"]["tilt_alignment"] = round(alignment, 3) + if np.isfinite(alignment) and alignment < TILT_ALIGNMENT_WARN: + warnings.append( + f"the solution's loss is barely correlated with the importance-sampling reference " + f"(alignment {alignment:.2f}); the tilt did not help here - consider " + f"importance_sampling='adaptive' or 'off'") + if spec.cvar_limit is not None and d["cvar"] > spec.cvar_limit * (1 + 1e-6): + rg = extra_info.get("row_generation") or {} + oa = extra_info.get("outer_approximation") or {} + over = d["cvar"] / spec.cvar_limit - 1 + # enough precision for a small overshoot: "by 0.0%" reads as no breach at all + amount = f"{d['cvar'] - spec.cvar_limit:,.0f}" if over >= 5e-4 else f"{d['cvar'] - spec.cvar_limit:,.2f}" + # name the mechanism that left the gap, and the remedy that applies to *it* + if rg and not rg.get("converged"): + cause = (f": the tail-row generation stopped after {rg.get('iterations')} passes " + f"({rg.get('rows_final')} of {spec.n_scenarios} rows), so scenarios outside the " + f"working set are not covered - raise time_limit or set row_generation='off'") + elif oa and not oa.get("converged"): + cause = (f": the Gaussian outer approximation stopped after {oa.get('iterations')} cuts " + f"without converging, and it bounds the CVaR from below until it does - raise " + f"time_limit, or use scenario_model='gbm' for an exactly enforced limit") + else: + cause = (": the solver stopped within its MIP gap rather than at the true optimum - raise " + "time_limit, or lower n_scenarios / max_assets") + warnings.append( + f"the proposal BREACHES the CVaR limit by {amount} ({over:.3%}: {d['cvar']:,.0f} against " + f"{spec.cvar_limit:,.0f})" + cause) + # Minimising CVaR puts as little at risk as it is allowed to: cash has a CVaR of zero, so + # without a target return or a floor on the invested fraction the answer is trivially cash, + # and with a small target it is a sliver of the wealth. Both are correct and both look broken. + if sol.ok and req.objective == "min_cvar" and req.constraints.min_invested <= 0: + share = d["invested"] / spec.wealth if spec.wealth else 0.0 + if share < 0.5: + target = "no target return" if not req.target_return else f"a target return of {req.target_return:.3%}" + warnings.append( + f"only {share:.1%} of the wealth is invested: minimising CVaR holds as little risk " + f"as the constraints allow, and cash has a CVaR of zero. With {target} and no " + f"min_invested floor this is the true optimum - raise target_return, or set " + f"constraints.min_invested (e.g. 0.95) to make it deploy the money") + if sol.ok and d["invested"] <= 0: + if req.objective == "min_cvar": + pass # already explained above, in more useful detail + elif req.objective == "utility": + # E[P&L] and CVaR are both money over the horizon, so lambda is dimensionless and + # its natural scale is E[P&L]/CVaR at the optimum -- of the order of 0.02 over a + # week. At lambda = 1 every franc of expected shortfall costs a franc of expected + # return and staying in cash is genuinely optimal, which is easy to mistake for a bug. + warnings.append( + f"nothing was invested: with risk_aversion={req.risk_aversion:g} the penalty " + f"{req.risk_aversion:g} x CVaR exceeds the expected P&L of every feasible " + f"portfolio, so cash maximises the utility. Over {h} periods the trade-off is of " + f"the order of 0.02; lower risk_aversion, or use objective='max_return' with a " + f"cvar_limit") + elif req.objective != "min_cvar": + warnings.append("nothing was invested: the constraints leave no position worth taking") + if spec.cvar_limit is not None and d["invested"] > 0: + if d["cvar"] <= 0: + warnings.append( + f"the {req.confidence:.0%} CVaR of the proposal is a *gain* ({-d['cvar']:,.0f}): over " + f"{h} periods the expected return ({d['expected_pnl'] / spec.wealth:.1%} of wealth) is larger " + f"than the tail of the return distribution, so the risk limit constrains nothing and the " + f"portfolio is driven by the return forecast and the weight caps alone - use " + f"risk_drift='zero', a shorter horizon, or a smaller limit") + elif d["cvar"] < 0.25 * spec.cvar_limit: + warnings.append( + f"the CVaR limit is far from binding ({d['cvar']:,.0f} against a limit of " + f"{spec.cvar_limit:,.0f}): the proposal is set by the weight and sector caps, not by risk") + held_mu = np.array([mu_ann.get(s, 0.0) for i, s in enumerate(symbols) if d["shares"][i] > 0]) + if held_mu.size and np.nanmax(held_mu) > 0.60: + warnings.append( + f"the expected-return model gives up to {np.nanmax(held_mu):.0%} p.a. to a held stock " + f"({er_desc.get('type')}): an optimiser maximising expected return will concentrate in exactly " + f"those names - check the model before reading the proposal") + u = self.md.universe + positions = [] + for i, s in enumerate(symbols): + if d["shares"][i] == 0 and d["trades"][i] == 0: + continue + positions.append({ + "symbol": s, "name": u.at[s, "name"] if s in u.index else s, "sector": spec.sectors[i], + "currency": u.at[s, "listing_currency"] if s in u.index else None, + "price": float(spec.prices[i]), "shares": int(d["shares"][i]), + "shares_before": int(spec.holdings[i]), "trade": int(d["trades"][i]), + "value": float(d["value"][i]), "weight": float(d["weights"][i]), + "trade_value": float(d["trades"][i] * spec.prices[i]), "cost": float(d["cost"][i]), + "expected_return_ann": float(mu_ann.get(s, np.nan)), + "expected_return_horizon": float(exp_simple[i]), + }) + positions.sort(key=lambda p: -p["value"]) + sector_exposure = pd.Series(d["value"]).groupby(pd.Series(spec.sectors)).sum() / spec.wealth + return OptimizationResult( + cash_end=d["cash_end"], invested=d["invested"], total_cost=d["total_cost"], turnover=d["turnover"], + expected_pnl=d["expected_pnl"], expected_return=d["expected_pnl"] / spec.wealth, var=d["var"], + cvar=d["cvar"], n_positions=int((d["shares"] > 0).sum()), positions=positions, + sector_exposure={k: float(v) for k, v in sector_exposure.round(6).items() if v > 0}, **base_kwargs, + ) + + @staticmethod + def tilt_alignment(spec: ProblemSpec, exposure: np.ndarray, reference: np.ndarray) -> float: + """Correlation between the losses of the solution and of the tilt's reference portfolio. + + The importance-sampling tilt enriches the tail *of the reference portfolio*; a solution + whose loss is highly correlated with it gets the full variance reduction (measured: 2.3x + to 5.3x at beta = 0.99 for alignments of 0.9 to 1.0), one that is uncorrelated gets none + and one that is anti-correlated is estimated slightly *less* precisely than without the + tilt. Long-only portfolios under single-stock and sector caps are market-like in practice + (alignment around 0.9), but the figure is reported so that the exception is visible. + """ + w = spec.weights # both arguments are money exposures per asset (any common scale) + a = spec.scenarios @ np.asarray(exposure, dtype=float) + b = spec.scenarios @ np.asarray(reference, dtype=float) + a = a - w @ a + b = b - w @ b + denom = np.sqrt((w @ (a * a)) * (w @ (b * b))) + return float((w @ (a * b)) / denom) if denom > 0 else float("nan") + + @staticmethod + def portfolio_figures(spec: ProblemSpec, shares: np.ndarray, cash: float, total_cost: float = 0.0) -> dict: + """Expected P&L, VaR, CVaR and exposures of a share portfolio under this spec's model. + + The same quantities the solution reports, so that the starting portfolio of a rebalancing + can be compared with the proposal on identical scenarios (or, in Gaussian mode, on the + identical normal model). + """ + shares = np.asarray(shares, dtype=float) + value = shares * spec.prices + invested = float(value.sum()) + expected_pnl = float(spec.mean_returns @ value) - total_cost + if spec.is_gaussian: + sigma = float(np.sqrt(max(value @ spec.gaussian_cov @ value, 0.0))) + z = float(norm.ppf(spec.confidence)) + var_m, cvar_m = -expected_pnl + z * sigma, -expected_pnl + cvar_multiplier(spec.confidence) * sigma + else: + pnl = spec.scenarios @ value - total_cost + var_m, cvar_m = scenario_cvar(-pnl, spec.confidence, spec.probabilities) + sector = (pd.Series(value).groupby(pd.Series(spec.sectors)).sum() / spec.wealth).round(6) + return { + "invested": invested, "cash": float(cash), "wealth": float(cash + invested), + "n_positions": int((shares > 0).sum()), "expected_pnl": expected_pnl, + "expected_return": expected_pnl / spec.wealth if spec.wealth else 0.0, + "var": float(var_m), "cvar": float(cvar_m), + "cvar_fraction": float(cvar_m) / spec.wealth if spec.wealth else 0.0, + "weights": {s: float(value[i] / spec.wealth) for i, s in enumerate(spec.symbols) if value[i] != 0}, + "sector_exposure": {k: float(v) for k, v in sector.items() if v > 0}, + } + + # -- verification helper: risk of an arbitrary share portfolio on the same scenario set ----------- + @staticmethod + def portfolio_risk_on_scenarios(spec: ProblemSpec, shares: np.ndarray, confidence: float) -> dict: + pnl = spec.scenarios @ (np.asarray(shares) * spec.prices) + var_m, cvar_m = scenario_cvar(-pnl, confidence, spec.probabilities) + w = spec.weights + return {"expected_pnl": float((w @ pnl) / w.sum()), "var": var_m, "cvar": cvar_m} diff --git a/quantcore/optimization/solvers.py b/quantcore/optimization/solvers.py new file mode 100644 index 0000000..9897814 --- /dev/null +++ b/quantcore/optimization/solvers.py @@ -0,0 +1,317 @@ +"""MILP solvers behind one interface: NVIDIA cuOpt (REST, on the DGX Spark) and HiGHS (scipy). + +The cuOpt server solves LP/MILP from a JSON body with the constraint matrix in CSR form and +answers asynchronously (submit -> poll). :class:`CuOptSolver` wraps this with timeouts, retry +with back-off on connection errors and a circuit breaker; :func:`solve_with_fallback` uses +the local HiGHS solver when cuOpt is unreachable (deterministic analysis must keep working +when an external dependency is down -- and the result says which solver produced it). +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field + +import numpy as np +import requests + +from quantcore.config import Settings +from quantcore.data.yahoo import CircuitBreaker, CircuitOpenError +from quantcore.optimization.problem import MILPModel + +log = logging.getLogger(__name__) + +_HEALTH_CACHE: dict[str, tuple[float, bool]] = {} # base_url -> (checked_at, reachable) + +CUOPT_HEADERS = {"CLIENT-VERSION": "custom", "Accept": "application/json", "Content-Type": "application/json"} +OPTIMAL_STATUSES = {"Optimal", "FeasibleFound"} + +# cuOpt 26.08 aborts on some of these models rather than solving them, and which of its LP methods +# copes depends on the model. Measured on a 4 137-variable / 4 106-row mean-CVaR MILP (40 candidates, +# 4 096 scenarios, beta = 99 %): with CVaR in the *objective* (objective='min_cvar') the default +# concurrent method, the dual simplex (2) and method 3 all abort within ~5 s, while method 1 (PDLP) +# returns the optimum in 4.9 s, matching HiGHS to 7e-5. With CVaR as a *constraint* the default +# solves it. So an abort is retried down this ladder before the caller is told cuOpt could not do it. +CUOPT_RETRY_METHODS = (1, 2) + + +@dataclass +class SolverResult: + status: str # Optimal | FeasibleFound | Infeasible | Unbounded | TimeLimit | Error + x: np.ndarray | None + objective: float | None + solver: str + solve_time: float + mip_gap: float | None = None + message: str = "" + raw: dict = field(default_factory=dict) + + @property + def ok(self) -> bool: + return self.x is not None and self.status in OPTIMAL_STATUSES + + @property + def reproducible(self) -> bool: + """True when optimality was proven within the MIP gap: with the same model (same scenario + hash) any solver run returns the same objective value, and the same shares up to exact ties. + A ``FeasibleFound`` (time limit) result depends on the solver's timing and is not reproducible.""" + return self.ok and self.status == "Optimal" + + +def _bound(v: float) -> float | str: + if np.isposinf(v): + return "inf" + if np.isneginf(v): + return "ninf" + return float(v) + + +def cuopt_solver_config(settings: Settings, time_limit: float, extra_config: dict | None = None) -> dict: + """cuOpt ``solver_config``: time limit, MIP gaps (reproducibility), threads, no console log.""" + cfg = {"time_limit": float(time_limit), "log_to_console": False, + "tolerances": {"mip_relative_gap": settings.mip_rel_gap, "mip_absolute_gap": settings.mip_abs_gap}} + if settings.cuopt_threads: + cfg["num_cpu_threads"] = int(settings.cuopt_threads) + if extra_config: + tol = {**cfg["tolerances"], **(extra_config.get("tolerances") or {})} + cfg = {**cfg, **extra_config, "tolerances": tol} + return cfg + + +def to_cuopt_payload(model: MILPModel, time_limit: float, extra_config: dict | None = None, + settings: Settings | None = None) -> dict: + """The cuOpt LP/MILP request body for a :class:`MILPModel`.""" + A = model.A.tocsr() + payload = { + "csr_constraint_matrix": { + "offsets": A.indptr.tolist(), "indices": A.indices.tolist(), "values": A.data.tolist(), + }, + "constraint_bounds": { + "upper_bounds": [_bound(v) for v in model.row_ub], "lower_bounds": [_bound(v) for v in model.row_lb], + }, + "objective_data": {"coefficients": model.c.tolist(), "scalability_factor": 1.0, "offset": model.offset}, + "variable_bounds": { + "upper_bounds": [_bound(v) for v in model.var_ub], "lower_bounds": [_bound(v) for v in model.var_lb], + }, + "variable_types": list(model.var_types), + "variable_names": list(model.var_names), + "maximize": bool(model.maximize), + "solver_config": (cuopt_solver_config(settings, time_limit, extra_config) if settings is not None + else {"time_limit": float(time_limit), **(extra_config or {})}), + } + return payload + + +def encode_payload(payload: dict, use_msgpack: bool) -> tuple[bytes, str]: + """Serialise the request body: msgpack (about half the size of JSON, faster to parse) when + available and enabled, JSON otherwise.""" + if use_msgpack: + try: + import msgpack + + return msgpack.packb(payload, use_bin_type=True), "application/vnd.msgpack" + except ImportError: # pragma: no cover + pass + import json + + return json.dumps(payload).encode(), "application/json" + + +class CuOptSolver: + """Client of the cuOpt self-hosted server (``/cuopt/request`` + ``/cuopt/solution/{id}``).""" + + name = "cuopt" + + def __init__(self, settings: Settings, breaker: CircuitBreaker | None = None): + self.settings = settings + self.base_url = settings.cuopt_url.rstrip("/") + self.breaker = breaker or CircuitBreaker(threshold=3, cooldown=120.0, name="cuOpt") + + # -- health ---------------------------------------------------------------------------------- + def reachable(self, ttl: float = 30.0) -> bool: + """Is the server up? Cached for ``ttl`` seconds (a solve by row generation asks repeatedly). + + Without this an unreachable server costs the connect timeout on every attempt before the + HiGHS fallback takes over -- minutes of apparent hanging for ``solver='auto'``. + """ + now = time.monotonic() + hit = _HEALTH_CACHE.get(self.base_url) + if hit is not None and now - hit[0] < ttl: + return hit[1] + ok = bool(self.health(timeout=2.0).get("reachable")) + _HEALTH_CACHE[self.base_url] = (now, ok) + return ok + + def health(self, timeout: float = 5.0) -> dict: + try: + r = requests.get(f"{self.base_url}/cuopt/health", headers=CUOPT_HEADERS, timeout=timeout) + r.raise_for_status() + return {"reachable": True, **r.json()} + except Exception as exc: # noqa: BLE001 + return {"reachable": False, "error": str(exc)} + + # -- solve ------------------------------------------------------------------------------------- + def _post_with_retry(self, payload: dict) -> str: + body, content_type = encode_payload(payload, self.settings.cuopt_msgpack) + headers = {**CUOPT_HEADERS, "Content-Type": content_type} + # the POST blocks until the server has read and parsed the whole model, so the timeout has + # to scale with the payload: a client-side timeout here surfaces as an "aborted" request + # (connect, read): a big body needs a long *read* timeout, never a long connect wait -- and + # cuOpt serves one request at a time, so a busy server refuses the connection rather than stalling + # one scalar timeout: urllib3 governs the body write with the connect timeout, so a short + # connect timeout aborts large uploads mid-flight + timeout = self.settings.cuopt_request_timeout + self.settings.cuopt_upload_seconds_per_mb * len(body) / 1e6 + delay = 1.0 + last: Exception | None = None + for attempt in range(1, 4): + # a fresh connection per attempt: a timed-out POST leaves a partially written body in a + # pooled connection, and the retry would append a second body -- the server then reports + # "unpack received extra data" (422) or aborts the job + with requests.Session() as session: + try: + r = session.post(f"{self.base_url}/cuopt/request", data=body, + headers={**headers, "Connection": "close"}, timeout=timeout) + if r.status_code in (400, 422): # schema or decoding error: no point retrying + raise ValueError(f"cuOpt rejected the model ({r.status_code}): {r.text[:500]}") + r.raise_for_status() + return r.json()["reqId"] + except (requests.ConnectionError, requests.Timeout) as exc: + last = exc + log.warning("cuOpt submit failed (attempt %d/3): %s", attempt, exc) + time.sleep(delay) + delay *= 2 + raise ConnectionError( + f"cuOpt unreachable at {self.base_url} (payload {len(body) / 1e6:.1f} MB, timeout " + f"{timeout:.0f} s per attempt)") from last + + def solve(self, model: MILPModel, time_limit: float | None = None, extra_config: dict | None = None, + _retry: bool = True) -> SolverResult: + """Submit, poll, parse. An aborted solve is retried down :data:`CUOPT_RETRY_METHODS`.""" + self.breaker.check() + time_limit = time_limit or self.settings.cuopt_time_limit + payload = to_cuopt_payload(model, time_limit, extra_config, self.settings) + t0 = time.monotonic() + try: + req_id = self._post_with_retry(payload) + except (ConnectionError, ValueError): + self.breaker.failure() + raise + deadline = t0 + self.settings.cuopt_timeout + url = f"{self.base_url}/cuopt/solution/{req_id}" + try: + while time.monotonic() < deadline: + try: + r = requests.get(url, headers=CUOPT_HEADERS, timeout=self.settings.cuopt_request_timeout) + except (requests.ConnectionError, requests.Timeout) as exc: + # the server went away mid-solve (e.g. its solver process died and took the web + # server with it): raise the builtin ConnectionError that solve_with_fallback + # turns into a HiGHS solve -- requests' own class is not a subclass of it + self.breaker.failure() + raise ConnectionError(f"cuOpt connection lost during the solve at {self.base_url}: " + f"{str(exc)[:120]}") from exc + body = r.json() + if body.get("error_result"): + message = str(body.get("error")) + if _retry and "abort" in message.lower(): + for method in CUOPT_RETRY_METHODS: + if (extra_config or {}).get("method") == method: + continue + log.warning("cuOpt aborted (%s) - retrying with method=%d", message[:60], method) + retried = self.solve(model, time_limit, {**(extra_config or {}), "method": method}, + _retry=False) + if retried.ok: + retried.message = (f"{retried.message} (cuOpt aborted with its default method; " + f"solved with method={method})").strip() + return retried + message = (f"{message} - and every method in {CUOPT_RETRY_METHODS} aborted too") + self.breaker.failure() + return SolverResult("Error", None, None, self.name, time.monotonic() - t0, message=message, + raw=body) + if "response" in body: + self.breaker.success() + return self._parse(body["response"]["solver_response"], time.monotonic() - t0) + time.sleep(self.settings.cuopt_poll_interval) + finally: + try: + requests.delete(url, headers=CUOPT_HEADERS, timeout=10) + except requests.RequestException: + pass + return SolverResult("TimeLimit", None, None, self.name, time.monotonic() - t0, message="client-side timeout") + + @staticmethod + def _parse(resp: dict, elapsed: float) -> SolverResult: + status = str(resp.get("status", "Unknown")) + sol = resp.get("solution") or {} + x = sol.get("primal_solution") + stats = sol.get("milp_statistics") or {} + return SolverResult( + status=status, x=None if x is None else np.asarray(x, dtype=float), + objective=sol.get("primal_objective"), solver="cuopt", solve_time=float(sol.get("solver_time", elapsed)), + mip_gap=stats.get("mip_gap"), raw={"status": status, "milp_statistics": stats, + "problem_category": sol.get("problem_category")}, + ) + + +class HighsSolver: + """Local MILP solver (HiGHS via ``scipy.optimize.milp``) -- the fallback and the test oracle.""" + + name = "highs" + + def __init__(self, settings: Settings | None = None): + self.settings = settings + + def solve(self, model: MILPModel, time_limit: float | None = None, mip_gap: float | None = None) -> SolverResult: + from scipy.optimize import Bounds, LinearConstraint, milp + + if mip_gap is None: + mip_gap = self.settings.mip_rel_gap if self.settings is not None else 1e-4 + t0 = time.monotonic() + c = -model.c if model.maximize else model.c + integrality = np.array([1 if t == "I" else 0 for t in model.var_types]) + cons = LinearConstraint(model.A, model.row_lb, model.row_ub) + bounds = Bounds(model.var_lb, model.var_ub) + options = {"disp": False, "mip_rel_gap": mip_gap} + if time_limit: + options["time_limit"] = float(time_limit) + res = milp(c, constraints=cons, integrality=integrality, bounds=bounds, options=options) + elapsed = time.monotonic() - t0 + status_map = {0: "Optimal", 1: "IterationLimit", 2: "Infeasible", 3: "Unbounded", 4: "Error"} + status = status_map.get(res.status, "Unknown") + if res.status == 1 and res.x is not None: + status = "FeasibleFound" + x = None if res.x is None else np.asarray(res.x, dtype=float) + obj = None if x is None else float(model.c @ x + model.offset) + return SolverResult(status, x, obj, self.name, elapsed, mip_gap=getattr(res, "mip_gap", None), + message=str(res.message), raw={"status_code": int(res.status)}) + + +def solve_with_fallback(model: MILPModel, settings: Settings, solver: str = "auto", + time_limit: float | None = None) -> SolverResult: + """``solver``: ``'cuopt'``, ``'highs'`` or ``'auto'`` (cuOpt, HiGHS if cuOpt is unreachable).""" + if solver == "highs": + return HighsSolver(settings).solve(model, time_limit) + cu = CuOptSolver(settings) + if solver == "auto" and settings.solver_fallback and not cu.reachable(): + log.info("cuOpt is not reachable at %s - solving with HiGHS", settings.cuopt_url) + result = HighsSolver(settings).solve(model, time_limit) + result.message = (result.message + " (fallback: cuOpt unreachable)").strip() + return result + reason = "cuOpt unavailable" + try: + result = cu.solve(model, time_limit) + if result.status != "Error" or solver == "cuopt": + return result + log.warning("cuOpt returned an error (%s)", result.message) + # the server is up and answered -- say so, or the reader sees a green health pill next to a + # result that claims cuOpt was "unavailable" and concludes the GPU is being ignored + reason = ("cuOpt is reachable but aborted on this model" if "abort" in result.message.lower() + else f"cuOpt returned an error: {result.message[:80]}") + except (ConnectionError, CircuitOpenError, ValueError) as exc: + if solver == "cuopt" or not settings.solver_fallback: + raise + log.warning("cuOpt unavailable (%s) - falling back to HiGHS", exc) + reason = f"cuOpt unreachable: {str(exc)[:80]}" + result = HighsSolver(settings).solve(model, time_limit) + result.message = f"{result.message} (fallback to HiGHS: {reason})".strip() + return result diff --git a/quantcore/universe/__init__.py b/quantcore/universe/__init__.py new file mode 100644 index 0000000..3c31226 --- /dev/null +++ b/quantcore/universe/__init__.py @@ -0,0 +1,18 @@ +"""Investment universe: candidate sources, Yahoo symbol mapping and the one-off ``SetupService``.""" + +from quantcore.universe.setup_service import SetupService, UniverseBuildReport +from quantcore.universe.sources import ( + fetch_sp500_wikipedia, + fetch_stoxx600_ishares, + load_static_candidates, +) +from quantcore.universe.symbols import to_yahoo_symbol + +__all__ = [ + "SetupService", + "UniverseBuildReport", + "fetch_sp500_wikipedia", + "fetch_stoxx600_ishares", + "load_static_candidates", + "to_yahoo_symbol", +] diff --git a/quantcore/universe/setup_service.py b/quantcore/universe/setup_service.py new file mode 100644 index 0000000..0f35364 --- /dev/null +++ b/quantcore/universe/setup_service.py @@ -0,0 +1,209 @@ +"""``SetupService`` -- builds the investment universe once and persists everything. + +Steps (each is a method, so parts can be re-run): + +1. **candidates**: S&P 500 (US) and STOXX Europe 600 (Europe) constituents, see + :mod:`quantcore.universe.sources`. +2. **metadata**: Yahoo ``Ticker.info`` for every candidate -- listing currency, exchange, + domicile, Yahoo sector/industry, market capitalisation, shares outstanding, first trade date. +3. **prices**: complete adjusted-close history since ``settings.history_start`` for all + candidates (so later universe changes need no re-download); FX rates to the base currency. +4. **selection**: one listing per company, rank by market capitalisation in the base currency, + keep the largest ``n_us`` / ``n_europe``, then drop stocks without prices since 2005 + (``settings.required_history_from``) or with poor coverage -- which is why the final + universe has fewer than 1000 names. +5. **persist**: ``universe/universe.csv`` with ticker, company name, country, listing currency, + exchange, GICS sector, Yahoo sector/industry, market cap (local and base), first price date. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import datetime, timezone + +import numpy as np +import pandas as pd + +from quantcore.config import Settings +from quantcore.data.cleaning import clean_prices, longest_missing_run +from quantcore.data.fx import download_fx +from quantcore.data.store import DataStore +from quantcore.data.yahoo import YahooClient, apply_price_scale, coverage_since, first_valid_dates +from quantcore.universe.sources import EUROPE_COUNTRIES, fetch_candidates +from quantcore.universe.symbols import normalise_share_class_name + +log = logging.getLogger(__name__) + +UNIVERSE_COLUMNS = [ + "symbol", "name", "region", "country", "listing_currency", "exchange", "exchange_code", "gics_sector", + "gics_sub_industry", "yahoo_sector", "yahoo_industry", "market_cap", "market_cap_base", + "shares_outstanding", "last_price", "first_price_date", "price_coverage", "max_missing_run", "index_weight", + "source", "rank_in_region", +] + + +@dataclass +class UniverseBuildReport: + started_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat(timespec="seconds")) + n_candidates: int = 0 + n_resolved: int = 0 + n_with_prices: int = 0 + n_selected: int = 0 + n_universe: int = 0 + by_region: dict = field(default_factory=dict) + by_country: dict = field(default_factory=dict) + by_sector: dict = field(default_factory=dict) + dropped_no_history: list[str] = field(default_factory=list) + dropped_unresolved: list[str] = field(default_factory=list) + first_price_date: str | None = None + last_price_date: str | None = None + finished_at: str | None = None + + def as_dict(self) -> dict: + return self.__dict__.copy() + + +class SetupService: + """One-off construction of the investment universe (idempotent; re-run to refresh).""" + + def __init__(self, settings: Settings, store: DataStore | None = None, client: YahooClient | None = None): + self.settings = settings + self.store = store or DataStore(settings) + self.client = client or YahooClient(settings) + + # -- step 1 -------------------------------------------------------------------------------- + def candidates(self, allow_web: bool = True, max_per_region: int | None = None) -> pd.DataFrame: + us = fetch_candidates("US", allow_web=allow_web) + eu = fetch_candidates("Europe", allow_web=allow_web) + if max_per_region: + us, eu = us.head(max_per_region), eu.head(max_per_region) + df = pd.concat([us, eu], ignore_index=True).drop_duplicates(subset="symbol").reset_index(drop=True) + self.store.save_candidates(df) + log.info("candidates: %d US + %d Europe = %d", len(us), len(eu), len(df)) + return df + + # -- step 2 -------------------------------------------------------------------------------- + def metadata(self, symbols: list[str]) -> pd.DataFrame: + def _progress(i, n): + log.info("metadata %d/%d", i, n) + + meta = self.client.ticker_infos(symbols, progress=_progress) + self.store.save_metadata(meta) + return meta + + # -- step 3 -------------------------------------------------------------------------------- + def prices_and_fx(self, symbols: list[str], meta: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + hist = self.client.download_history(symbols, start=self.settings.history_start)["Close"] + prices = clean_prices(apply_price_scale(hist, meta["price_scale"])) + self.store.save_prices(prices) + currencies = meta.loc[meta.index.isin(prices.columns), "currency"].dropna().unique().tolist() + fx = download_fx(self.client, currencies, self.settings.base_currency, self.settings.history_start) + self.store.save_fx(fx) + return prices, fx + + # -- step 4 -------------------------------------------------------------------------------- + def select(self, candidates: pd.DataFrame, meta: pd.DataFrame, prices: pd.DataFrame, fx: pd.DataFrame, + report: UniverseBuildReport | None = None) -> pd.DataFrame: + report = report or UniverseBuildReport() + s = self.settings + df = candidates.merge(meta, left_on="symbol", right_index=True, how="left") + resolved = df["quote_type"].eq("EQUITY") & df["currency"].notna() + report.dropped_unresolved = df.loc[~resolved, "symbol"].tolist() + df = df[resolved].copy() + report.n_resolved = len(df) + + # price statistics + have = df["symbol"].isin(prices.columns) + df = df[have].copy() + report.n_with_prices = len(df) + first = first_valid_dates(prices[df["symbol"]]) + cov = coverage_since(prices[df["symbol"]], pd.Timestamp(s.required_history_from)) + gaps = longest_missing_run(prices[df["symbol"]], pd.Timestamp(s.required_history_from)) + df["first_price_date"] = df["symbol"].map(first).map(lambda d: None if pd.isna(d) else pd.Timestamp(d).date().isoformat()) + df["price_coverage"] = df["symbol"].map(cov).round(4) + df["max_missing_run"] = df["symbol"].map(gaps) + last_px = prices[df["symbol"]].ffill().iloc[-1] + df["last_price"] = df["last_price"].where(df["last_price"].notna(), df["symbol"].map(last_px)) + + # market capitalisation in the base currency + fx_last = fx.ffill().iloc[-1] + rate = df["currency"].map(fx_last).astype(float) + cap_local = df["market_cap"].astype(float) + est = df["shares_outstanding"].astype(float) * df["last_price"].astype(float) + cap_local = cap_local.where(cap_local.notna() & (cap_local > 0), est) + df["market_cap"] = cap_local + df["market_cap_base"] = cap_local * rate + df["listing_currency"] = df["currency"] + df["name"] = df["long_name"].where(df["long_name"].notna(), df["name"]) + df["country"] = df["yahoo_country"].where(df["yahoo_country"].notna(), df["country"]) + + # Europe = EU + CH + NO + UK by domicile of the *source* (STOXX country), US = S&P 500 members + eu_mask = (df["region"] == "Europe") & ~candidates.set_index("symbol")["country"].reindex(df["symbol"]).isin(EUROPE_COUNTRIES).values + df = df[~eu_mask].copy() + + # one listing per company: keep the largest market cap among share classes + df["company_key"] = df["region"] + "|" + df["name"].map(normalise_share_class_name) + df = df.sort_values("market_cap_base", ascending=False).drop_duplicates("company_key") + + # rank within region and keep the largest N + df = df[df["market_cap_base"].notna() & (df["market_cap_base"] > 0)].copy() + df["rank_in_region"] = df.groupby("region")["market_cap_base"].rank(ascending=False, method="first").astype(int) + limit = df["region"].map({"US": s.n_us, "Europe": s.n_europe}) + selected = df[df["rank_in_region"] <= limit].copy() + report.n_selected = len(selected) + + # history filter: prices since 2005 and adequate coverage + has_history = ( + (pd.to_datetime(selected["first_price_date"]) <= pd.Timestamp(s.required_history_from)) + & (selected["price_coverage"] >= s.min_price_coverage) + & (selected["max_missing_run"] <= s.max_missing_run) + ) + report.dropped_no_history = selected.loc[~has_history, "symbol"].tolist() + universe = selected[has_history].sort_values(["region", "rank_in_region"]).reset_index(drop=True) + for col in UNIVERSE_COLUMNS: + if col not in universe.columns: + universe[col] = np.nan + universe = universe[UNIVERSE_COLUMNS] + report.n_universe = len(universe) + report.by_region = universe["region"].value_counts().to_dict() + report.by_country = universe["country"].value_counts().to_dict() + report.by_sector = universe["gics_sector"].value_counts().to_dict() + return universe + + # -- orchestration ---------------------------------------------------------------------------- + def run(self, allow_web: bool = True, max_per_region: int | None = None) -> UniverseBuildReport: + """Full setup: candidates -> metadata -> prices/FX -> selection -> persist.""" + report = UniverseBuildReport() + self.store.ensure_dirs() + cands = self.candidates(allow_web=allow_web, max_per_region=max_per_region) + report.n_candidates = len(cands) + meta = self.metadata(cands["symbol"].tolist()) + ok = meta.index[meta["quote_type"].eq("EQUITY")].tolist() + prices, fx = self.prices_and_fx(ok, meta) + universe = self.select(cands, meta, prices, fx, report) + self.store.save_universe(universe) + report.first_price_date = prices.index.min().date().isoformat() + report.last_price_date = prices.index.max().date().isoformat() + report.finished_at = datetime.now(timezone.utc).isoformat(timespec="seconds") + self.store.update_manifest( + setup_at=report.finished_at, n_candidates=report.n_candidates, n_universe=report.n_universe, + by_region=report.by_region, first_price_date=report.first_price_date, + last_price_date=report.last_price_date, n_symbols=len(prices.columns), + base_currency=self.settings.base_currency, required_history_from=self.settings.required_history_from, + ) + log.info("universe built: %d stocks (%s)", report.n_universe, report.by_region) + return report + + def rebuild_selection(self) -> UniverseBuildReport: + """Re-run only the selection step on stored data (e.g. after changing ``n_us``).""" + report = UniverseBuildReport() + cands = self.store.load_candidates() + meta = self.store.load_metadata() + prices = self.store.load_prices() + fx = self.store.load_fx() + report.n_candidates = len(cands) + universe = self.select(cands, meta, prices, fx, report) + self.store.save_universe(universe) + self.store.update_manifest(n_universe=report.n_universe, by_region=report.by_region) + return report diff --git a/quantcore/universe/sources.py b/quantcore/universe/sources.py new file mode 100644 index 0000000..561e767 --- /dev/null +++ b/quantcore/universe/sources.py @@ -0,0 +1,302 @@ +"""Candidate lists for the investment universe. + +Two curated index memberships serve as the *candidates* for "the largest 500 US and European +stocks": + +* **US**: the S&P 500 constituents (Wikipedia, ``List of S&P 500 companies``). +* **Europe** (EU, CH, NO, UK): the STOXX Europe 600 constituents, read from the daily holdings + file of the iShares STOXX Europe 600 UCITS ETF (complete list with country, exchange, + currency, GICS sector and index weight). Wikipedia's STOXX 600 page is a partial fallback. + +Every fetch is snapshotted to ``quantcore/data/static/`` so that the setup can be reproduced +offline (``load_static_candidates``). All frames share the same columns, see +:data:`CANDIDATE_COLUMNS`. +""" + +from __future__ import annotations + +import io +import logging +from datetime import date +from importlib import resources +from pathlib import Path + +import pandas as pd +import requests + +from quantcore.universe.symbols import to_yahoo_symbol + +log = logging.getLogger(__name__) + +CANDIDATE_COLUMNS = [ + "symbol", # Yahoo symbol + "source_ticker", # ticker as written by the source + "name", + "region", # 'US' | 'Europe' + "country", # domicile as given by the source (Yahoo's view is added later) + "exchange", # exchange name as given by the source + "listing_currency", # currency of the listing as given by the source (may be empty) + "gics_sector", # GICS sector, English + "gics_sub_industry", # only for the S&P 500 + "index_weight", # index / ETF weight in %, if the source gives one + "source", + "source_date", +] + +USER_AGENT = "Mozilla/5.0 (compatible; quantcore-setup/0.1; +https://fhgr.ch)" + +SP500_URL = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies" +STOXX600_WIKI_URL = "https://en.wikipedia.org/wiki/STOXX_Europe_600" +# iShares STOXX Europe 600 UCITS ETF (DE), holdings download of the German retail site +ISHARES_STOXX600_URL = ( + "https://www.ishares.com/de/privatanleger/de/produkte/251931/" + "ishares-stoxx-europe-600-ucits-etf-de-fund/1478358465952.ajax" + "?fileType=csv&fileName=EXSA_holdings&dataType=fund" +) + +# German labels of the iShares file -> English +ISHARES_SECTOR_EN = { + "IT": "Information Technology", + "Finanzwesen": "Financials", + "Gesundheitsversorgung": "Health Care", + "Energie": "Energy", + "Nichtzyklische Konsumgüter": "Consumer Staples", + "Zyklische Konsumgüter": "Consumer Discretionary", + "Industrie": "Industrials", + "Materialien": "Materials", + "Grundstoffe": "Materials", + "Kommunikation": "Communication Services", + "Versorger": "Utilities", + "Immobilien": "Real Estate", +} +ISHARES_COUNTRY_EN = { + "Vereinigtes Königreich": "United Kingdom", + "Frankreich": "France", + "Deutschland": "Germany", + "Schweiz": "Switzerland", + "Schweden": "Sweden", + "Italien": "Italy", + "Niederlande": "Netherlands", + "Spanien": "Spain", + "Norwegen": "Norway", + "Dänemark": "Denmark", + "Finnland": "Finland", + "Polen": "Poland", + "Belgien": "Belgium", + "Österreich": "Austria", + "Griechenland": "Greece", + "Irland": "Ireland", + "Portugal": "Portugal", + "Luxemburg": "Luxembourg", + "Tschechische Republik": "Czech Republic", + "Ungarn": "Hungary", + "Vereinigte Staaten": "United States", + "Bermuda": "Bermuda", + "Israel": "Israel", +} + +# The countries that make up "Europe" for this universe: EU members + CH + NO + UK. +# Domiciles outside (Bermuda, Israel, ...) are excluded even if they are STOXX 600 members. +EUROPE_COUNTRIES = { + "Austria", "Belgium", "Bulgaria", "Croatia", "Cyprus", "Czech Republic", "Denmark", "Estonia", + "Finland", "France", "Germany", "Greece", "Hungary", "Ireland", "Italy", "Latvia", "Lithuania", + "Luxembourg", "Malta", "Netherlands", "Poland", "Portugal", "Romania", "Slovakia", "Slovenia", + "Spain", "Sweden", "Switzerland", "Norway", "United Kingdom", +} + + +_GERMAN_MONTHS = {"jan": 1, "feb": 2, "mär": 3, "mar": 3, "apr": 4, "mai": 5, "jun": 6, "jul": 7, "aug": 8, + "sep": 9, "okt": 10, "nov": 11, "dez": 12} + + +def _parse_german_date(text: str) -> str | None: + """``'18.Sept.2026'`` -> ``'2026-09-18'`` (``None`` when unparseable).""" + parts = [p for p in text.replace(" ", "").split(".") if p] + if len(parts) != 3: + return None + try: + day, year = int(parts[0]), int(parts[2]) + month = _GERMAN_MONTHS.get(parts[1][:3].lower()) + return date(year, month, day).isoformat() if month else None + except ValueError: + return None + + +def _get(url: str, timeout: float = 30.0) -> bytes: + resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=timeout) + resp.raise_for_status() + return resp.content + + +def _empty() -> pd.DataFrame: + return pd.DataFrame(columns=CANDIDATE_COLUMNS) + + +# ---------------------------------------------------------------------------------------- +# US: S&P 500 from Wikipedia +# ---------------------------------------------------------------------------------------- +def fetch_sp500_wikipedia(timeout: float = 30.0) -> pd.DataFrame: + """S&P 500 constituents with GICS sector and sub-industry (Wikipedia).""" + html = _get(SP500_URL, timeout) + tables = pd.read_html(io.BytesIO(html)) + table = next(t for t in tables if "Symbol" in t.columns and "GICS Sector" in t.columns) + today = date.today().isoformat() + df = pd.DataFrame( + { + "symbol": [to_yahoo_symbol(s, country="United States") for s in table["Symbol"].astype(str)], + "source_ticker": table["Symbol"].astype(str), + "name": table["Security"].astype(str), + "region": "US", + "country": "United States", + "exchange": "", + "listing_currency": "USD", + "gics_sector": table["GICS Sector"].astype(str), + "gics_sub_industry": table["GICS Sub-Industry"].astype(str), + "index_weight": float("nan"), + "source": "wikipedia_sp500", + "source_date": today, + } + ) + log.info("S&P 500 candidates from Wikipedia: %d", len(df)) + return df[CANDIDATE_COLUMNS] + + +# ---------------------------------------------------------------------------------------- +# Europe: STOXX Europe 600 from the iShares ETF holdings file +# ---------------------------------------------------------------------------------------- +def parse_ishares_holdings(text: str, source_date: str | None = None) -> pd.DataFrame: + """Parse the German iShares holdings CSV (``Fondsposition per ...`` header, ``;``-free).""" + lines = text.lstrip("").splitlines() + if source_date is None: + first = lines[0] if lines else "" + raw_date = first.split(",", 1)[1].strip().strip('"') if "," in first else "" + source_date = _parse_german_date(raw_date) or date.today().isoformat() + try: + header_idx = next(i for i, line in enumerate(lines) if line.startswith("Emittententicker")) + except StopIteration as exc: + raise ValueError("iShares holdings file: header line not found") from exc + body = "\n".join(lines[header_idx:]) + raw = pd.read_csv(io.StringIO(body), sep=",", quotechar='"', thousands=".", decimal=",") + eq = raw[raw["Anlageklasse"] == "Aktien"].copy() + eq = eq[eq["Emittententicker"].notna() & (eq["Emittententicker"].astype(str).str.strip() != "-")] + + def _symbol(row: pd.Series) -> str: + try: + return to_yahoo_symbol(str(row["Emittententicker"]), exchange=str(row["Börse"])) + except ValueError: + country = ISHARES_COUNTRY_EN.get(str(row["Standort"]), str(row["Standort"])) + return to_yahoo_symbol(str(row["Emittententicker"]), country=country) + + clean = lambda v: str(v).replace("\xa0", " ").strip() # noqa: E731 + df = pd.DataFrame( + { + "symbol": eq.apply(_symbol, axis=1), + "source_ticker": eq["Emittententicker"].astype(str).map(clean), + "name": eq["Name"].astype(str).map(clean).str.title(), + "region": "Europe", + "country": eq["Standort"].map(lambda c: ISHARES_COUNTRY_EN.get(clean(c), clean(c))), + "exchange": eq["Börse"].astype(str).map(clean), + "listing_currency": eq["Marktwährung"].astype(str).map(clean), + "gics_sector": eq["Sektor"].map(lambda s: ISHARES_SECTOR_EN.get(clean(s), clean(s))), + "gics_sub_industry": "", + "index_weight": pd.to_numeric(eq["Gewichtung (%)"], errors="coerce"), + "source": "ishares_stoxx600", + "source_date": source_date, + } + ) + df = df.drop_duplicates(subset="symbol").reset_index(drop=True) + return df[CANDIDATE_COLUMNS] + + +def fetch_stoxx600_ishares(timeout: float = 30.0) -> pd.DataFrame: + """STOXX Europe 600 constituents from the iShares ETF holdings (complete, with weights).""" + raw = _get(ISHARES_STOXX600_URL, timeout).decode("utf-8-sig", errors="replace") + df = parse_ishares_holdings(raw) + log.info("STOXX 600 candidates from iShares: %d", len(df)) + return df + + +def fetch_stoxx600_wikipedia(timeout: float = 30.0) -> pd.DataFrame: + """Partial STOXX 600 list from Wikipedia (fallback; no exchange, so the suffix is by country).""" + html = _get(STOXX600_WIKI_URL, timeout) + tables = pd.read_html(io.BytesIO(html)) + table = next(t for t in tables if "Ticker" in t.columns and "Country" in t.columns) + today = date.today().isoformat() + rows = [] + for _, r in table.iterrows(): + try: + symbol = to_yahoo_symbol(str(r["Ticker"]), country=str(r["Country"])) + except ValueError: + continue + rows.append( + { + "symbol": symbol, + "source_ticker": str(r["Ticker"]), + "name": str(r["Company"]), + "region": "Europe", + "country": str(r["Country"]), + "exchange": "", + "listing_currency": "", + "gics_sector": "", + "gics_sub_industry": str(r.get("ICB Sector", "")), + "index_weight": float("nan"), + "source": "wikipedia_stoxx600", + "source_date": today, + } + ) + df = pd.DataFrame(rows, columns=CANDIDATE_COLUMNS) + log.info("STOXX 600 candidates from Wikipedia: %d", len(df)) + return df + + +# ---------------------------------------------------------------------------------------- +# Static snapshots shipped with the package +# ---------------------------------------------------------------------------------------- +def _static_path(region: str) -> Path: + name = "candidates_us.csv" if region.upper() == "US" else "candidates_europe.csv" + return Path(str(resources.files("quantcore.data").joinpath(f"static/{name}"))) + + +def load_static_candidates(region: str) -> pd.DataFrame: + """Candidates snapshotted into the package (offline fallback).""" + path = _static_path(region) + if not path.exists(): + log.warning("No static candidate file for region %s (%s)", region, path) + return _empty() + df = pd.read_csv(path, dtype={"symbol": str, "source_ticker": str}) + log.info("Static %s candidates: %d (%s)", region, len(df), path.name) + return df[CANDIDATE_COLUMNS] + + +def save_static_candidates(df: pd.DataFrame, region: str, path: Path | None = None) -> Path: + """Refresh the shipped snapshot (called by the setup after a successful web fetch).""" + path = path or _static_path(region) + path.parent.mkdir(parents=True, exist_ok=True) + df[CANDIDATE_COLUMNS].to_csv(path, index=False) + return path + + +def fetch_candidates(region: str, allow_web: bool = True, timeout: float = 30.0) -> pd.DataFrame: + """Best-effort candidate list: web sources first, then the static snapshot. + + Every web source is wrapped: a failing source is logged and the next one is tried + (timeout, retry and fallback -- the resilience the platform requires on external calls). + """ + region = "US" if region.upper() == "US" else "Europe" + fetchers = [fetch_sp500_wikipedia] if region == "US" else [fetch_stoxx600_ishares, fetch_stoxx600_wikipedia] + if allow_web: + for fetch in fetchers: + try: + df = fetch(timeout) + except Exception as exc: # noqa: BLE001 - any failure -> next source + log.warning("Candidate source %s failed: %s", fetch.__name__, exc) + continue + if len(df) >= 100: + if region == "Europe": + df = df[df["country"].isin(EUROPE_COUNTRIES)].reset_index(drop=True) + try: + save_static_candidates(df, region) + except OSError as exc: # read-only install + log.debug("Could not refresh static snapshot: %s", exc) + return df + return load_static_candidates(region) diff --git a/quantcore/universe/symbols.py b/quantcore/universe/symbols.py new file mode 100644 index 0000000..4e85b56 --- /dev/null +++ b/quantcore/universe/symbols.py @@ -0,0 +1,129 @@ +"""Mapping of exchange-local ticker codes to Yahoo Finance symbols. + +Yahoo identifies a listing by ```` where the suffix encodes the exchange +(``.SW`` SIX, ``.L`` London, ``.DE`` Xetra, ...). Share classes are written with a dash +(``NOVO-B.CO``, ``INVE-B.ST``, ``BRK-B``), and London codes lose their trailing dot +(``RR.`` -> ``RR.L``, ``BT.A`` -> ``BT-A.L``). + +Known exceptions that no rule catches live in ``data/static/symbol_overrides.json`` and are +applied last. +""" + +from __future__ import annotations + +import json +import re +from functools import lru_cache +from importlib import resources + +# iShares exchange names (STOXX 600 holdings file) -> Yahoo suffix +EXCHANGE_SUFFIX: dict[str, str] = { + "London Stock Exchange": ".L", + "Xetra": ".DE", + "Deutsche Börse AG": ".DE", + "Deutsche Boerse AG": ".DE", + "Nyse Euronext - Euronext Paris": ".PA", + "Euronext Paris": ".PA", + "SIX Swiss Exchange": ".SW", + "Nasdaq Omx Nordic": ".ST", # Stockholm + "Nasdaq Stockholm": ".ST", + "Borsa Italiana": ".MI", + "Euronext Amsterdam": ".AS", + "Bolsa De Madrid": ".MC", + "Oslo Bors Asa": ".OL", + "Omx Nordic Exchange Copenhagen A/S": ".CO", + "Nasdaq Omx Helsinki Ltd.": ".HE", + "Warsaw Stock Exchange/Equities/Main Market": ".WA", + "Nyse Euronext - Euronext Brussels": ".BR", + "Wiener Boerse Ag": ".VI", + "Athens Exchange S.A. Cash Market": ".AT", + "Irish Stock Exchange - All Market": ".IR", + "Nyse Euronext - Euronext Lisbon": ".LS", + "Prague Stock Exchange": ".PR", + "Budapest Stock Exchange": ".BD", +} + +# Country of an exchange (used when a source only gives the country of the listing) +COUNTRY_SUFFIX: dict[str, str] = { + "United Kingdom": ".L", + "Germany": ".DE", + "France": ".PA", + "Switzerland": ".SW", + "Sweden": ".ST", + "Italy": ".MI", + "Netherlands": ".AS", + "Spain": ".MC", + "Norway": ".OL", + "Denmark": ".CO", + "Finland": ".HE", + "Poland": ".WA", + "Belgium": ".BR", + "Austria": ".VI", + "Greece": ".AT", + "Ireland": ".IR", + "Portugal": ".LS", + "Czech Republic": ".PR", + "Hungary": ".BD", + "Luxembourg": ".PA", + "United States": "", +} + + +@lru_cache(maxsize=1) +def symbol_overrides() -> dict[str, str]: + """Hand-maintained corrections ``{rule_based_symbol: actual_yahoo_symbol}``.""" + try: + text = resources.files("quantcore.data").joinpath("static/symbol_overrides.json").read_text() + except FileNotFoundError: + return {} + data = json.loads(text) + return {k: v for k, v in data.items() if not k.startswith("_")} + + +def _clean_code(code: str) -> str: + code = code.strip().upper() + code = re.sub(r"\s+", "-", code) # "NOVO B" -> "NOVO-B" + code = code.rstrip(".") # "RR." -> "RR" + code = code.replace(".", "-") # "BT.A" -> "BT-A" + return code + + +def to_yahoo_symbol(source_ticker: str, exchange: str | None = None, country: str | None = None) -> str: + """Build the Yahoo symbol for an exchange-local ticker code. + + >>> to_yahoo_symbol("NOVO B", "Omx Nordic Exchange Copenhagen A/S") + 'NOVO-B.CO' + >>> to_yahoo_symbol("BT.A", "London Stock Exchange") + 'BT-A.L' + >>> to_yahoo_symbol("BRK.B", country="United States") + 'BRK-B' + """ + code = _clean_code(source_ticker) + suffix = None + if exchange is not None: + suffix = EXCHANGE_SUFFIX.get(exchange.strip()) + if suffix is None and country is not None: + suffix = COUNTRY_SUFFIX.get(country.strip()) + if suffix is None: + raise ValueError(f"Unknown exchange/country for ticker {source_ticker!r}: {exchange!r}/{country!r}") + symbol = f"{code}{suffix}" + return symbol_overrides().get(symbol, symbol) + + +def normalise_share_class_name(name: str) -> str: + """Reduce a company name to a key that is equal for its share classes. + + Used to keep only one listing per company (``Alphabet A`` / ``Alphabet C``, + ``Investor A`` / ``Investor B``). The heuristic strips legal forms and class markers. + """ + s = name.upper() + s = re.sub(r"[^A-Z0-9 ]", " ", s) + tokens = s.split() + drop = { + "INC", "CORP", "CORPORATION", "PLC", "AG", "SA", "SE", "NV", "N", "V", "AB", "ASA", "AS", "SPA", + "OYJ", "LTD", "LIMITED", "CO", "COMPANY", "GROUP", "HOLDING", "HOLDINGS", "THE", "CLASS", "CL", + "SER", "SERIES", "SHS", "SHARE", "SHARES", "ORD", "REG", "REGISTERED", "BEARER", "PS", "GS", + "A", "B", "C", "N", "R", "NEW", "KGAA", "SDB", "DI", "NPV", "ADR", "OY", "ORDINARY", + } + kept = [t for t in tokens if t not in drop] + return " ".join(kept[:3]) if kept else s.strip() diff --git a/scripts/experiments/README.md b/scripts/experiments/README.md new file mode 100644 index 0000000..57402e6 --- /dev/null +++ b/scripts/experiments/README.md @@ -0,0 +1,16 @@ +# Experiments behind the scenario-sampling section + +These scripts produced the tables in `docs/optimization_model.pdf`, Section "Scenario sets: +sampling error, variance reduction and large sets". They run against the local data store +(`quantcore setup` first) and print their tables; they are not part of the package or the tests. + +| script | what it measures | +|---|---| +| `scenario_noise.py N S SEEDS [cont\|int] [kinds] [beta]` | GBM model: plain MC vs antithetic / moment matching / LHS / Sobol' at equal `S`, continuous (weights) or integer problem, against the exact Gaussian optimum (cutting planes) — Table "Sampling noise" | +| `empirical_noise.py N S SEEDS BETA [kinds]` | empirical-innovation model: bootstrap vs moment-matched / balanced / rank-stratified resampling, evaluated on a 500 000-draw set | +| `is_noise.py N SIZES SEEDS BETAS` | importance sampling (two-stage tilt, defensive mixture) vs moment-matched Sobol' — Table "Importance sampling" | +| `scaling.py N SIZES highs\|cuopt` | solve time versus the number of scenarios (HiGHS locally / cuOpt on the Spark) — Table "Solve times" | +| `rowgen.py N SIZES [full]` | tail-row generation versus the full MILP | +| `gurobi_ref.py N BETA` | the exact Gaussian MISOCP solved directly with Gurobi (needs `gurobipy` and a licence large enough for `N`) — validation of the cutting-plane reference | + +Example: `python scripts/experiments/scenario_noise.py 30 2048 8 cont mc,antithetic,mc+mm,sobol,sobol+mm 0.95` diff --git a/scripts/experiments/empirical_noise.py b/scripts/experiments/empirical_noise.py new file mode 100644 index 0000000..4544111 --- /dev/null +++ b/scripts/experiments/empirical_noise.py @@ -0,0 +1,136 @@ +"""Sampling noise of the mean-CVaR LP (weights) under the empirical-innovation model with different resampling schemes. + +'truth' = a common evaluation set of 500k plain bootstrap draws (sampling error of CVaR for a fixed portfolio ~0.1%). +""" +import sys, time +import numpy as np, pandas as pd +from scipy.stats import qmc + +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.optimization.service import OptimizationService, OptimizationRequest +from quantcore.optimization.problem import MeanCVaRProblem, ProblemSpec, OptimizationConstraints, TransactionCosts +from quantcore.optimization.solvers import HighsSolver +from quantcore.models.covariance import CovarianceModel, symmetric_sqrt, symmetric_inv_sqrt +from quantcore.models.simulation import EmpiricalInnovationSimulator +from quantcore.measurement.risk import scenario_cvar + +N_ASSETS = int(sys.argv[1]) if len(sys.argv) > 1 else 30 +S = int(sys.argv[2]) if len(sys.argv) > 2 else 2048 +N_SEEDS = int(sys.argv[3]) if len(sys.argv) > 3 else 8 +BETA = float(sys.argv[4]) if len(sys.argv) > 4 else 0.95 +W, H = 1_000_000.0, 5 +LAMBDA = 0.05 * W +S_EVAL = 500_000 + +settings = get_settings(); md = MarketDataService(settings); svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, solver="highs", cash=W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_model = CovarianceModel.from_dict(req.covariance) +sim = EmpiricalInnovationSimulator(cov_model, max_history=req.innovation_history, standardize=req.innovation_standardize) +t0 = time.time() +eps = sim.innovations(r) +e = eps.values - eps.values.mean(axis=0) +e, info = sim._restandardize(e) +T, N = e.shape +cov = cov_model.estimate(r).values +A = symmetric_sqrt(cov) +mu_h = (mu_ann / 252).reindex(symbols).values * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown").tolist() +print(f"N={N} assets, T={T} innovations ({time.time()-t0:.0f}s), S={S}, h={H}, beta={BETA}, {info}") +# market-factor ordering of the innovation dates (projection on the first eigenvector of Sigma_T) +w1 = np.linalg.eigh(cov)[1][:, -1] +factor = e @ (A @ w1) # innovation of the dominant direction +rank_order = np.argsort(factor) # dates sorted from worst to best market innovation + + +def moment_match(x, target_var): + x = x - x.mean(axis=0) + c = x.T @ x / len(x) + return x @ symmetric_inv_sqrt(c) * np.sqrt(target_var) + + +def summed_innovations(kind, seed): + rng = np.random.default_rng(seed) + if kind == "bootstrap": # current implementation + idx = rng.integers(0, T, size=(S, H)) + return e[idx].sum(axis=1) + if kind == "bootstrap+mm": + idx = rng.integers(0, T, size=(S, H)) + return moment_match(e[idx].sum(axis=1), H) + if kind == "balanced+mm": # every date used equally often + reps = int(np.ceil(S * H / T)) + idx = rng.permutation(np.tile(np.arange(T), reps))[: S * H].reshape(S, H) + return moment_match(e[idx].sum(axis=1), H) + if kind == "lhs-ranks+mm": # stratified on the market-factor rank, per day + u = qmc.LatinHypercube(d=H, scramble=True, seed=seed).random(S) + idx = rank_order[np.minimum((u * T).astype(int), T - 1)] + return moment_match(e[idx].sum(axis=1), H) + if kind == "sobol-ranks+mm": + u = qmc.Sobol(d=H, scramble=True, seed=seed).random(S) + idx = rank_order[np.minimum((u * T).astype(int), T - 1)] + return moment_match(e[idx].sum(axis=1), H) + if kind == "windows+mm": # deterministic: all overlapping h-day windows + idx = np.arange(T - H + 1)[:, None] + np.arange(H)[None, :] + return moment_match(e[idx].sum(axis=1), H) + if kind == "windows": + idx = np.arange(T - H + 1)[:, None] + np.arange(H)[None, :] + return e[idx].sum(axis=1) + raise ValueError(kind) + + +def scenarios(kind, seed): + return mu_h + summed_innovations(kind, seed) @ A.T + + +def solve_lp(scen): + spec = ProblemSpec(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=W, holdings=np.zeros(N), + confidence=BETA, cvar_limit=LAMBDA, objective="max_return", constraints=OptimizationConstraints(), + costs=TransactionCosts(proportional_buy=0.0, proportional_sell=0.0)) + prob = MeanCVaRProblem(spec) + model = prob.build() + model.var_types = ["C"] * len(model.var_types) + sol = HighsSolver().solve(model, time_limit=300) + n = sol.x[prob.i_n] + pnl = scen @ (n * prices) + return n, float(pnl.mean()), scenario_cvar(-pnl, BETA)[1] + + +# common evaluation set (plain bootstrap, independent seed) +rng_eval = np.random.default_rng(987654321) +eval_scen = mu_h + e[rng_eval.integers(0, T, size=(S_EVAL, H))].sum(axis=1) @ A.T +print(f"evaluation set: {S_EVAL} draws, mean of column means {eval_scen.mean():.2e}") + + +def true_stats(n): + pnl = eval_scen @ (n * prices) + return float(pnl.mean()), scenario_cvar(-pnl, BETA)[1] + + +results = {} +KINDS = sys.argv[5].split(",") if len(sys.argv) > 5 else ["bootstrap", "bootstrap+mm", "balanced+mm", "lhs-ranks+mm", "sobol-ranks+mm"] +for kind in KINDS: + rows, t0 = [], time.time() + for seed in range(1 if kind.startswith("windows") else N_SEEDS): + n, epnl_in, cvar_in = solve_lp(scenarios(kind, seed)) + epnl_t, cvar_t = true_stats(n) + rows.append({"epnl_in": epnl_in, "cvar_in": cvar_in, "epnl_true": epnl_t, "cvar_true": cvar_t, "n_pos": int((n > 1e-6).sum()), "w": n * prices / W}) + df = pd.DataFrame(rows) + ws = np.vstack(df["w"].values) + pair = [np.abs(ws[i] - ws[j]).sum() for i in range(len(ws)) for j in range(i + 1, len(ws))] + results[kind] = { + "E[P&L] in-sample mean": df.epnl_in.mean(), "E[P&L] in-sample sd": df.epnl_in.std(), + "E[P&L] true mean": df.epnl_true.mean(), "E[P&L] true sd": df.epnl_true.std(), + "true CVaR mean": df.cvar_true.mean(), "true CVaR sd": df.cvar_true.std(), + "CVaR excess over limit (%)": 100 * (df.cvar_true.mean() / LAMBDA - 1), + "positions": df.n_pos.mean(), "turnover between seeds (%)": 100 * np.mean(pair) if pair else float("nan"), "seconds": time.time() - t0, + } + print(f"{kind:15s} done in {time.time()-t0:.0f}s") + +out = pd.DataFrame(results).T +pd.set_option("display.width", 250); pd.set_option("display.float_format", lambda v: f"{v:10.2f}") +print(f"\nEMPIRICAL model, continuous (weights) problem, CVaR limit {LAMBDA:.0f} CHF, beta={BETA}") +print(out.T.to_string()) diff --git a/scripts/experiments/gurobi_ref.py b/scripts/experiments/gurobi_ref.py new file mode 100644 index 0000000..b524de2 --- /dev/null +++ b/scripts/experiments/gurobi_ref.py @@ -0,0 +1,46 @@ +"""Exact Gaussian mean-CVaR problem in integer shares as a MISOCP with Gurobi (validation of the cutting planes).""" +import sys, time +import numpy as np, gurobipy as gp +from scipy.stats import norm +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.optimization.service import OptimizationService, OptimizationRequest +from quantcore.optimization.problem import OptimizationConstraints +from quantcore.models.covariance import CovarianceModel + +N_ASSETS = int(sys.argv[1]) if len(sys.argv) > 1 else 30 +BETA = float(sys.argv[2]) if len(sys.argv) > 2 else 0.95 +W, H = 1_000_000.0, 5 +LAMBDA = 0.05 * W +settings = get_settings(); md = MarketDataService(settings); svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, scenario_model="gbm", solver="highs", cash=W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_h = CovarianceModel.from_dict(req.covariance).estimate(r).values * H +mu_h = (mu_ann / 252).reindex(symbols).values * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown").tolist() +N = len(symbols); k = norm.pdf(norm.ppf(BETA)) / (1 - BETA) +L = np.linalg.cholesky(cov_h) + +for integer in (True, False): + t0 = time.time() + m = gp.Model(); m.Params.OutputFlag = 0; m.Params.MIPGap = 1e-9; m.Params.MIPGapAbs = 1e-6 + ub = np.floor(0.10 * W / prices) + n = m.addMVar(N, vtype="I" if integer else "C", lb=0, ub=ub) + t = m.addVar(lb=0) + y = m.addMVar(N, lb=-gp.GRB.INFINITY) # y = L' v -> ||y|| = sqrt(v' Sigma v) + v = prices * n + m.addConstr(y == L.T @ v) + m.addConstr(y @ y <= t * t) # second-order cone + m.addConstr(k * t <= LAMBDA + mu_h @ v) # Gaussian CVaR <= Lambda + m.addConstr(prices @ n <= W) + for sec in sorted(set(sectors)): + mask = np.array([s == sec for s in sectors], dtype=float) + m.addConstr((prices * mask) @ n <= OptimizationConstraints().max_sector_weight * W) + m.setObjective(mu_h @ v, gp.GRB.MAXIMIZE) + m.optimize() + nn = n.X; vv = prices * nn; sd = np.sqrt(vv @ cov_h @ vv) + print(f"{'integer' if integer else 'continuous':10s} N={N} beta={BETA}: status {m.Status}, E[P&L]={mu_h @ vv:.2f} CHF, " + f"CVaR={-mu_h @ vv + k * sd:.2f}, positions={(nn > 0.5).sum()}, {time.time()-t0:.1f}s") diff --git a/scripts/experiments/is_noise.py b/scripts/experiments/is_noise.py new file mode 100644 index 0000000..085e3fd --- /dev/null +++ b/scripts/experiments/is_noise.py @@ -0,0 +1,173 @@ +"""Importance sampling for the mean-CVaR LP (weights): two-stage exponential tilt along the loss direction of a +first-pass solution, defensive mixture with the nominal density, self-normalised weights, weighted moment matching, +exact mean in the objective. Compared with Sobol+mm at equal S against the exact Gaussian optimum.""" +import sys, time +import numpy as np, pandas as pd +import scipy.sparse as sps +from scipy.optimize import milp, LinearConstraint, Bounds +from scipy.stats import norm, qmc +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.optimization.service import OptimizationService, OptimizationRequest +from quantcore.optimization.problem import OptimizationConstraints +from quantcore.models.covariance import CovarianceModel, symmetric_inv_sqrt + +N_ASSETS = int(sys.argv[1]); SIZES = [int(x) for x in sys.argv[2].split(",")]; N_SEEDS = int(sys.argv[3]) +BETAS = [float(x) for x in sys.argv[4].split(",")] +W = 1_000_000.0; LAMBDA = 0.05 * W; H = 5; WMAX = 0.10; SMAX = 0.25 +settings = get_settings(); md = MarketDataService(settings); svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, solver="highs", cash=W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_h = CovarianceModel.from_dict(req.covariance).estimate(r).values * H +mu_h = (mu_ann / 252).reindex(symbols).values * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown").tolist() +N = len(symbols) +vals, vecs = np.linalg.eigh(cov_h); o = np.argsort(vals)[::-1]; A = vecs[:, o] * np.sqrt(vals[o]) +sector_rows = [(np.array([s == sec for s in sectors], dtype=float)) for sec in sorted(set(sectors))] + + +def k_of(beta): + return norm.pdf(norm.ppf(beta)) / (1 - beta) + + +def true_stats(n, beta): + v = n * prices; mean = v @ mu_h; sd = np.sqrt(v @ cov_h @ v) + return mean, -mean + k_of(beta) * sd, sd + + +def weighted_mm(z, p): + m = p @ z + zc = z - m + c = (zc * p[:, None]).T @ zc + return zc @ symmetric_inv_sqrt(c) + + +def solve_lp(scen, p, beta): + """max mu_h'(n*P) s.t. RU-CVaR with scenario probabilities p <= Lambda, budget, caps, sectors; n continuous.""" + S = len(scen) + nv = N + 1 + S + c = np.zeros(nv); c[:N] = -(prices * mu_h) # exact model mean in the objective + G = scen * prices[None, :] + rows = [np.repeat(np.arange(S), N), np.arange(S), np.arange(S)] + cols = [np.tile(np.arange(N), S), np.full(S, N), N + 1 + np.arange(S)] + vals_ = [G.ravel(), np.ones(S), np.ones(S)] + lb = [np.zeros(S)]; ub = [np.full(S, np.inf)] + ri = S + rows.append(np.concatenate([[ri], np.full(S, ri)])); cols.append(np.concatenate([[N], N + 1 + np.arange(S)])) + vals_.append(np.concatenate([[1.0], p / (1 - beta)])); lb.append([-np.inf]); ub.append([LAMBDA]); ri += 1 + rows.append(np.full(N, ri)); cols.append(np.arange(N)); vals_.append(prices); lb.append([-np.inf]); ub.append([W]); ri += 1 + for mask in sector_rows: + rows.append(np.full(N, ri)); cols.append(np.arange(N)); vals_.append(prices * mask); lb.append([-np.inf]); ub.append([SMAX * W]); ri += 1 + Am = sps.csr_matrix((np.concatenate(vals_), (np.concatenate(rows), np.concatenate(cols))), shape=(ri, nv)) + vlb = np.zeros(nv); vub = np.full(nv, np.inf); vub[:N] = WMAX * W / prices; vlb[N] = -np.inf + res = milp(c, constraints=LinearConstraint(Am, np.concatenate(lb), np.concatenate(ub)), integrality=np.zeros(nv), + bounds=Bounds(vlb, vub)) + n = res.x[:N]; alpha = res.x[N] + return n, alpha + + +def scenarios_sobol(S, seed): + z = norm.ppf(qmc.Sobol(d=N, scramble=True, seed=seed).random(S)) + p = np.full(S, 1.0 / S) + return mu_h + weighted_mm(z, p) @ A.T, p + + +def scenarios_is(S, seed, beta, n1, eps=0.5, shift=None): + """Mixture proposal: first eps*S Sobol points shifted by theta along the loss direction of n1, rest nominal.""" + v = n1 * prices + a = A.T @ v # loss = -v'mu - a'Z + d = -a / np.linalg.norm(a) # unit direction of increasing loss in Z-space + theta = (shift if shift is not None else norm.ppf(beta)) * d + z = norm.ppf(qmc.Sobol(d=N, scramble=True, seed=seed).random(S)) + m = int(round(eps * S)) + z[:m] += theta + # likelihood ratio f/g with g = (1-eps) f + eps f(.-theta): 1 / ((1-eps) + eps exp(theta'z - |theta|^2/2)) + lr = 1.0 / ((1 - eps) + eps * np.exp(z @ theta - theta @ theta / 2)) + p = lr / lr.sum() + ess = 1.0 / (p ** 2).sum() + return mu_h + weighted_mm(z, p) @ A.T, p, ess + + +def scenarios_is_multi(S, seed, n1, shifts=(1.6449, 2.3263), eps=(0.25, 0.25)): + """Multi-component defensive mixture: nominal + one shifted block per shift, all along the loss direction of n1.""" + v = n1 * prices + a = A.T @ v + d = -a / np.linalg.norm(a) + z = norm.ppf(qmc.Sobol(d=N, scramble=True, seed=seed).random(S)) + thetas = [sh * d for sh in shifts] + start = 0 + for th, e in zip(thetas, eps): + m = int(round(e * S)); z[start:start + m] += th; start += m + dens = (1 - sum(eps)) * np.ones(S) + for th, e in zip(thetas, eps): + dens = dens + e * np.exp(z @ th - th @ th / 2) + lr = 1.0 / dens + p = lr / lr.sum() + return mu_h + weighted_mm(z, p) @ A.T, p, 1.0 / (p ** 2).sum() + + +def exact_reference(beta, tol=1e-7): + k = k_of(beta) + cuts = [] + ub = WMAX * W / prices + c = -(prices * mu_h) + base_rows = [prices] + [prices * m for m in sector_rows] + base_ub = [W] + [SMAX * W] * len(sector_rows) + for it in range(500): + Am = np.vstack(base_rows + [g for g, _ in cuts]); hi = np.array(base_ub + [b for _, b in cuts]) + res = milp(c, constraints=LinearConstraint(Am, np.full(len(hi), -np.inf), hi), integrality=np.zeros(N), bounds=Bounds(np.zeros(N), ub)) + x = res.x; mean, cvar, sd = true_stats(x, beta) + if cvar <= LAMBDA * (1 + tol): + return x, mean + v = x * prices + grad = -prices * mu_h + k * prices * (cov_h @ v) / sd + cuts.append((grad, LAMBDA - cvar + grad @ x)) + raise RuntimeError + + +market_w = md.market_weights(symbols).reindex(symbols).fillna(0.0).values +market_n = market_w * W / prices # cap-weighted candidate portfolio (continuous shares) +KINDS = sys.argv[5].split(",") if len(sys.argv) > 5 else ["sobol+mm", "sobol+mm+IS(eps=.5)", "IS-market(z_b)", "IS-market(2.33)"] +pd.set_option("display.width", 250); pd.set_option("display.float_format", lambda v: f"{v:10.2f}") +for beta in BETAS: + x_ref, mean_ref = exact_reference(beta) + print(f"\n===== beta={beta}: exact optimum E[P&L]={mean_ref:.2f} CHF, CVaR limit {LAMBDA:.0f}") + for S in SIZES: + results = {} + for kind in KINDS: + rows = [] + t0 = time.time() + for seed in range(N_SEEDS): + if kind == "sobol+mm": + scen, p = scenarios_sobol(S, seed); ess = S + n, alpha = solve_lp(scen, p, beta) + elif kind == "IS-market(mix)": + scen, p, ess = scenarios_is_multi(S, seed + 1000, market_n) + n, alpha = solve_lp(scen, p, beta) + elif kind.startswith("IS-market"): + shift = norm.ppf(beta) if "z_b" in kind else 2.33 + scen, p, ess = scenarios_is(S, seed + 1000, beta, market_n, eps=0.5, shift=shift) + n, alpha = solve_lp(scen, p, beta) + else: + eps = 0.5 if ".5" in kind else 0.3 + scen1, p1 = scenarios_sobol(S, seed) # stage 1: direction from a plain solve + n1, _ = solve_lp(scen1, p1, beta) + scen, p, ess = scenarios_is(S, seed + 1000, beta, n1, eps=eps) + n, alpha = solve_lp(scen, p, beta) + mean_t, cvar_t, _ = true_stats(n, beta) + L = -(scen * prices) @ n + rows.append({"epnl_true": mean_t, "cvar_true": cvar_t, "w": n * prices / W, "ess": ess, + "tail_pts": int((L > alpha).sum())}) + df = pd.DataFrame(rows); ws = np.vstack(df["w"].values) + pair = [np.abs(ws[i] - ws[j]).sum() for i in range(len(ws)) for j in range(i + 1, len(ws))] + results[kind] = { + "true E[P&L] mean": df.epnl_true.mean(), "loss vs optimum (CHF)": mean_ref - df.epnl_true.mean(), + "true CVaR excess (%)": 100 * (df.cvar_true.mean() / LAMBDA - 1), "true CVaR sd (% of limit)": 100 * df.cvar_true.std() / LAMBDA, + "turnover between seeds (%)": 100 * np.mean(pair), "turnover to optimum (%)": 100 * np.mean(np.abs(ws - x_ref * prices / W).sum(axis=1)), + "scenarios beyond VaR": df.tail_pts.mean(), "ESS": df.ess.mean(), "seconds": time.time() - t0, + } + print(f"--- S={S}") + print(pd.DataFrame(results).to_string()) diff --git a/scripts/experiments/rank_and_size.py b/scripts/experiments/rank_and_size.py new file mode 100644 index 0000000..4089104 --- /dev/null +++ b/scripts/experiments/rank_and_size.py @@ -0,0 +1,125 @@ +"""Two questions about large candidate sets. + +A. Conditioning: with ``N`` stocks and an estimator whose effective window is ``T_eff < N`` the + covariance is (numerically) singular. Does the optimiser exploit the near-null directions -- + does it build a portfolio that looks riskless in sample and is not? + +B. How many scenarios does a problem with ``N`` assets need? Measured against the exact Gaussian + optimum (no scenarios) for several ``N`` and ``S``. + +Usage: python scripts/experiments/rank_and_size.py [A|B] [sizes] +""" +import sys +import time + +import numpy as np +import pandas as pd + +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.measurement.estimators import effective_observations +from quantcore.models.covariance import CovarianceModel +from quantcore.optimization.service import OptimizationRequest, OptimizationService + +PART = sys.argv[1] if len(sys.argv) > 1 else "A" +SIZES = [int(x) for x in sys.argv[2].split(",")] if len(sys.argv) > 2 else [30, 100, 300, 745] +W, H, BETA = 1_000_000.0, 5, 0.95 + +settings = get_settings() +md = MarketDataService(settings) +svc = OptimizationService(settings, md) +caps = md.market_caps().sort_values(ascending=False) +pd.set_option("display.width", 240) + + +def universe(n): + return caps.index[:n].tolist() + + +# ============================================================================================= +if PART == "A": + rows = [] + for n in SIZES: + symbols = universe(n) + r = md.returns(symbols, currency="base", freq="daily").dropna() + for label, model in (("ewm l=0.99", CovarianceModel("ewm", lam=0.99, shrinkage=0.0)), + ("ewm l=0.99 + auto", CovarianceModel("ewm", lam=0.99, shrinkage="auto")), + ("rolling 200", CovarianceModel("rolling", window=200, shrinkage=0.0)), + ("rolling 200 + auto", CovarianceModel("rolling", window=200, shrinkage="auto"))): + t0 = time.time() + cov = model.estimate(r).values + vals = np.linalg.eigvalsh(cov) + t_eff = effective_observations(model.method, model.window, model.lam) + scale = np.diag(cov).mean() + rows.append({ + "N": n, "estimator": label, "T_eff": round(t_eff), + "shrinkage": round(model.intensity(r), 3), + "min eig / mean var": vals.min() / scale, "cond": vals.max() / max(vals.min(), 1e-300), + "eigs < 1e-6 of max": int((vals < 1e-6 * vals.max()).sum()), + "s": round(time.time() - t0, 1), + }) + print("A1. conditioning of the covariance estimate\n") + print(pd.DataFrame(rows).to_string(index=False, float_format=lambda v: f"{v:.3g}")) + + # does a singular estimate let the optimiser build a 'riskless' portfolio? + print("\nA2. minimum-variance portfolio under the caps: in-sample vs out-of-sample volatility") + out = [] + for n in SIZES: + symbols = universe(n) + r = md.returns(symbols, currency="base", freq="daily").dropna() + truth = CovarianceModel("rolling", window=len(r), shrinkage=0.0).estimate(r).values # all history + for label, model in (("ewm l=0.99", CovarianceModel("ewm", lam=0.99, shrinkage=0.0)), + ("ewm l=0.99 + auto", CovarianceModel("ewm", lam=0.99, shrinkage="auto"))): + cov = model.estimate(r).values + # long-only, fully invested, at most 10 % per name: minimise w'Cw by projected gradient + w = np.full(n, 1.0 / n) + cap = max(0.10, 1.5 / n) + for _ in range(4000): + g = cov @ w + w = w - g / (np.abs(g).max() + 1e-18) * 0.01 + w = np.clip(w, 0, cap) + if w.sum() <= 0: + w = np.full(n, 1.0 / n) + w = w / w.sum() + vol_in = np.sqrt(w @ cov @ w * 252) * 100 + vol_out = np.sqrt(w @ truth @ w * 252) * 100 + out.append({"N": n, "estimator": label, "in-sample vol (%)": vol_in, + "long-history vol (%)": vol_out, "understated by": f"{vol_out / vol_in:.2f}x", + "positions > 0.1 %": int((w > 0.001).sum())}) + print(pd.DataFrame(out).to_string(index=False, float_format=lambda v: f"{v:.2f}")) + +# ============================================================================================= +if PART == "B": + scen_sizes = [int(x) for x in sys.argv[3].split(",")] if len(sys.argv) > 3 else [512, 2048, 8192] + seeds = int(sys.argv[4]) if len(sys.argv) > 4 else 6 + solver = sys.argv[5] if len(sys.argv) > 5 else "highs" + rows = [] + for n in SIZES: + symbols = universe(n) + base = dict(symbols=symbols, cash=W, horizon="week", cvar_limit=0.05 * W, scenario_model="gbm", + solver=solver, confidence=BETA, costs={"proportional_buy": 0.0, "proportional_sell": 0.0}) + t0 = time.time() + exact = svc.optimize(OptimizationRequest(**{**base, "scenario_model": "gaussian_exact", "solver": "highs"})) + t_exact = time.time() - t0 + ref_w = {p["symbol"]: p["weight"] for p in exact.positions} + for S in scen_sizes: + ws, pnls = [], [] + t1 = time.time() + for seed in range(seeds): + res = svc.optimize(OptimizationRequest(**{**base, "n_scenarios": S, "seed": seed})) + if res.status != "Optimal": + continue + ws.append({p["symbol"]: p["weight"] for p in res.positions}) + pnls.append(res.expected_pnl) + def turn(a, b): + keys = set(a) | set(b) + return sum(abs(a.get(k, 0) - b.get(k, 0)) for k in keys) + pair = [turn(ws[i], ws[j]) for i in range(len(ws)) for j in range(i + 1, len(ws))] + rows.append({"N": n, "S": S, "S/N": S / n, + "turnover between seeds (%)": 100 * np.mean(pair), + "turnover to exact (%)": 100 * np.mean([turn(w, ref_w) for w in ws]), + "E[P&L] mean": np.mean(pnls), "E[P&L] sd": np.std(pnls, ddof=1), + "exact E[P&L]": exact.expected_pnl, "s per solve": (time.time() - t1) / max(len(ws), 1)}) + print(pd.DataFrame(rows[-1:]).to_string(index=False, float_format=lambda v: f"{v:.2f}"), flush=True) + print("\nB. scenarios needed as the candidate set grows (exact Gaussian reference)\n") + print(pd.DataFrame(rows).to_string(index=False, float_format=lambda v: f"{v:.2f}")) diff --git a/scripts/experiments/rowgen.py b/scripts/experiments/rowgen.py new file mode 100644 index 0000000..3eb2a73 --- /dev/null +++ b/scripts/experiments/rowgen.py @@ -0,0 +1,115 @@ +"""Tail-row generation for the RU mean-CVaR MILP: solve on a working set of scenarios, add violated ones, repeat. + +Full problem (new mode, proportional buy costs c): max (g_full - c p)'n s.t. + g_s'n - c p'n + alpha + u_s >= 0 (all s) [u_s >= L_s - alpha, L_s = -(g_s - c p)'n] + alpha + sum_s u_s / ((1-beta) S) <= Lambda + (1 + c) p'n <= W, p_i n_i <= w_max W, sector caps, n integer >= 0 +Restricted problem: rows only for s in the working set (u_s = 0 elsewhere) -> a relaxation; optimal for the +full problem as soon as no omitted scenario has L_s > alpha*. +""" +import sys, time +import numpy as np +import scipy.sparse as sps +from scipy.optimize import milp, LinearConstraint, Bounds +from scipy.stats import norm, qmc +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.optimization.service import OptimizationService, OptimizationRequest +from quantcore.optimization.problem import MeanCVaRProblem, ProblemSpec, OptimizationConstraints, TransactionCosts +from quantcore.optimization.solvers import HighsSolver +from quantcore.models.covariance import CovarianceModel, symmetric_inv_sqrt + +N_ASSETS = int(sys.argv[1]); SIZES = [int(x) for x in sys.argv[2].split(",")] +FULL = len(sys.argv) > 3 and sys.argv[3] == "full" +BETA = 0.95; W = 1_000_000.0; LAMBDA = 0.05 * W; H = 5; C = 0.001; WMAX = 0.10; SMAX = 0.25; GAP = 1e-4 +settings = get_settings(); md = MarketDataService(settings); svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, solver="highs", cash=W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_h = CovarianceModel.from_dict(req.covariance).estimate(r).values * H +mu_h = (mu_ann / 252).reindex(symbols).values * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown").tolist() +N = len(symbols) +vals, vecs = np.linalg.eigh(cov_h); o = np.argsort(vals)[::-1]; A = vecs[:, o] * np.sqrt(vals[o]) + + +def scen_sobol_mm(S, seed=0): + z = norm.ppf(qmc.Sobol(d=N, scramble=True, seed=seed).random(S)) + z = z - z.mean(0); z = z @ symmetric_inv_sqrt(z.T @ z / S) + return np.expm1(mu_h + z @ A.T) # simple returns, as in production + + +def solve_restricted(gain, work, S_full, mean_gain): + """MILP on the working set `work` (indices into gain rows). Returns n, alpha, objective, seconds.""" + K = len(work) + p = prices + # variables: n (N), alpha (1), u (K) + nv = N + 1 + K + c = np.zeros(nv); c[:N] = -(mean_gain - C * p) # minimise negative expected P&L (net of costs) + rows, cols, vals_ = [], [], [] + lb, ub = [], [] + ri = 0 + G = gain[work] - C * p[None, :] # net gain per share in the working scenarios + # scenario rows: G_s' n + alpha + u_s >= 0 + rows.extend(np.repeat(np.arange(K), N)); cols.extend(np.tile(np.arange(N), K)); vals_.extend(G.ravel()) + rows.extend(np.arange(K)); cols.extend([N] * K); vals_.extend([1.0] * K) + rows.extend(np.arange(K)); cols.extend(N + 1 + np.arange(K)); vals_.extend([1.0] * K) + lb.extend([0.0] * K); ub.extend([np.inf] * K); ri = K + # cvar row: alpha + sum u / ((1-beta) S_full) <= Lambda + rows.append(ri); cols.append(N); vals_.append(1.0) + rows.extend([ri] * K); cols.extend(N + 1 + np.arange(K)); vals_.extend([1.0 / ((1 - BETA) * S_full)] * K) + lb.append(-np.inf); ub.append(LAMBDA); ri += 1 + # budget (1+c) p'n <= W + rows.extend([ri] * N); cols.extend(range(N)); vals_.extend((1 + C) * p); lb.append(-np.inf); ub.append(W); ri += 1 + # sector caps + for sec in sorted(set(sectors)): + idx = [i for i, s in enumerate(sectors) if s == sec] + rows.extend([ri] * len(idx)); cols.extend(idx); vals_.extend(p[idx]); lb.append(-np.inf); ub.append(SMAX * W); ri += 1 + Am = sps.csr_matrix((vals_, (rows, cols)), shape=(ri, nv)) + vlb = np.zeros(nv); vub = np.full(nv, np.inf); vub[:N] = np.floor(WMAX * W / p + 1e-9); vlb[N] = -np.inf + integrality = np.zeros(nv); integrality[:N] = 1 + t0 = time.time() + res = milp(c, constraints=LinearConstraint(Am, np.array(lb), np.array(ub)), integrality=integrality, + bounds=Bounds(vlb, vub), options={"mip_rel_gap": GAP, "time_limit": 1800}) + n = np.rint(res.x[:N]); alpha = res.x[N] + return n, alpha, -res.fun, time.time() - t0 + + +def row_generation(gain, verbose=True): + S = len(gain) + mean_gain = gain.mean(0) + p = prices + # initial working set: the worst 2 (1-beta) S scenarios of the equal-weight (cap-respecting) portfolio + n0 = np.floor(W / N / p) + L0 = -(gain - C * p) @ n0 + K0 = int(2 * (1 - BETA) * S) + work = np.argsort(L0)[-K0:] + t_solve = 0.0 + for it in range(1, 30): + n, alpha, obj, dt = solve_restricted(gain, work, S, mean_gain) + t_solve += dt + L = -(gain - C * p) @ n + viol = np.setdiff1d(np.where(L > alpha + 1e-9)[0], work) + if verbose: + print(f" iter {it}: |work|={len(work)}, MILP {dt:.1f}s, obj {obj:.2f}, alpha {alpha:.1f}, violated {len(viol)}", flush=True) + if len(viol) == 0: + u = np.maximum(L - alpha, 0) + cvar = alpha + u.sum() / ((1 - BETA) * S) + return n, obj, cvar, it, t_solve + work = np.union1d(work, viol) + raise RuntimeError("no convergence") + + +print(f"N={N}, beta={BETA}, gap={GAP}") +for S in SIZES: + gain = scen_sobol_mm(S) * prices[None, :] + t0 = time.time() + n, obj, cvar, iters, t_solve = row_generation(gain) + t_rg = time.time() - t0 + print(f"S={S:>7}: row generation {iters} iterations, {t_rg:.1f}s total ({t_solve:.1f}s in MILPs), E[P&L]={obj:.2f}, CVaR={cvar:.1f}, positions={(n>0).sum()}", flush=True) + if FULL: + t0 = time.time() + n_f, alpha_f, obj_f, dt = solve_restricted(gain, np.arange(S), S, gain.mean(0)) + print(f"S={S:>7}: full MILP {dt:.1f}s, E[P&L]={obj_f:.2f}, same shares: {np.array_equal(n, n_f)}, max |dn| = {np.abs(n - n_f).max():.0f}", flush=True) diff --git a/scripts/experiments/scaling.py b/scripts/experiments/scaling.py new file mode 100644 index 0000000..1b71b09 --- /dev/null +++ b/scripts/experiments/scaling.py @@ -0,0 +1,54 @@ +"""Time scaling of the mean-CVaR LP / MILP in the number of scenarios S (HiGHS locally, optionally cuOpt).""" +import sys, time, json, zlib +import numpy as np +from scipy.stats import norm, qmc +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.optimization.service import OptimizationService, OptimizationRequest +from quantcore.optimization.problem import MeanCVaRProblem, ProblemSpec, OptimizationConstraints, TransactionCosts +from quantcore.optimization.solvers import HighsSolver, CuOptSolver, to_cuopt_payload +from quantcore.models.covariance import CovarianceModel, symmetric_inv_sqrt + +N_ASSETS = int(sys.argv[1]); SIZES = [int(x) for x in sys.argv[2].split(",")]; MODE = sys.argv[3] # highs | cuopt +BETA = 0.95; W = 1_000_000.0; LAMBDA = 0.05 * W; H = 5 +settings = get_settings(); md = MarketDataService(settings); svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, solver="highs", cash=W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_h = CovarianceModel.from_dict(req.covariance).estimate(r).values * H +mu_h = (mu_ann / 252).reindex(symbols).values * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown").tolist() +N = len(symbols) +vals, vecs = np.linalg.eigh(cov_h); o = np.argsort(vals)[::-1]; A = vecs[:, o] * np.sqrt(vals[o]) + +def scen_sobol_mm(S, seed=0): + z = norm.ppf(qmc.Sobol(d=N, scramble=True, seed=seed).random(S)) + z = z - z.mean(0); z = z @ symmetric_inv_sqrt(z.T @ z / S) + return mu_h + z @ A.T + +print(f"N={N}, beta={BETA}, mode={MODE}") +print(f"{'S':>8} {'nnz':>10} {'gen s':>7} {'build s':>8} {'LP s':>8} {'MILP s':>8} {'status':>10} {'E[P&L]':>9} {'json MB':>8} {'cuopt s':>8} {'cu solve':>9}") +for S in SIZES: + t0 = time.time(); scen = scen_sobol_mm(S); t_gen = time.time() - t0 + spec = ProblemSpec(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=W, holdings=np.zeros(N), + confidence=BETA, cvar_limit=LAMBDA, objective="max_return", constraints=OptimizationConstraints(), + costs=TransactionCosts()) + prob = MeanCVaRProblem(spec) + t0 = time.time(); model = prob.build(); t_build = time.time() - t0 + nnz = model.A.nnz + row = f"{S:>8} {nnz:>10} {t_gen:>7.1f} {t_build:>8.1f}" + if MODE == "highs": + lp = MeanCVaRProblem(spec).build(); lp.var_types = ["C"] * len(lp.var_types) + t0 = time.time(); sol_lp = HighsSolver().solve(lp, time_limit=300); t_lp = time.time() - t0 + t0 = time.time(); sol = HighsSolver().solve(model, time_limit=300, mip_gap=1e-4); t_mip = time.time() - t0 + obj = prob.decode(sol.x)["expected_pnl"] if sol.x is not None else float("nan") + row += f" {t_lp:>8.1f} {t_mip:>8.1f} {sol.status:>10} {obj:>9.1f}" + else: + t0 = time.time(); payload = to_cuopt_payload(model, 300.0, {"mip_relative_gap": 1e-4, "log_to_console": False}) + body = json.dumps(payload); mb = len(body) / 1e6; t_json = time.time() - t0 + t0 = time.time(); sol = CuOptSolver(settings).solve(model, time_limit=300.0, extra_config={"mip_relative_gap": 1e-4, "log_to_console": False}); t_cu = time.time() - t0 + obj = prob.decode(sol.x)["expected_pnl"] if sol.x is not None else float("nan") + row += f" {'':>8} {'':>8} {sol.status:>10} {obj:>9.1f} {mb:>8.1f} {t_cu:>8.1f} {sol.solve_time:>9.1f}" + print(row, flush=True) diff --git a/scripts/experiments/scenario_noise.py b/scripts/experiments/scenario_noise.py new file mode 100644 index 0000000..efaa5d0 --- /dev/null +++ b/scripts/experiments/scenario_noise.py @@ -0,0 +1,177 @@ +"""Sampling noise of the mean-CVaR MILP under different scenario generators (GBM model). + +Reference: the exact Gaussian problem max mu'x s.t. -mu'x + k_beta * sqrt(x'Sigma x) <= Lambda, +caps, budget, integer shares -- solved with an outer-approximation (Kelley cutting planes) on the +same MILP (no scenarios at all). Under a normal model this is the mean-variance solution. +""" +import sys, time, json +import numpy as np, pandas as pd +from scipy.stats import norm, qmc +from scipy.optimize import milp, LinearConstraint, Bounds +import scipy.sparse as sps + +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.optimization.service import OptimizationService, OptimizationRequest +from quantcore.optimization.problem import MeanCVaRProblem, ProblemSpec, OptimizationConstraints, TransactionCosts +from quantcore.optimization.solvers import HighsSolver +from quantcore.models.covariance import CovarianceModel, symmetric_inv_sqrt + +N_ASSETS = int(sys.argv[1]) if len(sys.argv) > 1 else 30 +S = int(sys.argv[2]) if len(sys.argv) > 2 else 2048 +N_SEEDS = int(sys.argv[3]) if len(sys.argv) > 3 else 8 +INTEGER = (sys.argv[4] == "int") if len(sys.argv) > 4 else False +KINDS = sys.argv[5].split(",") if len(sys.argv) > 5 else ["mc", "antithetic", "mc+mm", "lhs", "sobol", "sobol+mm"] +BETA = float(sys.argv[6]) if len(sys.argv) > 6 else 0.95 +W = 1_000_000.0 +LAMBDA = 0.05 * W +H = 5 + +settings = get_settings() +md = MarketDataService(settings) +svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, scenario_model="gbm", n_scenarios=S, solver="highs", cash=W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_model = CovarianceModel.from_dict(req.covariance) +cov_d = cov_model.estimate(r).values +mu_h = (mu_ann / 252).reindex(symbols).values * H +cov_h = cov_d * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown").tolist() +N = len(symbols) +k_beta = norm.pdf(norm.ppf(BETA)) / (1 - BETA) +print(f"N={N} assets, S={S} scenarios, horizon {H} days, beta={BETA}, k_beta={k_beta:.4f}") +print("weekly vol range (%%): %.2f .. %.2f" % (100 * np.sqrt(np.diag(cov_h)).min(), 100 * np.sqrt(np.diag(cov_h)).max())) + +# eigen root (PCA ordering) and Cholesky root +vals, vecs = np.linalg.eigh(cov_h) +order = np.argsort(vals)[::-1] +A_pca = vecs[:, order] * np.sqrt(vals[order]) # columns = principal directions, largest first +A_chol = np.linalg.cholesky(cov_h) + + +def moment_match(z): + z = z - z.mean(axis=0) + c = z.T @ z / len(z) + return z @ symmetric_inv_sqrt(c) + + +def base_sample(kind, seed): + rng = np.random.default_rng(seed) + if kind == "mc": + return rng.standard_normal((S, N)), A_chol + if kind == "antithetic": + z = rng.standard_normal((S // 2, N)) + return np.vstack([z, -z]), A_chol + if kind == "mc+mm": + return moment_match(rng.standard_normal((S, N))), A_chol + if kind == "lhs": + u = qmc.LatinHypercube(d=N, scramble=True, seed=seed).random(S) + return norm.ppf(u), A_pca + if kind == "sobol": + u = qmc.Sobol(d=N, scramble=True, seed=seed).random(S) + return norm.ppf(u), A_pca + if kind == "sobol+mm": + u = qmc.Sobol(d=N, scramble=True, seed=seed).random(S) + return moment_match(norm.ppf(u)), A_pca + raise ValueError(kind) + + +def scenarios(kind, seed): + z, a = base_sample(kind, seed) + return mu_h + z @ a.T # S x N horizon log-returns (used linearly: exact Gaussian P&L) + + +def solve_milp(scen, integer): + spec = ProblemSpec(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=W, + holdings=np.zeros(N), confidence=BETA, cvar_limit=LAMBDA, objective="max_return", + constraints=OptimizationConstraints(), costs=TransactionCosts(proportional_buy=0.0, proportional_sell=0.0)) + prob = MeanCVaRProblem(spec) + model = prob.build() + if not integer: + model.var_types = ["C"] * len(model.var_types) + sol = HighsSolver().solve(model, time_limit=300, mip_gap=1e-5) + n = np.rint(sol.x[prob.i_n]) if integer else sol.x[prob.i_n] + pnl = scen @ (n * prices) + from quantcore.measurement.risk import scenario_cvar + _, cvar_in = scenario_cvar(-pnl, BETA) + return n, float(pnl.mean()), cvar_in, sol + + +def true_stats(n): + v = n * prices + mean = v @ mu_h + sd = np.sqrt(v @ cov_h @ v) + return mean, -mean + k_beta * sd, sd + + +# ---- exact Gaussian reference by outer approximation on the same integer feasible set ---------------- +def exact_reference(tol=1e-6, max_iter=200, integer=True): + ub = np.floor(np.minimum(0.10, 1.0) * W / prices) + c = -(prices * mu_h) # minimise -E[P&L] + rows = [prices] # budget + lb = [-np.inf]; ubs = [W] + for sec in sorted(set(sectors)): # sector caps (defaults of OptimizationConstraints) + mask = np.array([s == sec for s in sectors], dtype=float) + rows.append(prices * mask); lb.append(-np.inf); ubs.append(OptimizationConstraints().max_sector_weight * W) + cuts = [] + n_prev = None + for it in range(max_iter): + A = np.vstack(rows + [cut[0] for cut in cuts]) + lo = np.array(lb + [-np.inf] * len(cuts)); hi = np.array(ubs + [cut[1] for cut in cuts]) + res = milp(c, constraints=LinearConstraint(A, lo, hi), integrality=np.ones(N) if integer else np.zeros(N), + bounds=Bounds(np.zeros(N), ub), options={"mip_rel_gap": 1e-9}) + n = np.round(res.x) if integer else res.x + mean, cvar, sd = true_stats(n) + if cvar <= LAMBDA + tol * LAMBDA: + return n, mean, cvar, it + 1 + v = n * prices + grad = -prices * mu_h + k_beta * prices * (cov_h @ v) / sd # d CVaR / d n + # cut: cvar(n_k) + grad'(n - n_k) <= Lambda -> grad'n <= Lambda - cvar(n_k) + grad'n_k + cuts.append((grad, LAMBDA - cvar + grad @ n)) + raise RuntimeError("no convergence") + + +t0 = time.time() +n_ref, mean_ref, cvar_ref, iters = exact_reference() +x_rel, mean_rel, cvar_rel, iters_rel = exact_reference(integer=False) +print(f"\nexact Gaussian reference (integer): E[P&L]={mean_ref:.2f} CHF, CVaR={cvar_ref:.2f}, " + f"{int((n_ref > 0).sum())} positions, {iters} cuts, {time.time()-t0:.1f}s") +print(f"continuous relaxation: E[P&L]={mean_rel:.2f} CHF, CVaR={cvar_rel:.2f}") + +results = {} +for kind in KINDS: + rows = [] + t0 = time.time() + for seed in range(N_SEEDS): + n, epnl, cvar_in, sol = solve_milp(scenarios(kind, seed), INTEGER) + mean_t, cvar_t, _ = true_stats(n) + rows.append({"seed": seed, "epnl_in": epnl, "cvar_in": cvar_in, "epnl_true": mean_t, "cvar_true": cvar_t, + "n_pos": int((n > 0).sum()), "shares": n, "status": sol.status}) + df = pd.DataFrame(rows) + shares = np.vstack(df["shares"].values) + # dissimilarity between seeds: mean over pairs of (value traded to go from one to the other) / W + pair_turn = [] + for i in range(len(shares)): + for j in range(i + 1, len(shares)): + pair_turn.append(np.abs(shares[i] - shares[j]) @ prices / W) + dist_ref = np.abs(shares - (n_ref if INTEGER else x_rel)) @ prices / W + results[kind] = { + "E[P&L] in-sample mean": df.epnl_in.mean(), "E[P&L] in-sample sd": df.epnl_in.std(), + "E[P&L] true mean": df.epnl_true.mean(), "E[P&L] true sd": df.epnl_true.std(), + "loss vs ref (CHF)": (mean_ref if INTEGER else mean_rel) - df.epnl_true.mean(), + "true CVaR mean": df.cvar_true.mean(), "true CVaR sd": df.cvar_true.std(), + "CVaR excess over limit (%)": 100 * (df.cvar_true.mean() / LAMBDA - 1), + "positions": df.n_pos.mean(), + "turnover between seeds (%)": 100 * np.mean(pair_turn), "turnover to ref (%)": 100 * dist_ref.mean(), + "seconds": time.time() - t0, + } + print(f"{kind:10s} done in {time.time()-t0:.0f}s statuses={df.status.unique().tolist()}") + +out = pd.DataFrame(results).T +pd.set_option("display.width", 250); pd.set_option("display.float_format", lambda v: f"{v:10.2f}") +print("\n%s problem; reference E[P&L] = %.2f CHF, CVaR limit = %.0f CHF (5%% of wealth), weekly" % ("INTEGER" if INTEGER else "CONTINUOUS (weights)", mean_ref if INTEGER else mean_rel, LAMBDA)) +print(out.T.to_string()) +out.to_csv(f"{'int' if INTEGER else 'cont'}_scenario_noise_N{N}_S{S}.csv") diff --git a/scripts/experiments/tilt_correlation.py b/scripts/experiments/tilt_correlation.py new file mode 100644 index 0000000..5ed11de --- /dev/null +++ b/scripts/experiments/tilt_correlation.py @@ -0,0 +1,155 @@ +"""What the importance-sampling tilt does with the correlation structure. + +1. The per-asset shift is proportional to ``Sigma v`` (covariance with the reference portfolio), + not to the assets' own volatilities. +2. The tilted set keeps the full covariance (an exponential tilt of a Gaussian is a mean shift; + moment matching makes it exact), so correlations are untouched. +3. How much tail coverage does a portfolio get whose loss direction differs from the reference's? + Measured for portfolios ranging from market-like to deliberately orthogonal. + +Usage: python scripts/experiments/tilt_correlation.py [N] [S] [SEEDS] [BETA] +""" +import sys + +import numpy as np +import pandas as pd +from scipy.stats import norm + +from quantcore.config import get_settings +from quantcore.data.market_data import MarketDataService +from quantcore.measurement.risk import scenario_cvar +from quantcore.models import sampling as smp +from quantcore.models.covariance import CovarianceModel +from quantcore.models.simulation import GBMSimulator, TiltSpec +from quantcore.optimization.service import OptimizationRequest, OptimizationService + +N_ASSETS = int(sys.argv[1]) if len(sys.argv) > 1 else 30 +S = int(sys.argv[2]) if len(sys.argv) > 2 else 4096 +N_SEEDS = int(sys.argv[3]) if len(sys.argv) > 3 else 12 +BETA = float(sys.argv[4]) if len(sys.argv) > 4 else 0.95 +W, H = 1_000_000.0, 5 + +settings = get_settings() +md = MarketDataService(settings) +svc = OptimizationService(settings, md) +req = OptimizationRequest(max_assets=N_ASSETS, scenario_model="gbm", n_scenarios=S, horizon="week", cvar_limit=0.05 * W) +symbols = svc.candidate_symbols(req) +mu_ann, _, _ = svc.expected_returns(req, symbols) +r = md.returns(symbols, currency="base", freq="daily").dropna() +cov_model = CovarianceModel.from_dict(req.covariance) +cov_h = cov_model.estimate(r).values * H +prices = md.latest_prices(symbols, currency="base").reindex(symbols).values +sectors = md.sectors(symbols).fillna("Unknown") +N = len(symbols) +vol_h = np.sqrt(np.diag(cov_h)) +market_w = svc.market_exposure(symbols) + +print(f"N={N}, S={S}, beta={BETA}, weekly horizon; average correlation " + f"{np.mean((cov_h / np.outer(vol_h, vol_h))[np.triu_indices(N, 1)]):.2f}") + +# --------------------------------------------------------------------------------------------- +# 1. what the tilt does per asset +# --------------------------------------------------------------------------------------------- +A = smp.pca_root(cov_h) +v_ref = market_w * W +d = smp.loss_direction(A, v_ref) +shift_returns = norm.ppf(BETA) * (A @ d) # the return shift of the tilted block +sigma_ref = np.sqrt(v_ref @ cov_h @ v_ref) +implied = -norm.ppf(BETA) * (cov_h @ v_ref) / sigma_ref # -z_beta * Sigma v / sigma +beta_i = (cov_h @ market_w) / (market_w @ cov_h @ market_w) # beta of each asset to the reference +table = pd.DataFrame({ + "weight (%)": market_w * 100, "weekly vol (%)": vol_h * 100, "beta to ref": beta_i, + "shift (%)": shift_returns * 100, "shift / own vol": shift_returns / vol_h, +}, index=symbols) +print("\n1. per-asset shift of the tilted block (z_beta along the loss direction)") +print(" max |A d - (-z Sigma v / sigma)| = %.2e (the shift *is* -z_beta Sigma v / sigma)" + % np.abs(shift_returns - implied).max()) +print(" corr(shift, beta) = %.4f, corr(shift, own vol) = %.4f" + % (np.corrcoef(shift_returns, beta_i)[0, 1], np.corrcoef(shift_returns, vol_h)[0, 1])) +show = table.sort_values("beta to ref") +print(pd.concat([show.head(4), show.tail(4)]).round(3).to_string()) + +# --------------------------------------------------------------------------------------------- +# 2. the tilted set keeps the covariance +# --------------------------------------------------------------------------------------------- +sim = GBMSimulator(cov_model) +plain = sim.simulate(r, n_scenarios=S, horizon_periods=H, mu=mu_ann / 252, seed=1) +tilted = sim.simulate(r, n_scenarios=S, horizon_periods=H, mu=mu_ann / 252, seed=1, + tilt=TiltSpec(exposure=v_ref, label="market")) +w = tilted.weights +centred = tilted.log_returns - w @ tilted.log_returns +cov_t = (centred * w[:, None]).T @ centred +corr_t = cov_t / np.sqrt(np.outer(np.diag(cov_t), np.diag(cov_t))) +corr_m = cov_h / np.outer(vol_h, vol_h) +# unweighted (i.e. ignoring the likelihood ratios) for comparison +cov_u = np.cov(tilted.log_returns, rowvar=False) +corr_u = cov_u / np.sqrt(np.outer(np.diag(cov_u), np.diag(cov_u))) +print("\n2. covariance of the tilted set (weighted with the likelihood ratios)") +print(" max |weighted cov - h Sigma| = %.2e" % np.abs(cov_t - cov_h).max()) +print(" max |weighted corr - model corr| = %.2e" % np.abs(corr_t - corr_m).max()) +print(" max |UNweighted corr - model| = %.2e (the raw tilted sample is not the model)" + % np.abs(corr_u - corr_m).max()) + +# --------------------------------------------------------------------------------------------- +# 3. coverage of portfolios that are not market-like +# --------------------------------------------------------------------------------------------- +def portfolio(kind: str) -> np.ndarray: + if kind == "market": + return market_w.copy() + if kind == "equal weight": + return np.full(N, 1.0 / N) + if kind == "optimised": + res = svc.optimize(OptimizationRequest(max_assets=N_ASSETS, scenario_model="gbm", n_scenarios=S, + horizon="week", cvar_limit=0.05 * W, cash=W, solver="highs", + confidence=BETA)) + held = {p["symbol"]: p["value"] for p in res.positions} + v = np.array([held.get(s, 0.0) for s in symbols]) + return v / v.sum() + if kind == "highest beta": + idx = np.argsort(beta_i)[-5:] + v = np.zeros(N); v[idx] = 1 / len(idx); return v + if kind == "lowest beta": + idx = np.argsort(beta_i)[:5] + v = np.zeros(N); v[idx] = 1 / len(idx); return v + if kind == "min variance": + inv = np.linalg.pinv(cov_h) @ np.ones(N) + v = np.maximum(inv, 0); return v / v.sum() + if kind == "least market-like": # maximise the angle to the reference loss direction, long only + c = cov_h @ market_w + v = np.zeros(N); v[np.argsort(c / vol_h)[:3]] = 1 / 3; return v + raise ValueError(kind) + + +kinds = ["market", "equal weight", "optimised", "highest beta", "min variance", "lowest beta", "least market-like"] +rows = [] +for kind in kinds: + wgt = portfolio(kind) + v = wgt * W + d_p = smp.loss_direction(A, v) + cos = float(d_p @ d) # alignment of the loss directions + losses_p, losses_t, cvars_p, cvars_t = [], [], [], [] + for seed in range(N_SEEDS): + pl = sim.simulate(r, n_scenarios=S, horizon_periods=H, mu=mu_ann / 252, seed=seed) + ti = sim.simulate(r, n_scenarios=S, horizon_periods=H, mu=mu_ann / 252, seed=seed, + tilt=TiltSpec(exposure=v_ref, label="market")) + lp = -(pl.simple_returns() @ v) + lt = -(ti.simple_returns() @ v) + cvars_p.append(scenario_cvar(lp, BETA)[1]) + cvars_t.append(scenario_cvar(lt, BETA, ti.probabilities)[1]) + var_p = np.quantile(lp, BETA) + losses_p.append((lp > var_p).sum()) + losses_t.append((lt > var_p).sum()) + exact = -(v @ (np.expm1(mu_ann.reindex(symbols).values / 252 * H))) + \ + norm.pdf(norm.ppf(BETA)) / (1 - BETA) * np.sqrt(v @ cov_h @ v) + rows.append({ + "cos to ref direction": cos, "tail points plain": np.mean(losses_p), "tail points tilted": np.mean(losses_t), + "CVaR sd plain (%)": 100 * np.std(cvars_p, ddof=1) / np.mean(cvars_p), + "CVaR sd tilted (%)": 100 * np.std(cvars_t, ddof=1) / np.mean(cvars_t), + "noise reduction": np.std(cvars_p, ddof=1) / max(np.std(cvars_t, ddof=1), 1e-12), + "CVaR bias tilted (%)": 100 * (np.mean(cvars_t) / exact - 1), + "CVaR bias plain (%)": 100 * (np.mean(cvars_p) / exact - 1), + }) +out = pd.DataFrame(rows, index=kinds) +pd.set_option("display.width", 220) +print(f"\n3. tail coverage by portfolio type ({N_SEEDS} seeds; 'cos' = alignment of loss directions)") +print(out.round(2).to_string()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8bf7505 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,62 @@ +"""Shared fixtures: a synthetic data store with 12 stocks in four currencies.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from quantcore.config import Settings +from quantcore.data.market_data import MarketDataService +from quantcore.data.store import DataStore + +N, T = 12, 3000 +SYMBOLS = [f"S{i}" for i in range(N)] +CURRENCIES = ["CHF", "USD", "EUR", "GBP"] * 3 +SECTORS = ["Information Technology", "Financials", "Health Care", "Energy"] * 3 + + +def make_prices(seed: int = 0) -> tuple[pd.DataFrame, np.ndarray]: + rng = np.random.default_rng(seed) + dates = pd.bdate_range("2013-01-01", periods=T) + f = rng.standard_normal((T, 2)) * 0.01 + beta = rng.uniform(0.5, 1.5, size=(N, 2)) + eps = rng.standard_t(5, size=(T, N)) * 0.01 + r = f @ beta.T + eps + 0.0002 + return pd.DataFrame(100 * np.exp(np.cumsum(r, axis=0)), index=dates, columns=SYMBOLS), r + + +@pytest.fixture(scope="session") +def settings(tmp_path_factory) -> Settings: + return Settings(data_dir=tmp_path_factory.mktemp("data"), cuopt_url="http://127.0.0.1:9", solver_fallback=True) + + +@pytest.fixture(scope="session") +def store(settings) -> DataStore: + store = DataStore(settings) + store.ensure_dirs() + prices, _ = make_prices() + rng = np.random.default_rng(1) + uni = pd.DataFrame({ + "symbol": SYMBOLS, "name": [f"Company {s}" for s in SYMBOLS], "region": ["US"] * 6 + ["Europe"] * 6, + "country": ["United States"] * 6 + ["Switzerland"] * 6, "listing_currency": CURRENCIES, "exchange": "", + "exchange_code": "", "gics_sector": SECTORS, "gics_sub_industry": "", "yahoo_sector": "", "yahoo_industry": "", + "market_cap": 1e9, "market_cap_base": rng.uniform(1e9, 1e11, N), "shares_outstanding": rng.uniform(1e7, 1e9, N), + "last_price": prices.iloc[-1].values, "first_price_date": "2013-01-01", "price_coverage": 1.0, + "index_weight": np.nan, "source": "synthetic", "rank_in_region": range(1, N + 1), + }) + store.save_universe(uni) + store.save_prices(prices) + fx = pd.DataFrame({ + "USD": 0.85 * np.exp(np.cumsum(rng.standard_normal(T) * 0.004)), + "EUR": 0.95 * np.exp(np.cumsum(rng.standard_normal(T) * 0.003)), + "GBP": 1.10 * np.exp(np.cumsum(rng.standard_normal(T) * 0.004)), "CHF": 1.0, + }, index=prices.index) + store.save_fx(fx) + store.update_manifest(last_price_date=str(prices.index[-1].date()), n_universe=N) + return store + + +@pytest.fixture(scope="session") +def md(settings, store) -> MarketDataService: + return MarketDataService(settings, store) diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..00bf676 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,209 @@ +import numpy as np +import pytest +from fastapi.testclient import TestClient + +import quantcore.api.app as app_module + + +@pytest.fixture(scope="module") +def client(settings, store): + app_module.state = app_module.State(settings) + with TestClient(app_module.app) as c: + yield c + + +def test_health_and_universe(client): + h = client.get("/health").json() + assert h["status"] == "ok" and h["store"]["prices"] is True and h["cuopt"]["reachable"] is False + u = client.get("/universe", params={"region": "Europe"}).json() + assert len(u) == 6 and all(row["region"] == "Europe" for row in u) + assert client.get("/universe/S0").json()["symbol"] == "S0" + assert client.get("/universe/NOPE").status_code == 404 + + +def test_prices_returns_and_market(client): + p = client.get("/prices", params={"symbols": "S0,S1", "freq": "weekly", "start": "2020-01-01"}).json() + assert p["columns"] == ["S0", "S1"] and len(p["index"]) > 100 + r = client.get("/returns", params={"symbols": "S0", "currency": "local"}).json() + assert r["kind"] == "log_returns" + m = client.get("/market/index", params={"freq": "weekly"}).json() + assert "mean_ann" in m and m["n"] > 100 + w = client.get("/market/weights").json()["weights"] + assert abs(sum(w.values()) - 1) < 1e-6 + + +def test_risk_and_performance_endpoints(client): + body = {"symbols": ["S0", "S1", "S2"], "horizon": "week", "confidence": [0.95, 0.99]} + rows = client.post("/risk/assets", json=body).json() + assert len(rows) == 3 and rows[0]["cvar99_hist"] > rows[0]["var99_hist"] + corr = client.post("/risk/correlation", json={"symbols": ["S0", "S1"], "method": "ewm", "lam": 0.99}).json() + assert corr["data"][0][0] == pytest.approx(1.0) + pr = client.post("/risk/portfolio", json={"holdings": {"S0": 10, "S3": 5}, "cash": 500, "horizon": "month"}).json() + assert pr["confidence"]["0.95"]["cvar_money"] > 0 and pr["horizon_periods"] == 21 + series = client.post("/risk/series", json={"symbol": "S0", "window": 100}).json() + assert "vol_ewm_ann" in series["columns"] + perf = client.post("/performance/assets", json={"symbols": ["S0"], "freq": "weekly"}).json() + assert "sharpe" in perf[0] + pp = client.post("/performance/portfolio", json={"holdings": {"S0": 10, "S1": 20}}).json() + assert "max_drawdown" in pp + roll = client.post("/performance/rolling", json={"symbol": "S1", "window": 60}).json() + assert "sharpe" in roll["columns"] + assert client.post("/risk/assets", json={"symbols": ["NOPE"]}).status_code == 422 + + +def test_models_and_optimize(client): + er = client.post("/model/expected-returns", json={"symbols": ["S0", "S1", "S2", "S3"], + "views": [{"symbol": "S1", "sentiment": 0.8, "confidence": 0.6}]}).json() + assert er["posterior_ann"]["S1"] > er["prior_ann"]["S1"] and len(er["views"]) == 1 + sc = client.post("/model/scenarios", json={"symbols": ["S0", "S1"], "model": "gbm", "n_scenarios": 500}).json() + assert sc["description"]["n_scenarios"] == 512 and sc["description"]["n_requested"] == 500 # Sobol': power of two + assert sc["description"]["sampling"] == "sobol" and sc["description"]["moment_matched"] and len(sc["summary"]) == 2 + assert len(sc["description"]["hash"]) == 16 + body = {"mode": "new", "cash": 200000, "symbols": ["S0", "S1", "S2", "S3", "S4", "S5"], "cvar_limit": 8000, + "n_scenarios": 400, "scenario_model": "gbm", "solver": "highs", "horizon": "week", + "importance_sampling": "off"} + res = client.post("/optimize", json=body).json() + assert res["status"] == "Optimal" and res["solver"] == "highs" and res["reproducible"] is True + assert res["cvar"] <= 8000 + 1e-3 and res["cash_end"] >= 0 + assert all(p["shares"] >= 0 for p in res["positions"]) + # identical request -> identical scenario hash and identical portfolio + again = client.post("/optimize", json=body).json() + assert again["scenario_hash"] == res["scenario_hash"] and again["positions"] == res["positions"] + # importance sampling and the exact Gaussian mode are accepted and give proposals of the same kind + tilted = client.post("/optimize", json={**body, "importance_sampling": "market"}).json() + assert tilted["status"] == "Optimal" and tilted["model"]["scenarios"]["weighted"] is True + assert tilted["model"]["scenarios"]["importance_sampling"]["direction"] == "market" + adaptive = client.post("/optimize", json={**body, "importance_sampling": "adaptive"}).json() + assert adaptive["status"] == "Optimal" + assert adaptive["solver_info"]["importance_sampling"]["stage1_status"] == "Optimal" + # the market tilt does not depend on the risk limit: same scenario key and hash at another limit + other = client.post("/optimize", json={**body, "importance_sampling": "market", "cvar_limit": 5000}).json() + assert other["scenario_key"]["id"] == tilted["scenario_key"]["id"] + assert other["scenario_hash"] == tilted["scenario_hash"] and other["cvar"] <= 5000 + 1e-3 + exact = client.post("/optimize", json={**body, "scenario_model": "gaussian_exact"}).json() + assert exact["status"] == "Optimal" and exact["reproducible"] and exact["scenario_hash"] is None + assert exact["solver_info"]["outer_approximation"]["converged"] and exact["cvar"] <= 8000 * (1 + 1e-5) + hold = {p["symbol"]: p["shares"] for p in res["positions"]} + reb = client.post("/optimize", json={**body, "mode": "rebalance", "holdings": hold, "cash": res["cash_end"], + "cvar_limit": 5000}).json() + assert reb["status"] == "Optimal" and reb["mode"] == "rebalance" + # the starting portfolio is reported on the same scenarios, so before and after are comparable + before = reb["before"] + assert res["before"] is None # a new portfolio has nothing to compare against + assert before["n_positions"] == sum(1 for v in hold.values() if v > 0) + assert before["invested"] == pytest.approx(res["invested"], rel=1e-9) + assert before["cvar"] > 5000 > reb["cvar"] # the rebalancing is what brings the risk under the new limit + assert before["expected_pnl"] > reb["expected_pnl"] # ... at the cost of expected return + assert sum(before["sector_exposure"].values()) == pytest.approx(before["invested"] / reb["wealth"], rel=1e-6) + + +def test_jobs_and_snapshots(client): + snaps = client.get("/snapshots").json() + assert isinstance(snaps, list) + created = client.post("/snapshots", json={"name": "test-snap"}) + assert created.status_code == 201 + assert "test-snap" in client.get("/snapshots").json() + assert client.get("/jobs").json() == [] + assert client.get("/jobs/nope").status_code == 404 + + +def test_asset_risk_in_value_space(client): + body = {"symbols": ["S0", "S1"], "horizon": "month", "confidence": [0.95, 0.99]} + log = client.post("/risk/assets", json=body).json() + val = client.post("/risk/assets", json={**body, "space": "value"}).json() + for a, b in zip(log, val): + # a VaR is a quantile: it carries over through 1 - e^(-x) (up to the interpolation between + # order statistics, which does not commute with a non-linear map) + assert b["var95_hist"] < a["var95_hist"] + assert b["var95_hist"] == pytest.approx(1 - np.exp(-a["var95_hist"]), rel=1e-4) + assert b["var95_normal_ewm"] == pytest.approx(1 - np.exp(-a["var95_normal_ewm"]), rel=1e-8) + # a CVaR is a tail mean: computed in value space it is strictly below the naive conversion + for col in ("cvar95_normal_ewm", "cvar99_normal_ewm"): + assert b[col] < 1 - np.exp(-a[col]) + # this fixture drifts upwards, so at a monthly horizon the estimated mean can push the + # parametric VaR below zero -- the reason the UI asks for drift="zero" + assert a["var95_normal_ewm"] < 0 + assert b["cvar99_hist"] < 1.0 # a share of the value can never reach 100 % + assert b["vol_ewm_ann"] == a["vol_ewm_ann"] # only the loss columns change + # the market-risk convention: no drift extrapolated into the horizon + zero = client.post("/risk/assets", json={**body, "drift": "zero"}).json() + for a, z in zip(log, zero): + assert z["var95_normal_ewm"] != a["var95_normal_ewm"] + assert z["var95_hist"] == a["var95_hist"] # historical columns keep their window + + +def test_symbol_table_and_facets(client): + t = client.get("/data/symbols", params={"limit": 5}).json() + assert t["n_universe"] == 12 and t["n_matched"] == 12 and len(t["symbols"]) == 5 + caps = [row["market_cap"] for row in t["symbols"]] + assert caps == sorted(caps, reverse=True) # largest first + assert set(t["facets"]["regions"]) == {"US", "Europe"} and sum(t["facets"]["regions"].values()) == 12 + row = t["symbols"][0] + assert row["price"] > 0 and 0 < row["market_weight"] <= 1 and row["sector"] + eu = client.get("/data/symbols", params={"region": "Europe"}).json() + assert eu["n_matched"] == 6 and all(r["region"] == "Europe" for r in eu["symbols"]) + one = client.get("/data/symbols", params={"search": "S1"}).json() + assert {r["symbol"] for r in one["symbols"]} >= {"S1", "S10", "S11"} + + +def test_model_covariance_endpoint(client): + body = {"symbols": ["S0", "S1", "S2", "S3"], "freq": "daily", "horizon": "week", + "covariance": {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}} + c = client.post("/model/covariance", json=body).json() + assert c["correlation"]["columns"] == body["symbols"] + assert c["correlation"]["data"][0][0] == pytest.approx(1.0) + assert 0 <= c["shrinkage_applied"] <= 1 and c["conditioning"]["n_assets"] == 4 + assert c["conditioning"]["rank_deficient"] is False + assert sum(c["eigenvalues"]) == pytest.approx(1.0) + # the covariance is scaled to the horizon, the volatility to the year + vol_h = c["volatility_horizon"]["S0"] + assert c["covariance"]["data"][0][0] == pytest.approx(vol_h ** 2, rel=1e-9) + assert c["volatility_ann"]["S0"] == pytest.approx(vol_h * (252 / 5) ** 0.5, rel=1e-9) + + +def test_scenario_distribution_of_a_portfolio(client): + body = {"symbols": ["S0", "S1", "S2"], "model": "gbm", "n_scenarios": 1024, "horizon": "week", + "weights": {"S0": 0.5, "S1": 0.3, "S2": 0.2}, "value": 1_000_000, "bins": 30, + "focus": ["S0"], "importance_sampling": "off"} + d = client.post("/model/scenarios", json=body).json() + p = d["portfolio"] + assert p["value"] == 1_000_000 and p["invested"] == pytest.approx(1_000_000) + assert len(p["histogram"]["mass"]) == 30 and sum(p["histogram"]["mass"]) == pytest.approx(1.0) + assert p["risk"]["0.99"]["cvar"] > p["risk"]["0.95"]["cvar"] > p["risk"]["0.95"]["var"] + # the quantiles and the VaR are the same statistic seen from two sides (the VaR is the order + # statistic the Rockafellar-Uryasev rows use, the quantile interpolates - one rank apart) + assert p["quantiles"]["0.05"] == pytest.approx(-p["risk"]["0.95"]["var"], rel=1e-2) + assert p["min"] <= p["quantiles"]["0.01"] <= p["quantiles"]["0.5"] <= p["quantiles"]["0.99"] <= p["max"] + assert sum(d["asset_histograms"]["S0"]["mass"]) == pytest.approx(1.0) + # a tilted set reports the same model quantities, with probability weights + tilted = client.post("/model/scenarios", json={**body, "importance_sampling": "market"}).json() + assert tilted["description"]["weighted"] is True + assert tilted["description"]["effective_sample_size"] < tilted["description"]["n_scenarios"] + assert tilted["portfolio"]["risk"]["0.95"]["cvar"] == pytest.approx(p["risk"]["0.95"]["cvar"], rel=0.05) + # shares instead of weights + shares = client.post("/model/scenarios", json={**body, "weights": {}, "holdings": {"S0": 10, "S1": 5}}).json() + assert shares["portfolio"]["invested"] > 0 and len(shares["portfolio"]["exposure"]) == 2 + + +def test_the_ui_is_served(client): + page = client.get("/ui/") + assert page.status_code == 200 and "quantcore" in page.text + assert client.get("/ui/js/main.js").status_code == 200 + assert client.get("/", follow_redirects=False).status_code in (307, 308) + + +def test_the_ui_can_be_switched_off(settings): + """``QUANTCORE_ENABLE_UI=false`` ships the same image with the JSON API and /docs only.""" + from fastapi import FastAPI + + off = FastAPI() + assert app_module.mount_test_bench(off, settings.model_copy(update={"enable_ui": False})) is False + with TestClient(off) as c: + assert c.get("/ui/").status_code == 404 + assert c.get("/", follow_redirects=False).status_code == 404 + + on = FastAPI() + assert app_module.mount_test_bench(on, settings.model_copy(update={"enable_ui": True})) is True + with TestClient(on) as c: + assert c.get("/ui/").status_code == 200 + assert c.get("/", follow_redirects=False).headers["location"] == "/ui/" diff --git a/tests/test_black_litterman.py b/tests/test_black_litterman.py new file mode 100644 index 0000000..9d9b07f --- /dev/null +++ b/tests/test_black_litterman.py @@ -0,0 +1,91 @@ +import numpy as np +import pandas as pd +import pytest + +from quantcore.models.black_litterman import ( + BlackLittermanModel, View, idzorek_omega, idzorek_omega_numeric, view_from_sentiment, +) + + +@pytest.fixture +def model(): + symbols = ["A", "B", "C", "D"] + vols = np.array([0.20, 0.25, 0.15, 0.30]) / np.sqrt(52) + corr = np.array([[1, .5, .3, .2], [.5, 1, .4, .1], [.3, .4, 1, .3], [.2, .1, .3, 1]]) + cov = pd.DataFrame(np.outer(vols, vols) * corr, index=symbols, columns=symbols) + w = pd.Series([0.4, 0.3, 0.2, 0.1], index=symbols) + return BlackLittermanModel(cov, w, market_return=0.07 / 52, risk_free=0.01 / 52, tau=0.05, periods_per_year=52) + + +def test_equilibrium_is_consistent(model): + # the market portfolio earns exactly the market excess return under the implied returns + assert model.w @ model.pi == pytest.approx(model.market_return - model.risk_free) + assert model.delta > 0 + res = model.posterior([]) + pd.testing.assert_series_equal(res.posterior, res.prior) + # market weights are recovered as the unconstrained MV weights of the prior (tau-adjusted cov ignored here) + w_mv = np.linalg.solve(model.delta * model.sigma, model.pi) + np.testing.assert_allclose(w_mv, model.w) + + +def test_idzorek_closed_form_equals_numeric(model): + p = np.array([0.0, 1.0, 0.0, 0.0]) + q = (0.12 - 0.01) / 52 + for c in (0.1, 0.35, 0.5, 0.8, 0.99): + omega_cf = idzorek_omega(p, model.tau * model.sigma, c) + omega_num = idzorek_omega_numeric(p, q, model.pi, model.sigma, model.tau, model.delta, c) + assert omega_cf == pytest.approx(omega_num, rel=1e-4) + + +def test_confidence_moves_weights_by_fraction(model): + view = View({"B": 1.0}, q=0.15, confidence=0.4) + res = model.posterior([view]) + w_post = np.linalg.solve(model.delta * model.sigma, res.posterior.values) + full = model.posterior([View({"B": 1.0}, q=0.15, confidence=1.0)]) + w_full = np.linalg.solve(model.delta * model.sigma, full.posterior.values) + tilt_partial = w_post - model.w + tilt_full = w_full - model.w + np.testing.assert_allclose(tilt_partial, 0.4 * tilt_full, atol=1e-10) + + +def test_full_confidence_reproduces_view(model): + res = model.posterior([View({"B": 1.0}, q=0.15, confidence=1.0)]) + assert res.posterior_ann["B"] == pytest.approx(0.15, abs=1e-9) + + +def test_positive_view_raises_asset_and_relative_view(model): + res = model.posterior([View({"A": 1.0}, q=0.20, confidence=0.6)]) + assert res.posterior_ann["A"] > res.prior_ann["A"] + rel = model.posterior([View({"C": 1.0, "D": -1.0}, q=0.05, confidence=0.5, label="C beats D")]) + assert (rel.posterior_ann["C"] - rel.posterior_ann["D"]) > (rel.prior_ann["C"] - rel.prior_ann["D"]) + assert rel.views[0]["label"] == "C beats D" + assert rel.posterior_cov.shape == (4, 4) + + +def test_view_from_sentiment(model): + v = view_from_sentiment(model, "C", sentiment=1.0, confidence=0.7, strength=0.25) + assert v.q == pytest.approx(model.prior_ann["C"] + 0.25 * 0.15, rel=1e-6) + neg = view_from_sentiment(model, "C", sentiment=-1.0, confidence=0.7) + assert neg.q < model.prior_ann["C"] + with pytest.raises(ValueError): + View({"A": 1.0}, q=0.1, confidence=0.0) + + +def test_documentation_example_numbers(model): + """docs/measurement_and_models.tex, Example 7.2 (the fixture is that four-asset model).""" + assert model.market_var * 52 == pytest.approx(0.02502, abs=1e-5) + assert model.delta == pytest.approx(2.3986, abs=1e-4) + np.testing.assert_allclose(model.prior_ann.values, [0.0736, 0.0880, 0.0435, 0.0550], atol=5e-5) + assert model.w @ model.prior_ann.values == pytest.approx(0.07, abs=1e-9) + res = model.posterior([View({"B": 1.0}, q=0.15, confidence=0.4)]) + p = np.array([0.0, 1.0, 0.0, 0.0]) + base = p @ (model.tau * model.sigma) @ p + assert base == pytest.approx(6.0096e-5, rel=1e-4) + assert res.omega[0] == pytest.approx(1.5 * base, rel=1e-12) + assert res.omega[0] == pytest.approx(9.0144e-5, rel=1e-4) + np.testing.assert_allclose(res.posterior_ann.values, [0.0835, 0.1128, 0.0494, 0.0580], atol=5e-5) + w_post = np.linalg.solve(model.delta * model.sigma, res.posterior.values) + full = model.posterior([View({"B": 1.0}, q=0.15, confidence=1.0)]) + w_100 = np.linalg.solve(model.delta * model.sigma, full.posterior.values) + assert w_100[1] == pytest.approx(0.7139, abs=5e-5) and w_post[1] == pytest.approx(0.4656, abs=5e-5) + np.testing.assert_allclose(w_post[[0, 2, 3]], model.w[[0, 2, 3]], atol=1e-10) diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py new file mode 100644 index 0000000..60c0e22 --- /dev/null +++ b/tests/test_cleaning.py @@ -0,0 +1,132 @@ +import numpy as np +import pandas as pd +import pytest + +from quantcore.data.cleaning import ( + CleaningReport, + clean_fx, + clean_prices, + find_spike_blocks, + fx_scale_exponent, + longest_missing_run, +) + + +def test_spike_blocks_and_genuine_moves(): + idx = pd.bdate_range("2024-01-01", periods=40) + p = pd.Series(100.0, index=idx) + p.iloc[5] = 200.0 # single wrong tick (2x, reverses next day) + p.iloc[10:12] = 210.0 # two-day block + p.iloc[20:] = 40.0 # genuine crash that stays + p.iloc[30:33] = 45.0 # ordinary small moves + assert find_spike_blocks(p) == [5, 10, 11] + + +def test_clean_prices_report_and_gaps(): + idx = pd.bdate_range("2024-01-01", periods=40) + a = pd.Series(np.linspace(50, 60, 40), index=idx) + b = a.copy() + b.iloc[3:8] = -1.0 # broken adjustment stretch + b.iloc[25] = 300.0 # spike + from quantcore.data.cleaning import CleaningReport + + rep = CleaningReport() + out = clean_prices(pd.DataFrame({"A": a, "B": b}), report=rep) + assert rep.n_nonpositive == 5 and rep.symbols_nonpositive == {"B": 5} + assert rep.n_spikes == 1 and rep.symbols_spikes == {"B": 1} + assert out["A"].notna().all() + assert out["B"].isna().sum() == 6 + runs = longest_missing_run(out) + assert runs["A"] == 0 and runs["B"] == 5 + + +# --------------------------------------------------------------------------------------------- +# FX quoting-unit changes (the NOK/CHF artefact: 96 observations returned 100x too small) +# --------------------------------------------------------------------------------------------- +def _fx_series(n=500, seed=0): + rng = np.random.default_rng(seed) + idx = pd.bdate_range("2016-01-01", periods=n) + return pd.Series(0.12 * np.exp(np.cumsum(rng.standard_normal(n) * 0.004)), index=idx) + + +def test_a_quoting_unit_change_is_undone_wherever_it_sits(): + """A stretch quoted 100x too small is repaired, whatever part of the series it covers.""" + good = _fx_series() + for lo, hi in ((70, 140), (0, 60), (440, 500)): # middle, start, end + broken = good.copy() + broken.iloc[lo:hi] /= 100.0 + fixed = clean_fx(broken.to_frame("NOK"))["NOK"] + assert np.allclose(fixed.values, good.values), (lo, hi) + # the artefact is what made the volatility absurd; the repair takes it back + vol = lambda x: float(np.log(x).diff().std() * np.sqrt(252)) + assert vol(broken) > 2.0 and vol(fixed) == pytest.approx(vol(good), rel=1e-9) + + +def test_several_unit_changes_and_other_powers_of_ten(): + good = _fx_series() + broken = good.copy() + broken.iloc[30:60] /= 100.0 + broken.iloc[200:210] *= 10.0 + broken.iloc[300:301] /= 1000.0 + fixed = clean_fx(broken.to_frame("NOK"))["NOK"] + assert np.allclose(fixed.values, good.values) + assert set(fx_scale_exponent(broken).unique()) == {0, -2, 1, -3} + + +def test_genuine_moves_are_never_rescaled(): + """A devaluation is a price move, not a unit change: nothing short of a factor of ten is touched. + + The 6.1x case is real: ABVX.PA gained 510 % on its trial results (2025-07-23). A loose tolerance + reads that as a decimal shift and rewrites the year of history that follows it. + """ + good = _fx_series() + for factor in (0.75, 1.4, 0.5, 2.0, 3.0, 6.101, 6.421, 1 / 6.101, 8.0): + moved = good.copy() + moved.iloc[250:] *= factor + fixed = clean_fx(moved.to_frame("TRY"))["TRY"] + assert np.allclose(fixed.values, moved.values), factor + + +def test_clean_fx_is_idempotent_and_reports_per_currency(): + good = _fx_series() + frame = pd.DataFrame({"CHF": 1.0, "EUR": good * 8, "NOK": good}) + frame.iloc[100:180, frame.columns.get_loc("NOK")] /= 100.0 + report = CleaningReport() + fixed = clean_fx(frame, report=report) + assert report.n_fx_rescaled == 80 and report.fx_rescaled == {"NOK": 80} + assert np.allclose(fixed["NOK"].values, good.values) + assert fixed["EUR"].equals(frame["EUR"]) and fixed["CHF"].equals(frame["CHF"]) + again = CleaningReport() + assert clean_fx(fixed, report=again).equals(fixed) and again.n_fx_rescaled == 0 + + +def test_a_short_series_and_missing_values_are_handled(): + idx = pd.bdate_range("2024-01-01", periods=6) + assert fx_scale_exponent(pd.Series([0.1], index=idx[:1])).empty is False + gappy = pd.Series([0.1, np.nan, 0.001, 0.001, np.nan, 0.1], index=idx) + fixed = clean_fx(gappy.to_frame("NOK"))["NOK"] + assert fixed.dropna().round(6).tolist() == [0.1, 0.1, 0.1, 0.1] + assert fixed.isna().tolist() == [False, True, False, False, True, False] + + +def test_fx_spikes_are_removed_but_a_real_regime_break_is_kept(): + """A reversing tick is a quote error; 2015-01-16 (the SNB floor) is not, and must survive.""" + good = _fx_series(n=400) + broken = good.copy() + broken.iloc[120] *= 1.09 # one-day spike, reverses next day + broken.iloc[250:] *= 0.81 # a permanent 19 % repricing: never reversed + fixed = clean_fx(broken.to_frame("EUR"))["EUR"] + assert np.isnan(fixed.iloc[120]) # the tick is gone + assert fixed.iloc[249] == broken.iloc[249] and fixed.iloc[250] == broken.iloc[250] + assert fixed.drop(fixed.index[120]).equals(broken.drop(broken.index[120])) + + +def test_the_unit_repair_runs_before_the_spike_pass(): + """A 100x stretch must be rescaled, not mistaken for two enormous spikes at its edges.""" + good = _fx_series(n=300) + broken = good.copy() + broken.iloc[100:160] /= 100.0 + report = CleaningReport() + fixed = clean_fx(broken.to_frame("NOK"), report=report)["NOK"] + assert report.n_fx_rescaled == 60 and report.n_fx_spikes == 0 + assert np.allclose(fixed.values, good.values) diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..08dcce3 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,222 @@ +"""The HTTP client and the ``quantcore-api`` CLI, against the app running in this process. + +``TestClient`` satisfies the transport contract of :class:`QuantcoreClient` (``request(method, url, +params=, json=, timeout=)``), so the same code paths that talk to the Spark are exercised here +without a server: the URLs, the query parameters, the request bodies and the error handling. +""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +import quantcore.api.app as app_module +from quantcore import apicli +from quantcore.client import QuantcoreClient, QuantcoreError + + +@pytest.fixture(scope="module") +def api(settings, store): + app_module.state = app_module.State(settings) + with TestClient(app_module.app) as transport: + yield QuantcoreClient("http://testserver", session=transport) + + +@pytest.fixture +def cli(api, monkeypatch): + """Run the CLI against the in-process app: every command builds its client through ``_client``.""" + monkeypatch.setattr(apicli, "_client", lambda args: api) + return apicli.main + + +# ------------------------------------------------------------------------------------------------ +# client +# ------------------------------------------------------------------------------------------------ +def test_the_client_covers_the_api(api): + health = api.health() + assert health["status"] == "ok" and health["store"]["prices"] is True + + assert len(api.universe(region="Europe")) == 6 + assert api.symbol("S0")["symbol"] == "S0" + table = api.symbols_table(limit=5) + assert len(table["symbols"]) == 5 and "sectors" in table["facets"] + assert table["n_universe"] >= table["n_matched"] >= 5 + + px = api.prices(["S0", "S1"], freq="weekly") + assert px["columns"] == ["S0", "S1"] and px["freq"] == "weekly" + assert api.returns(["S0"])["kind"] == "log_returns" + assert api.market_index(freq="weekly")["n"] > 100 + assert sum(api.market_weights()["weights"].values()) == pytest.approx(1.0) + + rows = api.asset_risk(["S0", "S1"], horizon="week") + assert len(rows) == 2 and rows[0]["cvar95_hist"] > rows[0]["var95_hist"] + assert api.risk_series("S0", horizon="week")["index"] + assert api.covariance(["S0", "S1"])["columns"] == ["S0", "S1"] + assert api.correlation(["S0", "S1"])["data"][0][0] == pytest.approx(1.0) + assert api.portfolio_risk({"S0": 100, "S1": 50}, cash=1000.0, horizon="week")["value"] > 0 + + assert len(api.asset_performance(["S0", "S1"])) == 2 + assert api.rolling_performance("S0", window=252)["index"] + assert api.portfolio_performance({"S0": 100}) + + assert api.expected_returns(max_assets=6)["market_return_ann"] != 0 + assert api.model_covariance(["S0", "S1"])["conditioning"]["n_assets"] == 2 + scen = api.scenarios(["S0", "S1"], model="gbm", n_scenarios=1024, holdings={"S0": 10}) + assert scen["description"]["n_scenarios"] == 1024 and scen["portfolio"]["invested"] > 0 + + # order-independent: another module may have created a snapshot on the session-scoped store + assert isinstance(api.jobs(), list) and isinstance(api.snapshots(), list) + + +def test_the_client_optimises(api): + res = api.optimize(cash=1_000_000, max_assets=6, horizon="week", scenario_model="gaussian_exact", + solver="highs", time_limit=120) + assert res["status"] in ("Optimal", "FeasibleFound") + assert res["solver"] == "highs" and res["positions"] + # a rebalancing run carries the starting point, so before/after can be compared + holdings = {p["symbol"]: p["shares"] for p in res["positions"][:2]} + reb = api.optimize(mode="rebalance", holdings=holdings, cash=10_000, max_assets=6, horizon="week", + scenario_model="gaussian_exact", solver="highs", time_limit=120) + assert reb["before"] is not None and reb["before"]["invested"] > 0 + + +def test_none_arguments_are_left_to_the_api(api): + """Optional arguments that are ``None`` must not be sent, or the API would see "None".""" + sent: dict = {} + original = api.session.request + + def spy(method, url, **kw): + sent.update({"params": kw.get("params"), "json": kw.get("json")}) + return original(method, url, **kw) + + api.session.request = spy + try: + api.prices(["S0"], start=None, end=None) + assert "start" not in (sent["params"] or {}) and "end" not in (sent["params"] or {}) + api.expected_returns(symbols=None, max_assets=6) + assert "symbols" not in (sent["json"] or {}) + finally: + api.session.request = original + + +def test_the_client_reports_api_errors(api): + with pytest.raises(QuantcoreError) as exc: + api.symbol("NOPE") + assert exc.value.status_code == 404 and not exc.value.unreachable + + with pytest.raises(QuantcoreError) as exc: # 422: the field name must reach the caller + api.asset_risk(["S0"], window=1) + assert exc.value.status_code == 422 and "window" in str(exc.value) + assert "greater than or equal to 10" in str(exc.value) + + +def test_the_client_reports_an_unreachable_server(): + with QuantcoreClient("http://127.0.0.1:9", timeout=2.0) as qc, pytest.raises(QuantcoreError) as exc: + qc.health() + assert exc.value.unreachable and "127.0.0.1:9" in str(exc.value) + + +def test_the_client_retries_dropped_gets_but_never_posts(): + """A poll on a kept-alive connection the server just closed must not abort `update --wait`; + a POST (optimisation, job start) must never be sent twice.""" + with QuantcoreClient("http://127.0.0.1:9") as qc: + retry = qc.session.get_adapter("http://127.0.0.1:9/health").max_retries + assert retry.read >= 1 and "GET" in retry.allowed_methods and "POST" not in retry.allowed_methods + + +def test_wait_for_job_polls_until_the_job_ends(): + states = iter(["queued", "running", "running", "done"]) + + class Stub: + def request(self, method, url, **kw): + class Res: + status_code = 200 + + @staticmethod + def json(): + return {"job_id": "abc", "status": next(states)} + return Res() + + qc = QuantcoreClient("http://stub", session=Stub()) + seen: list[str] = [] + record = qc.wait_for_job("abc", poll=0.0, on_poll=lambda r: seen.append(r["status"])) + assert record["status"] == "done" and seen == ["queued", "running", "running"] + + with pytest.raises(QuantcoreError): # a job that never finishes hits the deadline + QuantcoreClient("http://stub", session=type("S", (), { + "request": lambda self, m, u, **k: type("R", (), { + "status_code": 200, "json": staticmethod(lambda: {"status": "running"})})()})() + ).wait_for_job("abc", poll=0.0, deadline=-1.0) + + +# ------------------------------------------------------------------------------------------------ +# CLI +# ------------------------------------------------------------------------------------------------ +def test_every_documented_command_parses(): + parser = apicli.build_parser() + for argv in ( + ["health"], ["endpoints"], ["universe", "--region", "Europe", "--sector", "Health Care"], + ["symbol", "S0"], ["prices", "S0", "S1", "--freq", "weekly", "--tail", "5"], + ["returns", "S0", "--currency", "local"], ["market", "weights", "--max-assets", "10"], + ["risk", "S0", "S1", "--horizon", "week", "--drift", "zero", "--space", "value"], + ["risk-portfolio", "--holdings", '{"S0": 10}'], ["correlation", "S0", "S1"], + ["covariance", "S0", "S1", "--method", "rolling"], ["performance", "S0"], + ["expected-returns", "--max-assets", "50", "--view", "S0:0.6:0.4"], + ["model-covariance", "S0", "S1"], ["scenarios", "S0", "--model", "gbm", "--n-scenarios", "1024"], + ["optimize", "--cash", "1000000", "--cvar-limit", "40000", "--risk-drift", "zero"], + ["optimize", "--request", "r.json", "--json"], ["jobs", "--job-id", "abc"], + ["setup", "--offline", "--wait"], ["update", "--full", "--wait"], + ["snapshots", "--create", "--name", "demo"], ["selftest", "--full", "--write"], + ): + args = parser.parse_args(argv) + assert callable(args.func), argv + + +def test_sentiment_views_are_parsed(): + assert apicli._views(["S0:0.6"]) == [{"symbol": "S0", "sentiment": 0.6}] + assert apicli._views(["S0:0.6:0.4:0.3"]) == [ + {"symbol": "S0", "sentiment": 0.6, "confidence": 0.4, "strength": 0.3}] + with pytest.raises(SystemExit): + apicli._views(["S0"]) + + +@pytest.mark.parametrize("argv", [ + ["health"], ["endpoints"], ["universe", "--region", "Europe"], ["symbol", "S0"], + ["prices", "S0", "S1", "--tail", "3"], ["returns", "S0", "--tail", "3"], ["market", "index"], + ["market", "weights"], ["risk", "S0", "S1", "--drift", "zero"], + ["risk-portfolio", "--holdings", '{"S0": 10}'], ["correlation", "S0", "S1"], + ["covariance", "S0", "S1"], ["performance", "S0"], ["expected-returns", "--max-assets", "6"], + ["model-covariance", "S0", "S1"], ["scenarios", "S0", "S1", "--n-scenarios", "1024"], + ["jobs"], ["snapshots"], +]) +def test_the_cli_runs_every_read_only_command(cli, capsys, argv): + assert cli(argv) == 0 + assert capsys.readouterr().out.strip() + + +def test_the_cli_optimises_and_prints_a_summary(cli, capsys): + assert cli(["optimize", "--max-assets", "6", "--scenario-model", "gaussian_exact", + "--solver", "highs", "--time-limit", "120"]) == 0 + out = capsys.readouterr().out + assert "status Optimal" in out and "E[P&L]" in out and "CVaR" in out + + +def test_the_selftest_passes_against_a_healthy_service(cli, capsys): + """The selftest is the deployment check: every endpoint, one table, exit 0 when all answered.""" + assert cli(["selftest", "--n-symbols", "2"]) == 0 + out = capsys.readouterr().out + assert "FAIL" not in out + assert "0 failed" in out and "3 skipped" in out # the writing endpoints stay untouched + for endpoint in ("GET /health", "POST /risk/portfolio", "POST /optimize", "GET /snapshots"): + assert endpoint in out + + +def test_the_selftest_fails_loudly_when_the_service_is_down(capsys, monkeypatch): + monkeypatch.setattr(apicli, "_client", lambda args: QuantcoreClient("http://127.0.0.1:9", timeout=1.0)) + assert apicli.main(["selftest"]) == 1 + assert "not usable" in capsys.readouterr().out + + +def test_the_cli_exits_non_zero_when_the_api_is_unreachable(capsys): + assert apicli.main(["--url", "http://127.0.0.1:9", "--timeout", "1", "health"]) == 1 + assert "is the service running?" in capsys.readouterr().err diff --git a/tests/test_data.py b/tests/test_data.py new file mode 100644 index 0000000..eb78305 --- /dev/null +++ b/tests/test_data.py @@ -0,0 +1,81 @@ +import numpy as np +import pandas as pd +import pytest + +from quantcore.data.download_service import splice_adjusted +from quantcore.data.fx import FXConverter +from quantcore.data.yahoo import apply_price_scale, normalise_info +from quantcore.universe.sources import parse_ishares_holdings +from quantcore.universe.symbols import normalise_share_class_name, to_yahoo_symbol + + +def test_fx_conversion_and_log_return_identity(): + idx = pd.bdate_range("2024-01-01", periods=5) + fx = pd.DataFrame({"USD": [0.85, 0.86, 0.84, 0.85, 0.87], "CHF": 1.0}, index=idx) + conv = FXConverter(fx, base="CHF") + px = pd.DataFrame({"A": [10.0, 11.0, 10.5, 10.8, 11.2], "B": [100.0, 101.0, 99.0, 98.0, 100.0]}, index=idx) + cur = pd.Series({"A": "USD", "B": "CHF"}) + chf = conv.convert_prices(px, cur) + np.testing.assert_allclose(chf["A"], px["A"] * fx["USD"]) + np.testing.assert_allclose(chf["B"], px["B"]) + # CHF log-return = local log-return + FX log-return + r_chf = np.log(chf).diff().iloc[1:] + r_loc = np.log(px).diff().iloc[1:] + r_fx = conv.fx_log_returns(cur, idx).iloc[1:] + np.testing.assert_allclose(r_chf.values, (r_loc + r_fx).values) + with pytest.raises(KeyError): + conv.convert_prices(px, pd.Series({"A": "JPY", "B": "CHF"})) + + +def test_gbp_pence_normalisation(): + info = normalise_info("HSBA.L", {"currency": "GBp", "marketCap": 1e11, "regularMarketPrice": 1533.8, + "firstTradeDateMilliseconds": 584089200000, "quoteType": "EQUITY"}) + assert info["currency"] == "GBP" and info["price_scale"] == 100.0 + assert info["last_price"] == pytest.approx(15.338) + assert info["first_trade_date"] == "1988-07-05" + px = pd.DataFrame({"HSBA.L": [1500.0, 1600.0], "NESN.SW": [80.0, 81.0]}) + scaled = apply_price_scale(px, pd.Series({"HSBA.L": 100.0})) + assert scaled["HSBA.L"].iloc[0] == 15.0 and scaled["NESN.SW"].iloc[1] == 81.0 + + +def test_splice_rescales_history_on_dividend(): + idx = pd.bdate_range("2024-01-01", periods=10) + stored = pd.DataFrame({"A": np.linspace(100, 109, 10), "B": np.linspace(50, 59, 10)}, index=idx) + fresh_idx = pd.bdate_range("2024-01-08", periods=7) # overlaps the last 5 stored days, adds 2 new + fresh = pd.DataFrame({"A": np.concatenate([stored["A"].iloc[5:].values * 0.98, [110 * 0.98, 111 * 0.98]]), + "B": np.concatenate([stored["B"].iloc[5:].values, [60.0, 61.0]])}, index=fresh_idx) + out, rescaled = splice_adjusted(stored, fresh) + assert rescaled == ["A"] + assert len(out) == 12 + np.testing.assert_allclose(out["A"].iloc[:5], stored["A"].iloc[:5] * 0.98) # history rescaled + np.testing.assert_allclose(out["A"].iloc[5:], fresh["A"]) # fresh window authoritative + np.testing.assert_allclose(out["B"], np.concatenate([stored["B"].values, [60.0, 61.0]])) + + +def test_symbol_mapping_rules(): + assert to_yahoo_symbol("NOVO B", "Omx Nordic Exchange Copenhagen A/S") == "NOVO-B.CO" + assert to_yahoo_symbol("BT.A", "London Stock Exchange") == "BT-A.L" + assert to_yahoo_symbol("RR.", "London Stock Exchange") == "RR.L" + assert to_yahoo_symbol("NESN", "SIX Swiss Exchange") == "NESN.SW" + assert to_yahoo_symbol("BRK.B", country="United States") == "BRK-B" + assert to_yahoo_symbol("NDA FI", "Nasdaq Omx Helsinki Ltd.") == "NDA-FI.HE" + with pytest.raises(ValueError): + to_yahoo_symbol("X", "Mars Exchange") + assert normalise_share_class_name("Alphabet Inc. Class A") == normalise_share_class_name("Alphabet Inc. Class C") + assert normalise_share_class_name("Investor AB ser. B") == normalise_share_class_name("Investor Class A") + + +def test_parse_ishares_holdings(): + text = ('Fondsposition per,"18.Sept.2026"\n \nEmittententicker,Name,Sektor,Anlageklasse,Marktwert,' + 'Gewichtung (%),Nominalwert,Nominale,Kurs,Standort,Börse,Marktwährung\n' + '"ASML","ASML HOLDING","IT","Aktien","438.829.188,00","4,23","438.829.188,00","303.478,00","1.446,00",' + '"Niederlande","Euronext Amsterdam","EUR"\n' + '"NOVO B","NOVO NORDISK CLASS B","Gesundheitsversorgung","Aktien","1,00","1,00","1,00","1,00","1,00",' + '"Dänemark","Omx Nordic Exchange Copenhagen A/S","DKK"\n' + '"XEUR","EUR CASH","Barmittel und/oder Derivate","Geldmarkt","1,00","0,10","1,00","1,00","1,00",' + '"Deutschland","-","EUR"\n') + df = parse_ishares_holdings(text) + assert list(df["symbol"]) == ["ASML.AS", "NOVO-B.CO"] + assert df.iloc[0]["gics_sector"] == "Information Technology" and df.iloc[0]["country"] == "Netherlands" + assert df.iloc[0]["index_weight"] == pytest.approx(4.23) + assert df.iloc[0]["source_date"] == "2026-09-18" diff --git a/tests/test_estimators.py b/tests/test_estimators.py new file mode 100644 index 0000000..b9f8149 --- /dev/null +++ b/tests/test_estimators.py @@ -0,0 +1,54 @@ +import numpy as np +import pandas as pd +import pytest + +from quantcore.measurement import estimators as est + + +@pytest.fixture +def r(): + rng = np.random.default_rng(3) + x = rng.standard_normal((600, 4)) @ np.array([[1, .5, .2, 0], [0, 1, .3, 0], [0, 0, 1, .4], [0, 0, 0, 1]]) * 0.01 + return pd.DataFrame(x, index=pd.bdate_range("2020-01-01", periods=600), columns=list("ABCD")) + + +def test_rolling_cov_matches_numpy(r): + cov = est.rolling_cov(r, 100) + np.testing.assert_allclose(cov.values, np.cov(r.values[-100:], rowvar=False, ddof=1)) + corr = est.rolling_corr(r, 100) + np.testing.assert_allclose(np.diag(corr.values), 1.0) + np.testing.assert_allclose(corr.values, np.corrcoef(r.values[-100:], rowvar=False), atol=1e-12) + + +def test_ewm_cov_diagonal_matches_pandas(r): + lam = 0.94 + cov = est.ewm_cov(r, lam, max_obs=len(r)) + pd_var = r.ewm(alpha=1 - lam, adjust=True).var(bias=True).iloc[-1] + np.testing.assert_allclose(np.diag(cov.values), pd_var.values, rtol=1e-8) + assert est.ewm_weights(10, lam).sum() == pytest.approx(1.0) + + +def test_ewm_sequence_recursion_converges_to_direct(r): + lam = 0.97 + last = None + for date, cov_prev, _ in est.covariance_sequence(r, "ewm", lam=lam, warmup=100): + last = (date, cov_prev) + # Sigma_{T-1} from the recursion vs. the direct (normalised-weight) estimate at T-1 + direct = est.ewm_cov(r.iloc[:-1], lam, max_obs=len(r)) + assert last[0] == r.index[-1] + np.testing.assert_allclose(last[1], direct.values, rtol=0.05, atol=1e-7) + + +def test_rolling_sequence_matches_cov(r): + seq = list(est.covariance_sequence(r, "rolling", window=50)) + date, cov_prev, ret = seq[-1] + assert date == r.index[-1] + np.testing.assert_allclose(cov_prev, np.cov(r.values[-51:-1], rowvar=False, ddof=1)) + np.testing.assert_allclose(ret, r.values[-1]) + + +def test_shrinkage_keeps_variances(r): + cov = est.rolling_cov(r, 200) + sh = est.shrink(cov, 0.3) + np.testing.assert_allclose(np.diag(sh.values), np.diag(cov.values)) + assert abs(sh.values[0, 1]) == pytest.approx(0.7 * abs(cov.values[0, 1])) diff --git a/tests/test_problem.py b/tests/test_problem.py new file mode 100644 index 0000000..c9a4034 --- /dev/null +++ b/tests/test_problem.py @@ -0,0 +1,166 @@ +import itertools + +import numpy as np +import pytest + +from quantcore.config import Settings +from quantcore.measurement.risk import scenario_cvar +from quantcore.optimization.problem import MeanCVaRProblem, OptimizationConstraints, ProblemSpec, TransactionCosts +from quantcore.optimization.solvers import HighsSolver + +SETTINGS = Settings(cuopt_url="http://127.0.0.1:9") + + +def make_spec(seed=0, n=6, S=300, **kw): + rng = np.random.default_rng(seed) + symbols = [f"S{i}" for i in range(n)] + prices = rng.uniform(20, 400, n) + scen = rng.standard_normal((S, n)) * 0.03 + rng.uniform(-0.002, 0.006, n) + sectors = ["Tech", "Fin", "Health"] * (n // 3) + ["Tech"] * (n % 3) + base = dict(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=100_000.0, confidence=0.95, + cvar_limit=4_000.0) + base.update(kw) + return ProblemSpec(**base) + + +def solve(spec): + problem = MeanCVaRProblem(spec) + model = problem.build() + res = HighsSolver(SETTINGS).solve(model, time_limit=60) + assert res.ok, res + return problem, model, res, problem.decode(res.x) + + +def test_new_portfolio_respects_all_constraints(): + # with a 4000 limit the 10 % caps bind first (CVaR ~1040); at 800 the CVaR limit must bind + spec = make_spec(cvar_limit=800.0, constraints=OptimizationConstraints(max_weight=0.10, max_sector_weight=0.25)) + problem, model, res, d = solve(spec) + W = spec.wealth + assert (d["shares"] >= 0).all() and np.issubdtype(d["shares"].dtype, np.integer) + assert d["cash_end"] >= -1e-6 + assert (d["weights"] <= 0.10 + 1e-9).all() + for sec in set(spec.sectors): + assert d["value"][np.array(spec.sectors) == sec].sum() <= 0.25 * W + 1e-6 + # the CVaR of the decoded portfolio (recomputed from the scenarios) is within the limit and binding + # up to the granularity of whole shares; alpha is *a* minimiser of the RU function, not necessarily the VaR + assert d["cvar"] <= spec.cvar_limit + 1e-6 + assert d["cvar"] >= 0.98 * spec.cvar_limit + assert d["var"] <= d["cvar"] + # objective equals expected P&L net of costs + assert res.objective == pytest.approx(d["expected_pnl"], rel=1e-6) + assert d["total_cost"] == pytest.approx(0.001 * d["invested"], rel=1e-9) + + +def test_brute_force_optimality_two_assets(): + spec = make_spec(n=2, S=80, cash=5_000.0, cvar_limit=300.0, + constraints=OptimizationConstraints(max_weight=0.6, max_sector_weight=1.0)) + _, _, res, d = solve(spec) + p, scen = spec.prices, spec.scenarios + best, best_obj = None, -np.inf + max_n = (0.6 * spec.wealth / p).astype(int) + 1 + for n0, n1 in itertools.product(range(max_n[0] + 1), range(max_n[1] + 1)): + n = np.array([n0, n1]) + cost = 0.001 * (n * p).sum() + if (n * p).sum() + cost > spec.cash + 1e-9 or (n * p > 0.6 * spec.wealth + 1e-9).any(): + continue + pnl = scen @ (n * p) - cost + _, cvar = scenario_cvar(-pnl, 0.95) + if cvar <= 300.0 + 1e-6 and pnl.mean() > best_obj: + best, best_obj = n, pnl.mean() + assert best_obj == pytest.approx(res.objective, rel=1e-6) + assert (d["shares"] == best).all() + + +def test_rebalancing_links_trades_and_costs(): + holdings = np.array([50, 0, 20, 0, 10, 0], dtype=float) + spec = make_spec(cash=20_000.0, holdings=holdings, cvar_limit=3_000.0, + costs=TransactionCosts(0.002, 0.003, fixed_per_trade=25.0), + constraints=OptimizationConstraints(max_weight=0.15, max_sector_weight=0.4, max_turnover=0.5)) + problem, model, res, d = solve(spec) + assert problem.rebalancing and problem.use_trade_bin + np.testing.assert_array_equal(d["shares"], holdings.astype(int) + d["buys"] - d["sells"]) + assert not ((d["buys"] > 0) & (d["sells"] > 0)).any() # never buy and sell the same stock + n_trades = int(((d["buys"] + d["sells"]) > 0).sum()) + expected_cost = 0.002 * (d["buys"] * spec.prices).sum() + 0.003 * (d["sells"] * spec.prices).sum() + 25.0 * n_trades + assert d["total_cost"] == pytest.approx(expected_cost) + assert d["cash_end"] == pytest.approx(spec.cash + (d["sells"] * spec.prices).sum() - (d["buys"] * spec.prices).sum() - d["total_cost"]) + assert d["cash_end"] >= -1e-6 + assert d["turnover"] <= 0.5 + 1e-9 + assert d["cvar"] <= 3_000.0 + 1e-6 + + +def test_objectives_min_cvar_and_utility(): + spec = make_spec(objective="min_cvar", cvar_limit=None, target_pnl=40.0) + _, _, res, d = solve(spec) + assert d["expected_pnl"] >= 40.0 - 1e-6 + assert res.objective == pytest.approx(d["cvar"], rel=1e-6) + spec_u = make_spec(objective="utility", cvar_limit=None, risk_aversion=0.5) + _, _, res_u, d_u = solve(spec_u) + assert res_u.objective == pytest.approx(d_u["expected_pnl"] - 0.5 * d_u["cvar"], rel=1e-6) + + +def test_cardinality_and_min_position(): + spec = make_spec(cvar_limit=5_000.0, constraints=OptimizationConstraints( + max_weight=0.3, max_sector_weight=1.0, max_positions=2, min_position_value=5_000.0)) + problem, model, res, d = solve(spec) + assert problem.use_pos_bin + held = d["value"] > 0 + assert held.sum() <= 2 + assert (d["value"][held] >= 5_000.0 - 1e-6).all() + + +def test_spec_validation(): + with pytest.raises(ValueError): + make_spec(cvar_limit=None) # max_return needs a limit + with pytest.raises(ValueError): + make_spec(holdings=np.array([-1, 0, 0, 0, 0, 0])) + with pytest.raises(ValueError): + make_spec(objective="min_cvar", cvar_limit=None) + + +def test_legacy_position_above_cap_is_sold_down_and_invested_bound(): + holdings = np.zeros(6) + holdings[0] = 1000.0 # far above the 10 % cap + spec = make_spec(cash=10_000.0, holdings=holdings, cvar_limit=50_000.0, + constraints=OptimizationConstraints(max_weight=0.10, max_sector_weight=1.0)) + _, model, res, d = solve(spec) + assert d["weights"][0] <= 0.10 + 1e-9 and d["sells"][0] > 0 + # the single-stock bound is min(max_weight, max_invested) * W, not their product + spec2 = make_spec(cvar_limit=50_000.0, constraints=OptimizationConstraints(max_weight=0.10, max_sector_weight=1.0, + max_invested=0.5)) + problem = MeanCVaRProblem(spec2) + np.testing.assert_array_equal(problem._max_shares(), np.floor(0.10 * spec2.wealth / spec2.prices + 1e-9)) + spec3 = make_spec(cvar_limit=50_000.0, constraints=OptimizationConstraints(max_weight=0.6, max_sector_weight=1.0, + max_invested=0.5)) + np.testing.assert_array_equal(MeanCVaRProblem(spec3)._max_shares(), np.floor(0.5 * spec3.wealth / spec3.prices + 1e-9)) + + +def test_documentation_example_rounding_fails(): + """The two-stock instance of docs/optimization_model.tex, Example 4.1 (numbers as printed there).""" + from scipy.optimize import Bounds, LinearConstraint, milp + + prices = np.array([572.0, 2208.0]) + scen = np.array([[0.005, -0.082], [0.014, -0.052], [-0.069, -0.020], [-0.002, 0.055], + [-0.009, 0.083], [0.048, 0.064], [0.044, 0.028], [-0.011, 0.042]]) + spec = ProblemSpec(symbols=["A", "B"], prices=prices, scenarios=scen, sectors=["X", "Y"], cash=10_000.0, + confidence=0.75, cvar_limit=250.0, costs=TransactionCosts(0.0, 0.0), + constraints=OptimizationConstraints(max_weight=0.7, max_sector_weight=1.0)) + problem = MeanCVaRProblem(spec) + model = problem.build() + # LP relaxation: (2.6605, 1.7876), objective 62.02, CVaR binding + lp = milp(-model.c, constraints=LinearConstraint(model.A, model.row_lb, model.row_ub), + integrality=np.zeros(model.n_vars), bounds=Bounds(model.var_lb, model.var_ub)) + np.testing.assert_allclose(lp.x[:2], [2.6605, 1.7876], atol=1e-4) + assert -lp.fun == pytest.approx(62.02, abs=0.01) + + def evaluate(n): + pnl = scen @ (n * prices) + return pnl.mean(), scenario_cvar(-pnl, 0.75)[1] + + assert evaluate(np.array([3, 2]))[1] == pytest.approx(280.13, abs=0.01) # nearest rounding: infeasible + assert evaluate(np.array([2, 1]))[0] == pytest.approx(35.43, abs=0.01) # rounding down: feasible, poor + res = HighsSolver(SETTINGS).solve(model) + assert res.ok + d = problem.decode(res.x) + np.testing.assert_array_equal(d["shares"], [7, 1]) # the integer optimum has a different structure + assert d["expected_pnl"] == pytest.approx(42.58, abs=0.01) and d["cvar"] == pytest.approx(240.74, abs=0.01) diff --git a/tests/test_returns.py b/tests/test_returns.py new file mode 100644 index 0000000..71e4eda --- /dev/null +++ b/tests/test_returns.py @@ -0,0 +1,67 @@ +import numpy as np +import pandas as pd +import pytest + +from quantcore.measurement.returns import ( + align_prices, horizon_periods, horizon_returns, log_returns, periods_per_year, portfolio_log_returns, resample_prices, +) + + +def test_log_returns_reference(): + p = pd.DataFrame({"A": [100.0, 110.0, 99.0]}, index=pd.bdate_range("2024-01-01", periods=3)) + r = log_returns(p) + assert len(r) == 2 + assert r["A"].iloc[0] == pytest.approx(np.log(1.1)) + assert r["A"].iloc[1] == pytest.approx(np.log(0.9)) + + +def test_horizon_periods(): + assert horizon_periods("day", "daily") == 1 + assert horizon_periods("week", "daily") == 5 + assert horizon_periods("month", "daily") == 21 + assert horizon_periods("week", "weekly") == 1 + assert horizon_periods("month", "weekly") == 4 + assert horizon_periods(10, "daily") == 10 + with pytest.raises(ValueError): + horizon_periods("day", "weekly") + assert periods_per_year("weekly") == 52 + + +def test_align_and_resample(): + idx = pd.bdate_range("2024-01-01", periods=10) + p = pd.DataFrame({"A": np.arange(10.0) + 1, "B": np.arange(10.0) + 1}, index=idx) + p.loc[idx[3], "B"] = np.nan # holiday on B's exchange + a = align_prices(p) + assert a.loc[idx[3], "B"] == a.loc[idx[2], "B"] # forward-filled + w = resample_prices(a, "weekly") + assert all(d.dayofweek == 4 for d in w.index) + assert w["A"].iloc[0] == 5.0 # Friday of the first week + + +def test_horizon_returns_sum(): + r = pd.Series(np.arange(1.0, 11.0) / 100, index=pd.bdate_range("2024-01-01", periods=10)) + h = horizon_returns(r, 5, overlapping=True) + assert len(h) == 6 and h.iloc[0] == pytest.approx(0.15) + nb = horizon_returns(r, 5, overlapping=False) + assert len(nb) == 2 and nb.iloc[-1] == pytest.approx(0.40) + + +def test_portfolio_log_returns_exact(): + r = pd.DataFrame({"A": [np.log(1.1), np.log(0.9)], "B": [np.log(1.0), np.log(1.2)]}) + rp = portfolio_log_returns(r, np.array([0.5, 0.5])) + assert rp.iloc[0] == pytest.approx(np.log(1 + 0.5 * 0.1 + 0.5 * 0.0)) + assert rp.iloc[1] == pytest.approx(np.log(1 + 0.5 * (-0.1) + 0.5 * 0.2)) + + +def test_monthly_resampling_works_on_both_pandas_spellings(): + """pandas renamed the month-end alias 'M' -> 'ME' in 2.2 and dropped 'M' in 3.0; the Mac runs + 2.1 and the container 3.x, so the alias is resolved at import time.""" + import pandas as pd + from quantcore.measurement.returns import MONTH_END, resample_prices + + assert MONTH_END in ("M", "ME") + pd.tseries.frequencies.to_offset(MONTH_END) # accepted by the installed pandas + px = pd.DataFrame({"A": np.arange(400.0)}, index=pd.bdate_range("2024-01-01", periods=400)) + monthly = resample_prices(px, "monthly") + assert len(monthly) == 19 and monthly.index.is_month_end.all() + assert (monthly["A"].diff().dropna() > 0).all() # month-end closes, in order diff --git a/tests/test_risk.py b/tests/test_risk.py new file mode 100644 index 0000000..fd1b0a1 --- /dev/null +++ b/tests/test_risk.py @@ -0,0 +1,208 @@ +import numpy as np +import pandas as pd +import pytest +from scipy import stats + +from quantcore.measurement import risk as rk + + +def test_normal_var_cvar_reference(): + # sigma = 1 %, mu = 0: VaR95 = 1.6449 %, CVaR95 = 2.0627 %, VaR99 = 2.3263 %, CVaR99 = 2.6652 % + assert rk.normal_var(0.0, 0.01, 0.95) == pytest.approx(0.016449, abs=1e-6) + assert rk.normal_cvar(0.0, 0.01, 0.95) == pytest.approx(0.020627, abs=1e-6) + assert rk.normal_var(0.0, 0.01, 0.99) == pytest.approx(0.023263, abs=1e-6) + assert rk.normal_cvar(0.0, 0.01, 0.99) == pytest.approx(0.026652, abs=1e-6) + # a positive mean lowers the loss quantile one-for-one + assert rk.normal_var(0.001, 0.01, 0.95) == pytest.approx(0.015449, abs=1e-6) + + +def test_historical_var_cvar_reference(): + r = pd.Series(np.arange(-0.10, 0.10, 0.001)) # 200 equally spaced returns + var95 = rk.historical_var(r, 0.95) + assert var95 == pytest.approx(-np.quantile(r, 0.05)) + cvar95 = rk.historical_cvar(r, 0.95) + tail = r[r <= np.quantile(r, 0.05)] + assert cvar95 == pytest.approx(-tail.mean()) + assert cvar95 > var95 + + +def test_scenario_cvar_is_tail_mean(): + losses = np.array([1.0, 5.0, 3.0, 9.0, 7.0, 2.0, 8.0, 4.0, 6.0, 10.0]) # S = 10 + var, cvar = rk.scenario_cvar(losses, 0.80) # k = 2 -> mean of the two largest + assert cvar == pytest.approx((10 + 9) / 2) + assert var == pytest.approx(9.0) + var, cvar = rk.scenario_cvar(losses, 0.95) # k = 0.5 -> fractional tail: 0.5 * 10 / 0.5 + assert cvar == pytest.approx(10.0) + var, cvar = rk.scenario_cvar(losses, 0.85) # k = 1.5 -> (10 + 0.5 * 9) / 1.5 + assert cvar == pytest.approx((10 + 0.5 * 9) / 1.5) + assert var == pytest.approx(9.0) + + +def test_scenario_cvar_matches_ru_minimisation(): + rng = np.random.default_rng(0) + losses = rng.standard_normal(500) + beta = 0.95 + _, cvar = rk.scenario_cvar(losses, beta) + grid = np.linspace(-1, 4, 20001) + ru = np.min([a + np.maximum(losses - a, 0).mean() / (1 - beta) for a in grid]) + assert cvar == pytest.approx(ru, abs=1e-3) + + +def test_loss_to_money(): + assert rk.loss_to_money(0.0, 1000.0) == 0.0 + assert rk.loss_to_money(np.log(1 / 0.9), 1000.0) == pytest.approx(100.0) + + +def test_asset_risk_table_columns_and_scaling(): + rng = np.random.default_rng(1) + r = pd.DataFrame(rng.standard_normal((1500, 2)) * 0.01, index=pd.bdate_range("2019-01-01", periods=1500), columns=["A", "B"]) + day = rk.asset_risk_table(r, "daily", 200, 0.94, (0.95,), "day") + week = rk.asset_risk_table(r, "daily", 200, 0.94, (0.95,), "week") + assert {"vol_rolling_ann", "vol_ewm_ann", "var95_normal_rolling", "cvar95_hist"} <= set(day.columns) + assert week["horizon_periods"].iloc[0] == 5 + # sqrt(5) scaling of the Gaussian VaR for a zero-mean window (approximately, mean is small) + ratio = week["var95_normal_rolling"] / day["var95_normal_rolling"] + assert ratio.between(2.0, 2.5).all() + assert (day["cvar95_normal_ewm"] > day["var95_normal_ewm"]).all() + + +def test_portfolio_risk_normal_vs_scenarios(md): + svc = rk.RiskService(md) + holdings = {"S0": 100, "S1": 50, "S2": 200} + pr = svc.portfolio_risk(holdings, cash=1000.0, horizon="week", method="normal") + assert pr.value > 0 and set(pr.confidence) == {"0.95", "0.99"} + assert pr.confidence["0.99"]["cvar_money"] > pr.confidence["0.95"]["cvar_money"] > 0 + # historical estimate is of the same order of magnitude + ph = svc.portfolio_risk(holdings, cash=1000.0, horizon="week", method="historical") + assert 0.3 < ph.confidence["0.95"]["cvar_money"] / pr.confidence["0.95"]["cvar_money"] < 3.0 + # scenarios: Gaussian scenarios with the same covariance reproduce the normal CVaR + rng = np.random.default_rng(0) + r = md.returns(list(holdings), currency="base", freq="daily") + from quantcore.measurement import estimators as est + cov = est.rolling_cov(r, 200) + mu_h = r.iloc[-200:].mean().values * 5 + scen = rng.multivariate_normal(mu_h, cov.values * 5, size=200_000) + ps = svc.portfolio_risk(holdings, cash=1000.0, horizon="week", method="scenarios", scenarios=scen) + # differences: sampling noise and the exact money mapping (exp) vs. the log-return closed form + assert ps.confidence["0.95"]["cvar_money"] == pytest.approx(pr.confidence["0.95"]["cvar_money"], rel=0.06) + + +def test_a_one_asset_portfolio_is_a_valid_request(md): + """``np.cov`` of a single column returns a 0-d array; the rolling estimator has to keep it 2-d. + + The simplest call anyone makes -- the risk of a single position -- went through this path. + """ + svc = rk.RiskService(md) + from quantcore.measurement import estimators as est + + r = md.returns(["S0"], currency="base", freq="daily") + assert est.rolling_cov(r, 200).shape == (1, 1) + assert est.ewm_cov(r, 0.94).shape == (1, 1) + for method in ("normal", "historical"): + for cov_method in ("rolling", "ewm"): + pr = svc.portfolio_risk({"S0": 100}, cash=1000.0, horizon="week", method=method, + cov_method=cov_method) + assert pr.value > 0 and pr.confidence["0.95"]["cvar_money"] > 0 + # a one-asset portfolio is the asset: the parametric VaR is z * sd_h - mean_h exactly + one = svc.portfolio_risk({"S0": 100}, cash=0.0, horizon="week", method="normal", cov_method="rolling") + sd_week = float(np.sqrt(est.rolling_cov(r, 200).iloc[0, 0] * 5)) + mean_week = float(r.iloc[-200:].mean().iloc[0] * 5) + assert one.confidence["0.95"]["var"] == pytest.approx(1.6448536269514722 * sd_week - mean_week, rel=1e-9) + + +def test_weekly_reference_values_from_documentation(): + """docs/measurement_and_models.tex, Example 4.1: sigma = 1 % daily, one week, V0 = 100 000.""" + s_h = 0.01 * np.sqrt(5) + assert rk.normal_var(0.0, s_h, 0.95) == pytest.approx(0.036780, abs=1e-6) + assert rk.normal_cvar(0.0, s_h, 0.95) == pytest.approx(0.046124, abs=1e-6) + assert rk.normal_var(0.0, s_h, 0.99) == pytest.approx(0.052019, abs=1e-6) + assert rk.normal_cvar(0.0, s_h, 0.99) == pytest.approx(0.059596, abs=1e-6) + v0 = 100_000.0 + assert rk.loss_to_money(rk.normal_var(0.0, s_h, 0.95), v0) == pytest.approx(3611.2, abs=0.1) + assert rk.loss_to_money(rk.normal_cvar(0.0, s_h, 0.95), v0) == pytest.approx(4507.6, abs=0.1) + assert rk.loss_to_money(rk.normal_var(0.0, s_h, 0.99), v0) == pytest.approx(5068.9, abs=0.1) + assert rk.loss_to_money(rk.normal_cvar(0.0, s_h, 0.99), v0) == pytest.approx(5785.5, abs=0.1) + + +def test_weighted_quantile_matches_numpy_and_handles_weights(): + rng = np.random.default_rng(3) + x = rng.standard_normal(5000) + qs = (0.01, 0.05, 0.5, 0.95) + np.testing.assert_allclose(rk.weighted_quantile(x, qs), np.quantile(x, qs), rtol=1e-12) + # equal weights reproduce the unweighted result (the weighted convention places the + # observations at the midpoints of their probability steps, so the tails differ by half a rank) + w = np.full(len(x), 1 / len(x)) + np.testing.assert_allclose(rk.weighted_quantile(x, qs, w), np.quantile(x, qs), atol=0.01) + # column-wise on a matrix + m = rng.standard_normal((2000, 3)) + np.testing.assert_allclose(rk.weighted_quantile(m, (0.1, 0.9)), np.quantile(m, (0.1, 0.9), axis=0), rtol=1e-12) + # a sample that over-represents the left tail, corrected by its weights, recovers the standard normal + tilted = np.concatenate([rng.standard_normal(4000), rng.standard_normal(4000) - 2.0]) + p = np.concatenate([np.full(4000, 1.0), stats.norm.pdf(tilted[4000:]) / stats.norm.pdf(tilted[4000:] + 2.0)]) + p = p / p.sum() + for q in (0.01, 0.05, 0.5): + assert rk.weighted_quantile(tilted, q, p)[0] == pytest.approx(stats.norm.ppf(q), abs=0.12) + + +def test_value_loss_maps_a_log_loss_to_a_share_of_the_position(): + assert rk.value_loss(0.0) == pytest.approx(0.0) + assert rk.value_loss(0.30) == pytest.approx(1 - np.exp(-0.30)) # 25.9 %, not 30 % + assert rk.value_loss(5.0) < 1.0 # can never reach 100 % + np.testing.assert_allclose(rk.value_loss([0.01, 0.1]), [1 - np.exp(-0.01), 1 - np.exp(-0.1)]) + # a money loss is the value-space loss times the position + assert rk.loss_to_money(0.30, 1_000.0) == pytest.approx(1_000.0 * rk.value_loss(0.30)) + + +def test_value_space_var_transforms_exactly_but_cvar_needs_the_sigma_squared_term(): + """``1 - e^{-x}`` is monotone, so a VaR carries over unchanged; a CVaR is a tail *mean* and + the concavity costs a ``sigma^2 / 2`` term -- the same correction as the Ito drift.""" + m, s, beta = 0.01, 0.30, 0.95 + r = np.random.default_rng(1).normal(m, s, 2_000_000) + loss = 1 - np.exp(r) + q = np.quantile(loss, beta) + assert rk.normal_value_var(m, s, beta) == pytest.approx(q, abs=2e-3) + assert rk.normal_value_var(m, s, beta) == pytest.approx(rk.value_loss(rk.normal_var(m, s, beta)), rel=1e-12) + exact = rk.normal_value_cvar(m, s, beta) + assert exact == pytest.approx(loss[loss >= q].mean(), abs=2e-3) + naive = rk.value_loss(rk.normal_cvar(m, s, beta)) # what converting afterwards would give + assert naive > exact # concavity: the naive figure overstates + assert 0.002 < naive - exact < 0.005 + # the correction vanishes as sigma -> 0 and grows with sigma + assert rk.value_loss(rk.normal_cvar(m, 0.01, beta)) - rk.normal_value_cvar(m, 0.01, beta) < 1e-5 + assert rk.value_loss(rk.normal_cvar(m, 0.6, beta)) - rk.normal_value_cvar(m, 0.6, beta) > 0.005 + assert rk.normal_value_cvar(m, 3.0, beta) < 1.0 # never a total loss, whatever the volatility + + +def test_asset_risk_table_in_value_space(): + rng = np.random.default_rng(5) + r = pd.DataFrame(rng.standard_normal((1500, 2)) * np.array([0.012, 0.03]), + index=pd.bdate_range("2019-01-01", periods=1500), columns=["A", "B"]) + log = rk.asset_risk_table(r, "daily", horizon="month", confidence=(0.95,)) + val = rk.asset_risk_table(r, "daily", horizon="month", confidence=(0.95,), space="value") + for col in ("var95_normal_ewm", "var95_hist"): + np.testing.assert_allclose(val[col], rk.value_loss(log[col]), rtol=1e-7) # quantiles: exact + for col in ("cvar95_normal_ewm", "cvar95_hist"): + assert (val[col] < rk.value_loss(log[col])).all() # tail means: corrected + assert (val[col] < 1.0).all() and (val[col] > 0).all() + np.testing.assert_allclose(val["vol_ewm_ann"], log["vol_ewm_ann"]) # only the losses move + with pytest.raises(ValueError): + rk.asset_risk_table(r, "daily", space="nonsense") + + +def test_zero_drift_removes_the_extrapolated_mean_from_the_parametric_risk(): + """At a quarterly horizon the estimated mean dominates the VaR: a stock with a strong recent + rally would otherwise look *less* risky than a quiet one.""" + rng = np.random.default_rng(11) + r = pd.DataFrame({"trend": rng.standard_normal(1500) * 0.02 + 0.004, # strong drift + "quiet": rng.standard_normal(1500) * 0.02}, + index=pd.bdate_range("2019-01-01", periods=1500)) + est_ = rk.asset_risk_table(r, "daily", horizon="quarter", confidence=(0.95,)) + zero = rk.asset_risk_table(r, "daily", horizon="quarter", confidence=(0.95,), drift="zero") + assert est_.loc["trend", "var95_normal_rolling"] < est_.loc["quiet", "var95_normal_rolling"] + assert zero.loc["trend", "var95_normal_rolling"] == pytest.approx( + zero.loc["quiet", "var95_normal_rolling"], rel=0.05) # same volatility -> same risk + assert zero.loc["trend", "var95_normal_rolling"] > est_.loc["trend", "var95_normal_rolling"] + # the historical columns keep the drift of their window either way + assert zero.loc["trend", "var95_hist"] == est_.loc["trend", "var95_hist"] + with pytest.raises(ValueError): + rk.asset_risk_table(r, "daily", drift="wishful") diff --git a/tests/test_rowgen_gaussian.py b/tests/test_rowgen_gaussian.py new file mode 100644 index 0000000..7b84d85 --- /dev/null +++ b/tests/test_rowgen_gaussian.py @@ -0,0 +1,275 @@ +"""Weighted scenarios, tail-row generation and the exact Gaussian (outer-approximation) mode.""" + +import itertools + +import numpy as np +import pytest +from scipy.stats import norm + +from quantcore.config import Settings +from quantcore.models import sampling as smp +from quantcore.optimization.gaussian_exact import solve_gaussian_exact +from quantcore.optimization.problem import ( + MeanCVaRProblem, + OptimizationConstraints, + ProblemSpec, + TransactionCosts, + cvar_multiplier, +) +from quantcore.optimization.rowgen import restricted_spec, solve_by_row_generation +from quantcore.optimization.solvers import HighsSolver + +SETTINGS = Settings(cuopt_url="http://127.0.0.1:9", mip_rel_gap=1e-6) + + +def highs(model): + return HighsSolver(SETTINGS).solve(model, time_limit=120) + + +def gaussian_inputs(seed=0, n=6): + rng = np.random.default_rng(seed) + symbols = [f"S{i}" for i in range(n)] + prices = rng.uniform(20, 400, n) + vol = rng.uniform(0.02, 0.06, n) + corr = 0.4 * np.ones((n, n)) + 0.6 * np.eye(n) + cov = corr * np.outer(vol, vol) + mean = rng.uniform(0.0, 0.006, n) + sectors = ["Tech", "Fin", "Health"] * (n // 3) + ["Tech"] * (n % 3) + return symbols, prices, mean, cov, sectors + + +def gaussian_scenarios(mean, cov, S, seed=1): + z = smp.moment_match(smp.standard_normal_sample(S, len(mean), "sobol", seed)) + return mean + z @ smp.pca_root(cov).T + + +def test_weighted_scenarios_equal_duplicated_scenarios(): + symbols, prices, mean, cov, sectors = gaussian_inputs() + scen = gaussian_scenarios(mean, cov, 256) + dup = np.vstack([scen, scen[:64]]) # the first 64 scenarios counted twice + p = np.ones(256) + p[:64] = 2 + p /= p.sum() + common = dict(symbols=symbols, prices=prices, sectors=sectors, cash=100_000.0, cvar_limit=3_000.0) + a = MeanCVaRProblem(ProblemSpec(scenarios=dup, **common)) + b = MeanCVaRProblem(ProblemSpec(scenarios=scen, probabilities=p, **common)) + ra, rb = highs(a.build()), highs(b.build()) + assert ra.ok and rb.ok + da, db = a.decode(ra.x), b.decode(rb.x) + assert abs(ra.objective - rb.objective) < 1e-6 * abs(ra.objective) + np.testing.assert_array_equal(da["shares"], db["shares"]) + assert abs(da["cvar"] - db["cvar"]) < 1e-6 and abs(da["expected_pnl"] - db["expected_pnl"]) < 1e-6 + + +def test_expected_returns_drive_the_objective(): + symbols, prices, mean, cov, sectors = gaussian_inputs() + scen = gaussian_scenarios(mean, cov, 256) + common = dict(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=100_000.0, cvar_limit=3_000.0, + costs=TransactionCosts(proportional_buy=0.0)) + shifted = mean + np.array([0.01, 0, 0, 0, 0, 0]) # the model says S0 earns 1 % more than the scenarios show + prob = MeanCVaRProblem(ProblemSpec(expected_returns=shifted, **common)) + res = highs(prob.build()) + d = prob.decode(res.x) + assert abs(d["expected_pnl"] - (shifted @ d["value"])) < 1e-6 # objective uses the model mean ... + assert d["expected_pnl"] > d["expected_pnl_scenarios"] + 0.005 * d["value"][0] - 1e-6 # ... not the sample mean + assert d["shares"][0] > 0 + with pytest.raises(ValueError): + ProblemSpec(expected_returns=np.zeros(3), **common) + + +def test_row_generation_matches_full_milp(): + symbols, prices, mean, cov, sectors = gaussian_inputs(seed=2) + scen = gaussian_scenarios(mean, cov, 2048) + spec = ProblemSpec(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=100_000.0, + cvar_limit=3_000.0, constraints=OptimizationConstraints(max_weight=0.3, max_sector_weight=0.6)) + full = MeanCVaRProblem(spec) + res_full = highs(full.build()) + res_rg, problem_rg, info = solve_by_row_generation(spec, highs) + assert res_rg.ok and info.converged and info.iterations >= 1 + assert max(info.working_set_sizes) < 0.5 * spec.n_scenarios # solved on a fraction of the rows + d_full, d_rg = full.decode(res_full.x), problem_rg.decode(res_rg.x) + assert abs(d_rg["expected_pnl"] - d_full["expected_pnl"]) <= 1e-5 * abs(d_full["expected_pnl"]) + 1e-6 + assert d_rg["cvar"] <= 3_000.0 * (1 + 1e-7) + # the expanded solution vector is feasible for the full model: u_s >= L_s - alpha for every scenario + m = full.build() + Ax = m.A @ res_rg.x + assert (Ax >= m.row_lb - 1e-6).all() and (Ax <= m.row_ub + 1e-6).all() + # a restricted spec keeps the full-set probabilities and expected returns + sub = restricted_spec(spec, np.arange(10)) + assert sub.n_scenarios == 10 and abs(sub.probabilities.sum() - 10 / 2048) < 1e-12 + np.testing.assert_allclose(sub.expected_returns, spec.mean_returns) + + +def test_row_generation_rebalancing_mode(): + symbols, prices, mean, cov, sectors = gaussian_inputs(seed=3) + scen = gaussian_scenarios(mean, cov, 1024) + holdings = np.array([40, 0, 10, 0, 5, 0], dtype=float) + spec = ProblemSpec(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=20_000.0, + holdings=holdings, cvar_limit=2_500.0, constraints=OptimizationConstraints(max_weight=0.3, max_sector_weight=0.6)) + res, problem, info = solve_by_row_generation(spec, highs) + assert res.ok and info.converged + d = problem.decode(res.x) + np.testing.assert_array_equal(d["shares"], holdings + d["buys"] - d["sells"]) + assert d["cvar"] <= 2_500.0 * (1 + 1e-7) and d["cash_end"] >= -1e-6 + + +def brute_force_gaussian(prices, mean, cov, sectors, cash, limit, beta, max_weight, costs): + """Enumerate all integer share vectors within the caps (small instance).""" + ub = np.floor(max_weight * cash / prices).astype(int) + k = cvar_multiplier(beta) + best, best_n = -np.inf, None + for shares in itertools.product(*[range(u + 1) for u in ub]): + s = np.array(shares, dtype=float) + v = s * prices + tc = costs * v.sum() + if v.sum() + tc > cash + 1e-9: + continue + pnl = mean @ v - tc + cvar = -pnl + k * np.sqrt(v @ cov @ v) + if cvar <= limit + 1e-9 and pnl > best: + best, best_n = pnl, s + return best, best_n + + +def test_gaussian_exact_matches_brute_force(): + prices = np.array([400.0, 650.0, 900.0]) + vol = np.array([0.03, 0.05, 0.04]) + corr = np.array([[1, 0.5, 0.2], [0.5, 1, 0.3], [0.2, 0.3, 1]]) + cov = corr * np.outer(vol, vol) + mean = np.array([0.003, 0.006, 0.004]) + sectors = ["A", "B", "C"] + cash, limit, beta = 10_000.0, 500.0, 0.95 + costs = TransactionCosts(proportional_buy=0.002) + spec = ProblemSpec(symbols=list("XYZ"), prices=prices, scenarios=None, sectors=sectors, cash=cash, cvar_limit=limit, + confidence=beta, gaussian_mean=mean, gaussian_cov=cov, costs=costs, + constraints=OptimizationConstraints(max_weight=0.6, max_sector_weight=1.0)) + res, problem, info = solve_gaussian_exact(spec, highs, tol=1e-9) + assert res.ok and info.converged and info.cuts >= 2 + d = problem.decode(res.x) + best, best_n = brute_force_gaussian(prices, mean, cov, sectors, cash, limit, beta, 0.6, 0.002) + np.testing.assert_array_equal(d["shares"], best_n) + assert abs(d["expected_pnl"] - best) < 1e-6 + assert d["cvar"] <= limit * (1 + 1e-9) + # closed-form risk figures of the decoded portfolio + v = d["value"] + sigma = np.sqrt(v @ cov @ v) + assert abs(d["sigma"] - sigma) < 1e-9 + assert abs(d["var"] - (-(mean @ v - d["total_cost"]) + norm.ppf(beta) * sigma)) < 1e-9 + assert res.reproducible and "outer approximation" in res.message + + +def test_gaussian_exact_other_objectives_and_validation(): + symbols, prices, mean, cov, sectors = gaussian_inputs(seed=4) + base = dict(symbols=symbols, prices=prices, scenarios=None, sectors=sectors, cash=100_000.0, + gaussian_mean=mean, gaussian_cov=cov) + # min CVaR at a target expected P&L: converges, meets the target, epigraph equals the true CVaR + spec = ProblemSpec(objective="min_cvar", target_pnl=150.0, **base) + res, problem, info = solve_gaussian_exact(spec, highs) + d = problem.decode(res.x) + assert info.converged and d["expected_pnl"] >= 150.0 - 1e-6 + assert abs(d["cvar"] - d["cvar_model"]) <= 1e-6 * max(d["cvar"], 1.0) + # utility objective + spec_u = ProblemSpec(objective="utility", risk_aversion=0.05, **base) # at 0.5 staying in cash is optimal + res_u, problem_u, info_u = solve_gaussian_exact(spec_u, highs) + du = problem_u.decode(res_u.x) + assert info_u.converged and du["invested"] > 0 + # the CVaR-limited solution is on the mean-variance frontier: no feasible portfolio with the same + # expected P&L has a smaller sigma (checked on the continuous relaxation via the cut model) + spec_m = ProblemSpec(cvar_limit=3_000.0, **base) + res_m, problem_m, _ = solve_gaussian_exact(spec_m, highs) + dm = problem_m.decode(res_m.x) + assert dm["cvar"] <= 3_000.0 * (1 + 1e-6) + with pytest.raises(ValueError): + ProblemSpec(symbols=symbols, prices=prices, scenarios=None, sectors=sectors, cash=1.0, cvar_limit=1.0) + with pytest.raises(ValueError): + MeanCVaRProblem(spec_m).build() # no cuts yet + with pytest.raises(ValueError): + solve_by_row_generation(spec_m, highs) + + +def test_moment_matched_sobol_scenarios_reproduce_the_exact_optimum(): + """The sample-average approximation with Gaussian scenarios (used linearly) converges to the exact + mean-variance-equivalent optimum; plain Monte Carlo at the same size does not.""" + symbols, prices, mean, cov, sectors = gaussian_inputs(seed=6, n=8) + common = dict(symbols=symbols, prices=prices, sectors=sectors, cash=1_000_000.0, cvar_limit=30_000.0, + costs=TransactionCosts(proportional_buy=0.0), + constraints=OptimizationConstraints(max_weight=0.25, max_sector_weight=0.6)) + exact_spec = ProblemSpec(scenarios=None, gaussian_mean=mean, gaussian_cov=cov, **common) + res_e, prob_e, _info = solve_gaussian_exact(exact_spec, highs, tol=1e-8) + d_e = prob_e.decode(res_e.x) + k = cvar_multiplier(0.95) + + def true_cvar(v): + return -(mean @ v) + k * np.sqrt(v @ cov @ v) + + # moment-matched Sobol' scenarios, model mean in the objective + scen = gaussian_scenarios(mean, cov, 4096, seed=3) + prob_s = MeanCVaRProblem(ProblemSpec(scenarios=scen, expected_returns=mean, **common)) + d_s = prob_s.decode(highs(prob_s.build()).x) + assert abs(d_s["expected_pnl"] - d_e["expected_pnl"]) <= 0.01 * d_e["expected_pnl"] + assert true_cvar(d_s["value"]) <= 30_000.0 * 1.02 + # plain Monte Carlo of the same size with the sample mean in the objective (the old default) + rng = np.random.default_rng(3) + mc = mean + rng.standard_normal((4096, 8)) @ np.linalg.cholesky(cov).T + prob_m = MeanCVaRProblem(ProblemSpec(scenarios=mc, **common)) + d_m = prob_m.decode(highs(prob_m.build()).x) + true_pnl_mc = mean @ d_m["value"] + assert abs(true_pnl_mc - d_e["expected_pnl"]) > abs(d_s["expected_pnl"] - d_e["expected_pnl"]) + assert d_m["expected_pnl"] > true_pnl_mc # in-sample optimism of the plain MC solution + + +def test_row_generation_stops_at_the_time_budget(): + """The time limit is the budget of the whole request, not of each pass. + + Every pass is a MILP of its own, so a deadline that has already passed must end the loop with + the incumbent (feasible, not proven optimal) instead of starting another solve. + """ + import time + + symbols, prices, mean, cov, sectors = gaussian_inputs(seed=2) + scen = gaussian_scenarios(mean, cov, 2048) + spec = ProblemSpec(symbols=symbols, prices=prices, scenarios=scen, sectors=sectors, cash=100_000.0, + cvar_limit=3_000.0, constraints=OptimizationConstraints(max_weight=0.3, max_sector_weight=0.6)) + calls = [] + + def counted(model): + calls.append(1) + return highs(model) + + res, problem, info = solve_by_row_generation(spec, counted, deadline=time.monotonic() - 1.0) + assert len(calls) == 1 and info.iterations == 1 and not info.converged + assert res.status == "FeasibleFound" and "time limit" in res.message + d = problem.decode(res.x) # the incumbent is a real portfolio, measured on all scenarios + assert d["shares"].sum() > 0 and np.isfinite(d["cvar"]) + # with a budget in the future the loop runs to convergence as before + res2, _, info2 = solve_by_row_generation(spec, highs, deadline=time.monotonic() + 600) + assert info2.converged and res2.status == "Optimal" + + +def test_the_solve_function_hands_on_the_remaining_budget(): + from quantcore.optimization.service import OptimizationRequest, OptimizationService + + svc = OptimizationService.__new__(OptimizationService) # no data needed for this + svc.settings = SETTINGS + req = OptimizationRequest(time_limit=30.0, solver="highs", cvar_limit=1.0) + limits = [] + deadline = __import__("time").monotonic() + 30.0 + + class Recorder: + def __init__(self, settings): + pass + + def solve(self, model, time_limit=None): + limits.append(time_limit) + return None + + import quantcore.optimization.service as service_module + original = service_module.HighsSolver + service_module.HighsSolver = Recorder + try: + fn = svc._solve_fn(req, prefer_local=True, deadline=deadline) + fn(None) + fn(None) + finally: + service_module.HighsSolver = original + assert 0 < limits[1] <= limits[0] <= 30.0 # each call gets what is left, never the full limit again diff --git a/tests/test_sampling.py b/tests/test_sampling.py new file mode 100644 index 0000000..fac1ffa --- /dev/null +++ b/tests/test_sampling.py @@ -0,0 +1,122 @@ +import numpy as np +import pytest +from scipy.stats import norm + +from quantcore.models import sampling as smp + + +def test_next_power_of_two(): + assert [smp.next_power_of_two(n) for n in (1, 2, 3, 2000, 2048, 2049, 100_000)] == [1, 2, 4, 2048, 2048, 4096, 131072] + + +def test_standard_normal_sample_methods(): + for method in smp.SAMPLING_METHODS: + z = smp.standard_normal_sample(1024, 5, method, seed=3) + assert z.shape == (1024, 5) and np.isfinite(z).all() + anti = smp.standard_normal_sample(1000, 3, "antithetic", seed=1) + np.testing.assert_allclose(anti[:500], -anti[500:], atol=1e-15) # exact zero mean by construction + with pytest.raises(ValueError): + smp.standard_normal_sample(1000, 3, "sobol") # not a power of two + with pytest.raises(ValueError): + smp.standard_normal_sample(1024, 3, "bogus") + # Sobol' points are deterministic for a seed and stratify the marginals far better than MC + a = smp.standard_normal_sample(2048, 4, "sobol", seed=7) + b = smp.standard_normal_sample(2048, 4, "sobol", seed=7) + np.testing.assert_array_equal(a, b) + lhs = smp.standard_normal_sample(4096, 2, "lhs", seed=7) + counts = np.histogram(norm.cdf(lhs[:, 0]), bins=64, range=(0, 1))[0] + assert counts.min() == counts.max() == 64 # exactly 64 points per stratum + + +def test_pca_root(): + rng = np.random.default_rng(0) + L = rng.standard_normal((6, 6)) + cov = L @ L.T + a = smp.pca_root(cov) + np.testing.assert_allclose(a @ a.T, cov, atol=1e-10) + vals, vecs = np.linalg.eigh(cov) + # first column = largest-variance direction, columns orthogonal + assert abs(abs(a[:, 0] @ vecs[:, -1]) - np.sqrt(vals[-1])) < 1e-9 + np.testing.assert_allclose(a.T @ a, np.diag(np.sort(vals)[::-1]), atol=1e-9) + + +def test_moment_match_exact_and_weighted(): + rng = np.random.default_rng(1) + z = rng.standard_normal((500, 7)) * 1.3 + 0.2 + m = smp.moment_match(z) + np.testing.assert_allclose(m.mean(axis=0), 0.0, atol=1e-13) + np.testing.assert_allclose(m.T @ m / len(m), np.eye(7), atol=1e-12) + p = rng.uniform(0.5, 2.0, 500) + p /= p.sum() + mw = smp.moment_match(z, p, target_variance=5.0) + np.testing.assert_allclose(p @ mw, 0.0, atol=1e-12) + np.testing.assert_allclose((mw * p[:, None]).T @ mw, 5.0 * np.eye(7), atol=1e-11) + with pytest.raises(ValueError): + smp.moment_match(rng.standard_normal((5, 7))) + + +def test_tilt_normal_sample_weights_bounded_and_tail_enriched(): + rng = np.random.default_rng(2) + n, d = 4096, 8 + z = smp.standard_normal_sample(n, d, "sobol", seed=5) + cov = np.eye(d) * 0.01 + root = smp.pca_root(cov) + exposure = rng.uniform(1, 3, d) * 1000 + direction = smp.loss_direction(root, exposure) + assert abs(np.linalg.norm(direction) - 1) < 1e-12 + beta = 0.95 + zt, p = smp.tilt_normal_sample(z, direction, norm.ppf(beta), mixture=0.5) + assert p.shape == (n,) and abs(p.sum() - 1) < 1e-12 and (p > 0).all() + assert p.max() <= 1.0 / (1 - 0.5) / n * 1.1 # likelihood ratios bounded by 1/(1-mixture), self-normalised + ess = smp.effective_sample_size(p) + assert 0.5 * n < ess < n + # the loss of the reference exposure exceeds its VaR far more often than 5 % of the time + loss_nominal = -(z @ root.T) @ exposure + loss_tilted = -(zt @ root.T) @ exposure + var = np.quantile(loss_nominal, beta) + assert (loss_tilted > var).mean() > 0.2 + # ... while the weighted tail probability is still ~ 1 - beta (unbiased) + assert abs(p[loss_tilted > var].sum() - (1 - beta)) < 0.01 + with pytest.raises(ValueError): + smp.tilt_normal_sample(z, direction, 1.0, mixture=1.0) + + +def test_tilted_day_probabilities_and_resampling(): + rng = np.random.default_rng(3) + scores = rng.standard_t(5, 2000) + q, theta = smp.tilted_day_probabilities(scores, target_mean=0.7) + assert theta > 0 and abs(q @ scores - 0.7) < 1e-8 and abs(q.sum() - 1) < 1e-12 + q0, theta0 = smp.tilted_day_probabilities(scores, target_mean=-1.0) + assert theta0 == 0.0 and np.allclose(q0, 1 / 2000) + with pytest.raises(ValueError): + smp.tilted_day_probabilities(scores, target_mean=scores.max() + 1) + idx, p, _theta = smp.tilted_resample(scores, 1000, 5, 0.7, seed=1, mixture=0.5) + assert idx.shape == (1000, 5) and abs(p.sum() - 1) < 1e-12 + assert p.max() <= 2.0 / 1000 * 1.1 # likelihood ratios <= 1/(1-mixture), self-normalised + # tilted half draws high scores more often + assert scores[idx[:500]].mean() > scores[idx[500:]].mean() + 0.3 + + +def test_resample_indices_methods(): + T, S, h = 300, 1024, 5 + mc = smp.resample_indices(T, S, h, "mc", seed=0) + assert mc.shape == (S, h) and mc.min() >= 0 and mc.max() < T + bal = smp.resample_indices(T, S, h, "balanced", seed=0) + counts = np.bincount(bal.ravel(), minlength=T) + assert counts.max() - counts.min() <= 1 # every observation used equally often + order = np.argsort(np.random.default_rng(1).standard_normal(T)) + sob = smp.resample_indices(T, S, h, "sobol", seed=0, order=order) + assert sob.shape == (S, h) and set(np.unique(sob)) <= set(range(T)) + ranks = smp.resample_indices(T, S, h, "sobol", seed=0) + np.testing.assert_array_equal(order[ranks], sob) + with pytest.raises(ValueError): + smp.resample_indices(T, 1000, h, "sobol") + + +def test_scenario_hash(): + rng = np.random.default_rng(4) + x = rng.standard_normal((50, 3)) + h = smp.scenario_hash(x) + assert len(h) == 16 and h == smp.scenario_hash(x.copy()) + assert h != smp.scenario_hash(x + 1e-9) + assert h != smp.scenario_hash(x, np.full(50, 0.02)) diff --git a/tests/test_service_equivalence.py b/tests/test_service_equivalence.py new file mode 100644 index 0000000..2f7ee46 --- /dev/null +++ b/tests/test_service_equivalence.py @@ -0,0 +1,281 @@ +"""Service-level checks: the scenario optimiser reproduces the exact Gaussian (mean-variance-equivalent) +optimum when the scenarios are moment-matched Sobol' points, the drift convention is arithmetic, and +importance sampling / row generation leave the optimum unchanged.""" + +import numpy as np +import pandas as pd +import pytest + +from quantcore.optimization.problem import MeanCVaRProblem +from quantcore.optimization.service import OptimizationRequest, OptimizationService +from tests.conftest import SYMBOLS + +BASE = dict(symbols=SYMBOLS, cash=500_000.0, cvar_limit=12_000.0, solver="highs", horizon="week", + costs={"proportional_buy": 0.0, "proportional_sell": 0.0}, importance_sampling="off", + constraints={"max_weight": 0.15, "max_sector_weight": 0.4}) + + +@pytest.fixture(scope="module") +def svc(settings, md): + return OptimizationService(settings, md) + + +def gaussian_cvar_of(svc, req, result): + """Closed-form CVaR of a proposal under the Gaussian model of the request.""" + symbols = svc.candidate_symbols(req) + mu_ann, _, _ = svc.expected_returns(req, symbols) + spec, _ = svc.build_gaussian_spec(req, symbols, mu_ann) + shares = {p["symbol"]: p["shares"] for p in result.positions} + value = np.array([shares.get(s, 0) for s in symbols]) * spec.prices + return MeanCVaRProblem(spec).gaussian_risk(value, result.total_cost) + + +def test_sobol_moment_matched_scenarios_reproduce_the_exact_optimum(svc): + exact = svc.optimize(OptimizationRequest(scenario_model="gaussian_exact", **BASE)) + assert exact.status == "Optimal" and exact.reproducible and exact.solver_info["outer_approximation"]["converged"] + scen = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **BASE)) + assert scen.status == "Optimal" and scen.model["scenarios"]["moment_matched"] + # The GBM scenarios are lognormal in simple returns while the exact mode is normal: at weekly + # volatilities of 4-5 % the lognormal left tail is ~2-3 % thinner, so the scenario optimum takes + # ~3 % more exposure. Within that model difference the two agree (the sharp test of the sampling + # itself, with Gaussian scenarios used linearly, is in test_rowgen_gaussian.py). + assert abs(scen.expected_pnl - exact.expected_pnl) <= 0.05 * exact.expected_pnl + _, cvar_true, _ = gaussian_cvar_of(svc, OptimizationRequest(scenario_model="gaussian_exact", **BASE), scen) + assert cvar_true <= BASE["cvar_limit"] * 1.05 + # the expected horizon return reported per position is the model's (arithmetic) expectation + mu_ann, _, _ = svc.expected_returns(OptimizationRequest(**BASE), SYMBOLS) + for p in scen.positions: + assert abs(p["expected_return_horizon"] - mu_ann[p["symbol"]] / 252 * 5) < 1e-12 + # plain Monte Carlo without moment matching: the sample mean disagrees with the model mean + mc = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, sampling="mc", moment_match=False, **BASE)) + assert mc.status == "Optimal" + dev_mc = abs(mc.solver_info["objective"] - mc.expected_pnl) # objective uses the model mean either way + assert dev_mc < 1e-6 # (the objective is always the model expectation) ... + scen_mean_mc = svc.build_spec(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, sampling="mc", + moment_match=False, **BASE), SYMBOLS, + svc.scenarios(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, sampling="mc", + moment_match=False, **BASE), SYMBOLS, mu_ann)) + # ... but the sample mean of a plain MC set is visibly off the model mean, a moment-matched one is not + err_mc = np.abs(scen_mean_mc.scenarios.mean(axis=0) - scen_mean_mc.expected_returns).max() + sobol_spec = svc.build_spec(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **BASE), SYMBOLS, + svc.scenarios(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **BASE), SYMBOLS, mu_ann)) + err_sobol = np.abs(sobol_spec.scenarios.mean(axis=0) - sobol_spec.expected_returns).max() + assert err_sobol < 0.1 * err_mc + + +def test_importance_sampling_and_row_generation_keep_the_optimum(svc): + plain = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **BASE)) + tilted = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, + **{**BASE, "importance_sampling": "market"})) + assert tilted.status == "Optimal" and tilted.model["scenarios"]["weighted"] + assert tilted.model["scenarios"]["effective_sample_size"] > 2000 + assert abs(tilted.expected_pnl - plain.expected_pnl) <= 0.02 * plain.expected_pnl + exact_req = OptimizationRequest(scenario_model="gaussian_exact", **BASE) + assert gaussian_cvar_of(svc, exact_req, tilted)[1] <= BASE["cvar_limit"] * 1.05 + rg = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **{**BASE, "row_generation": "on"})) + assert rg.status == "Optimal" and rg.solver_info["row_generation"]["converged"] + assert rg.scenario_hash == plain.scenario_hash # same scenario set ... + assert abs(rg.expected_pnl - plain.expected_pnl) <= 1e-4 * plain.expected_pnl # ... same optimum + assert max(rg.solver_info["row_generation"]["working_set_sizes"]) < 4096 + + +def test_empirical_model_options(svc): + req = OptimizationRequest(scenario_model="empirical", n_scenarios=2048, **{**BASE, "importance_sampling": "market"}) + res = svc.optimize(req) + assert res.status == "Optimal" and res.model["scenarios"]["model"] == "empirical" + assert all(t > 0 for t in res.model["scenarios"]["importance_sampling"]["theta"]) + assert res.model["scenarios"]["importance_sampling"]["direction"] == "market" + assert res.cvar <= BASE["cvar_limit"] * (1 + 1e-6) + bal = svc.optimize(OptimizationRequest(scenario_model="empirical", n_scenarios=1000, sampling="balanced", **BASE)) + assert bal.status == "Optimal" and bal.model["scenarios"]["n_scenarios"] == 1000 + with pytest.raises(ValueError): + OptimizationRequest(scenario_model="bogus", **BASE) + with pytest.raises(ValueError): + OptimizationRequest(**{**BASE, "importance_sampling": "maybe"}) + + +def test_one_scenario_set_for_a_family_of_runs(svc): + """Risk level, constraints, costs, objective, cash and holdings must not change the scenarios; + a different expected-return model must change only their drift, not their shocks.""" + svc.clear_scenario_cache() + base = dict(BASE, importance_sampling="market", scenario_model="gbm", n_scenarios=2048) + ref = svc.optimize(OptimizationRequest(**base)) + variants = { + "other CVaR limit": dict(base, cvar_limit=6_000.0), + "limit as a fraction": dict(base, cvar_limit=None, cvar_limit_fraction=0.02), + "confidence 99 %": dict(base, confidence=0.99, cvar_limit=20_000.0), + "tighter caps": dict(base, constraints={"max_weight": 0.10, "max_sector_weight": 0.3}), + "with costs": dict(base, costs={"proportional_buy": 0.002, "proportional_sell": 0.002}), + "min-CVaR objective": dict(base, objective="min_cvar", cvar_limit=None, target_return=0.001), + "utility objective": dict(base, objective="utility", cvar_limit=None, risk_aversion=0.2), + "more cash": dict(base, cash=2_000_000.0), + "rebalancing": dict(base, mode="rebalance", cash=50_000.0, + holdings={p["symbol"]: p["shares"] for p in ref.positions[:3]}), + } + for label, kwargs in variants.items(): + res = svc.optimize(OptimizationRequest(**kwargs)) + assert res.status == "Optimal", label + assert res.scenario_key["id"] == ref.scenario_key["id"], label + assert res.scenario_hash == ref.scenario_hash, label + assert res.solver_info["scenario_cache"]["reused"], label # and it was not regenerated + # a different expected-return model shifts the scenarios but leaves the shocks alone + symbols = svc.candidate_symbols(OptimizationRequest(**base)) + req = OptimizationRequest(**base) + mu_a, _, _ = svc.expected_returns(req, symbols) + scen_a = svc.scenarios(req, symbols, mu_a) + req_views = OptimizationRequest(**dict(base, views=[{"symbol": SYMBOLS[0], "sentiment": 0.9, "confidence": 0.8}])) + mu_b, _, _ = svc.expected_returns(req_views, symbols) + scen_b = svc.scenarios(req_views, symbols, mu_b) + assert scen_b.hash != scen_a.hash and scen_b.shock_hash == scen_a.shock_hash + np.testing.assert_allclose(scen_b.shocks, scen_a.shocks, atol=1e-15) + assert (scen_b.expected_simple[0] > scen_a.expected_simple[0]) # the view moved the drift + # what *must* change the set: symbols, horizon, covariance model, seed, sampling, scenario count + for label, kwargs in { + "fewer symbols": dict(base, symbols=SYMBOLS[:8]), + "monthly horizon": dict(base, horizon="month"), + "other covariance": dict(base, covariance={"method": "rolling", "window": 250}), + "other seed": dict(base, seed=7), + "plain Monte Carlo": dict(base, sampling="mc"), + "more scenarios": dict(base, n_scenarios=4096), + "no tilt": dict(base, importance_sampling="off"), + }.items(): + other = OptimizationRequest(**kwargs) + assert svc.scenario_key(other, svc.candidate_symbols(other))["id"] != ref.scenario_key["id"], label + # the adaptive tilt is honest about depending on the risk parameters + ad = OptimizationRequest(**dict(base, importance_sampling="adaptive")) + ad_other = OptimizationRequest(**dict(base, importance_sampling="adaptive", cvar_limit=6_000.0)) + assert svc.scenario_key(ad, symbols)["id"] != svc.scenario_key(ad_other, symbols)["id"] + assert "adaptive_tilt" in svc.scenario_key(ad, symbols) + + +def test_tilt_is_driven_by_the_covariance_and_reports_its_alignment(svc): + """The tilt shifts each asset by its covariance with the reference portfolio (not by its own + volatility), leaves the model correlations exactly intact, and the result says how well the + solution's loss lines up with the tilted direction.""" + from scipy.stats import norm + + from quantcore.models import sampling as smp + from quantcore.models.covariance import CovarianceModel + from quantcore.models.simulation import GBMSimulator, TiltSpec + from quantcore.optimization.problem import OptimizationConstraints, ProblemSpec, TransactionCosts + + req = OptimizationRequest(**dict(BASE, scenario_model="gbm", n_scenarios=2048, importance_sampling="market")) + symbols = svc.candidate_symbols(req) + r, cov_model = svc._returns_and_cov_model(req, symbols) + h = req.horizon_periods + cov_h = cov_model.estimate(r).values * h + root = smp.pca_root(cov_h) + reference = svc.market_exposure(symbols) * 1e6 + + # 1. the shift of a tilted block is exactly -z_beta * Sigma v / sigma: proportional to each + # asset's covariance with the reference, i.e. to its beta, not to its own volatility + z = norm.ppf(0.95) + direction = smp.loss_direction(root, reference) + shift = z * (root @ direction) + sigma = np.sqrt(reference @ cov_h @ reference) + np.testing.assert_allclose(shift, -z * (cov_h @ reference) / sigma, atol=1e-12) + beta_i = (cov_h @ reference) / (reference @ cov_h @ reference) + assert np.corrcoef(shift, beta_i)[0, 1] < -0.999 # perfectly ordered by beta + if np.std(np.sqrt(np.diag(cov_h))) > 0: # ... and only loosely by own volatility + assert np.corrcoef(shift, np.sqrt(np.diag(cov_h)))[0, 1] > -0.999 + + # 2. the weighted tilted sample has exactly the model covariance (and correlations) + mu_ann, _, _ = svc.expected_returns(req, symbols) + sim = GBMSimulator(cov_model) + tilted = sim.simulate(r, n_scenarios=2048, horizon_periods=h, mu=mu_ann / 252, seed=1, + tilt=TiltSpec(exposure=reference)) + w = tilted.weights + centred = tilted.log_returns - w @ tilted.log_returns + cov_t = (centred * w[:, None]).T @ centred + np.testing.assert_allclose(cov_t, cov_h, rtol=1e-8, atol=1e-14) + unweighted = np.cov(tilted.log_returns, rowvar=False) + assert np.abs(unweighted - cov_h).max() > 1e-6 # the raw tilted sample is *not* the model + + # 3. alignment: 1 for the reference itself, negative for a portfolio built to oppose it + spec = ProblemSpec(scenarios=tilted.simple_returns(), probabilities=tilted.probabilities, + expected_returns=tilted.expected_simple, **svc._spec_common(req, symbols)) + assert abs(svc.tilt_alignment(spec, reference, svc.market_exposure(symbols)) - 1) < 1e-9 # itself + opposed = np.zeros(len(symbols)) + opposed[np.argsort(beta_i)[:2]] = 1e5 # the two assets least exposed to the reference + assert svc.tilt_alignment(spec, opposed, svc.market_exposure(symbols)) < 0.9 + # a real run reports the figure and stays well above the warning threshold + res = svc.optimize(req) + assert res.solver_info["tilt_alignment"] > 0.5 and not res.warnings + + +def test_risk_drift_zero_keeps_the_limit_binding_when_the_forecast_is_large(md, settings): + """Over a long horizon a big expected return shifts the whole tail into profit, and a CVaR limit + measured on the drifted scenarios stops constraining anything. The zero-drift convention measures + risk on the centred scenarios -- so the risk figure does not depend on the return forecast at all + -- while the objective still uses it.""" + svc = OptimizationService(settings, md) + base = dict(symbols=SYMBOLS[:8], cash=1_000_000.0, horizon=252, freq="daily", confidence=0.99, + cvar_limit_fraction=0.12, scenario_model="gbm", n_scenarios=2048, solver="highs", + expected_returns="zero", row_generation="off", importance_sampling="off") + symbols = svc.candidate_symbols(OptimizationRequest(**base)) + v = np.full(len(symbols), 125_000.0) # an equally weighted portfolio of the whole wealth + + risk = {} + for mu_val in (0.8, 1.5): # 80 % and 150 % p.a. forecasts + for mode in ("model", "zero"): + req = OptimizationRequest(**{**base, "risk_drift": mode}) + spec = svc.build_spec(req, symbols, svc.scenarios(req, symbols, pd.Series(mu_val, index=symbols))) + risk[(mu_val, mode)] = OptimizationService.portfolio_risk_on_scenarios(spec, v / spec.prices, 0.99) + if mode == "zero": + np.testing.assert_allclose(spec.mean_returns, svc.build_spec( + OptimizationRequest(**{**base, "risk_drift": "model"}), symbols, + svc.scenarios(OptimizationRequest(**base), symbols, pd.Series(mu_val, index=symbols)) + ).mean_returns) # the objective sees the forecast either way + + # the zero-drift risk is the same whatever the forecast; the drifted one is not + assert risk[(0.8, "zero")]["cvar"] == pytest.approx(risk[(1.5, "zero")]["cvar"], rel=1e-12) + assert risk[(1.5, "model")]["cvar"] < 0 < risk[(1.5, "zero")]["cvar"] # the tail became a gain + assert risk[(0.8, "model")]["cvar"] < risk[(0.8, "zero")]["cvar"] + # the drift enters multiplicatively (e^{r+d} - 1), so the gap is well below mu * V + gap = risk[(1.5, "zero")]["cvar"] - risk[(1.5, "model")]["cvar"] + assert 0.3 * 1.5e6 < gap < 1.5e6 + + res = svc.optimize(OptimizationRequest(**{**base, "expected_returns": "historical"})) + if res.cvar <= 0: # the historical forecast is large enough to make the limit vacuous + assert any("constrains nothing" in w for w in res.warnings) + + +def test_an_all_cash_utility_solution_says_why(md, settings): + """objective='utility' with a large lambda is optimal in cash -- which must not look like a bug.""" + from quantcore.optimization.service import OptimizationRequest, OptimizationService + + svc = OptimizationService(settings, md) + common = dict(cash=1_000_000.0, max_assets=6, horizon="week", scenario_model="gaussian_exact", + solver="highs", time_limit=120, objective="utility") + big = svc.optimize(OptimizationRequest(risk_aversion=50.0, **common)) + assert big.status in ("Optimal", "FeasibleFound") and big.invested == 0 + assert any("nothing was invested" in w and "risk_aversion" in w for w in big.warnings) + small = svc.optimize(OptimizationRequest(risk_aversion=0.0, **common)) + assert small.invested > 0 and not any("nothing was invested" in w for w in small.warnings) + + +def test_a_weighted_cvar_objective_is_routed_to_highs_and_says_so(md, settings): + """cuOpt aborts when the CVaR is the objective *and* the scenarios carry probabilities. + + The routing is that narrow on purpose: the same objective with uniform weights solves on the GPU, + so only the weighted case may be diverted, and only under solver='auto'. + """ + from quantcore.optimization.service import OptimizationRequest, OptimizationService + + svc = OptimizationService(settings, md) + common = dict(cash=1_000_000.0, max_assets=6, horizon="week", scenario_model="gbm", + n_scenarios=1024, solver="auto", time_limit=120, risk_drift="zero") + weighted = svc.optimize(OptimizationRequest(objective="min_cvar", target_return=0.0005, + importance_sampling="market", **common)) + assert weighted.solver == "highs" + assert any("probability-weighted" in w for w in weighted.warnings) + + # uniform weights: no diversion (cuOpt is unreachable in the tests, so it still lands on HiGHS -- + # through the fallback, which says something different) + plain = svc.optimize(OptimizationRequest(objective="min_cvar", target_return=0.0005, + importance_sampling="off", **common)) + assert not any("probability-weighted" in w for w in plain.warnings) + # the constrained form is never diverted either + other = svc.optimize(OptimizationRequest(objective="max_return", cvar_limit_fraction=0.05, + importance_sampling="market", **common)) + assert not any("probability-weighted" in w for w in other.warnings) diff --git a/tests/test_simulation.py b/tests/test_simulation.py new file mode 100644 index 0000000..1e5be9b --- /dev/null +++ b/tests/test_simulation.py @@ -0,0 +1,197 @@ +import numpy as np +import pandas as pd +import pytest + +from quantcore.models.covariance import CovarianceModel, cholesky_factor, symmetric_inv_sqrt, symmetric_sqrt +from quantcore.models.simulation import EmpiricalInnovationSimulator, GBMSimulator + + +@pytest.fixture +def r(): + rng = np.random.default_rng(7) + L = np.array([[1.0, 0, 0], [0.6, 0.8, 0], [0.3, 0.2, 0.9]]) + x = rng.standard_t(6, size=(1500, 3)) @ L.T * 0.01 + return pd.DataFrame(x, index=pd.bdate_range("2019-01-01", periods=1500), columns=list("ABC")) + + +def test_matrix_roots(r): + cov = r.cov().values + s = symmetric_sqrt(cov) + np.testing.assert_allclose(s @ s, cov, atol=1e-12) + np.testing.assert_allclose(symmetric_inv_sqrt(cov) @ s, np.eye(3), atol=1e-8) + L = cholesky_factor(cov) + np.testing.assert_allclose(L @ L.T, cov, atol=1e-12) + + +def test_gbm_scenarios_have_target_moments(r): + cm = CovarianceModel("rolling", window=500) + mu = pd.Series([0.001, 0.0, -0.001], index=list("ABC")) + scen = GBMSimulator(cm).simulate(r, n_scenarios=100_000, horizon_periods=5, mu=mu, seed=1) + target = cm.estimate(r).values * 5 + # Sobol' sets are rounded up to a power of two; moment matching makes the covariance exact + assert scen.model == "gbm" and scen.n_scenarios == 131_072 and scen.horizon_periods == 5 + assert scen.extra["n_requested"] == 100_000 and scen.extra["sampling"] == "sobol" + np.testing.assert_allclose(np.cov(scen.log_returns, rowvar=False, ddof=0), target, rtol=1e-8, atol=1e-14) + # arithmetic drift convention: the expected simple return over the horizon is mu * h (Ito correction) + np.testing.assert_allclose(scen.expected_simple, mu.values * 5) + np.testing.assert_allclose(scen.mean_simple_returns(), mu.values * 5, atol=3e-5) + # log convention: the log-drift is mu * h exactly + scen_log = GBMSimulator(cm).simulate(r, n_scenarios=4096, horizon_periods=5, mu=mu, seed=1, drift_convention="log") + np.testing.assert_allclose(scen_log.log_returns.mean(axis=0), mu.values * 5, atol=1e-12) + assert (scen_log.expected_simple > np.expm1(mu.values * 5)).all() # sigma^2/2 bonus under the log convention + # plain Monte Carlo without moment matching: the old behaviour, moments only approximately right + mc = GBMSimulator(cm).simulate(r, n_scenarios=100_000, horizon_periods=5, mu=mu, seed=1, sampling="mc", + moment_match=False) + assert mc.n_scenarios == 100_000 + np.testing.assert_allclose(np.cov(mc.log_returns, rowvar=False), target, rtol=0.05, atol=1e-6) + + +def test_innovations_are_standardised(r): + sim = EmpiricalInnovationSimulator(CovarianceModel("ewm", lam=0.97)) + eps = sim.innovations(r) + assert eps.shape[1] == 3 and len(eps) > 1000 + cov = np.cov(eps.values, rowvar=False) + np.testing.assert_allclose(cov, np.eye(3), atol=0.15) + + +def test_empirical_scenarios_match_latest_covariance(r): + cm = CovarianceModel("ewm", lam=0.97) + sim = EmpiricalInnovationSimulator(cm) + scen = sim.simulate(r, n_scenarios=60_000, horizon_periods=3, seed=2, sampling="mc", moment_match=False) + target = cm.estimate(r).values * 3 + np.testing.assert_allclose(np.cov(scen.log_returns, rowvar=False), target, rtol=0.2, atol=2e-6) + # zero expected return: the log-drift is the Ito correction, the mean simple return ~ 0 + np.testing.assert_allclose(scen.mean_simple_returns(), 0.0, atol=5e-4) + exact = sim.simulate(r, n_scenarios=60_000, horizon_periods=3, seed=2) # Sobol' ranks + moment matching + assert exact.n_scenarios == 65_536 + np.testing.assert_allclose(np.cov(exact.log_returns, rowvar=False, ddof=0), target, rtol=1e-8, atol=1e-14) + # fat tails survive the filter: kurtosis of the innovations above the Gaussian 0 + from scipy.stats import kurtosis + assert kurtosis(sim.innovations(r).values).mean() > 0.3 + + +def test_cholesky_root_variant(r): + sim = EmpiricalInnovationSimulator(CovarianceModel("rolling", window=250), root="cholesky") + eps = sim.innovations(r) + np.testing.assert_allclose(np.cov(eps.values, rowvar=False), np.eye(3), atol=0.15) + scen = sim.simulate(r, n_scenarios=5000, horizon_periods=1, seed=3, sampling="mc") + assert scen.extra["root"] == "cholesky" and scen.log_returns.shape == (5000, 3) + + +def test_restandardisation_reproduces_model_covariance(r): + cm = CovarianceModel("ewm", lam=0.97, shrinkage=0.0) + for mode in ("diagonal", "full"): + sim = EmpiricalInnovationSimulator(cm, standardize=mode) + scen = sim.simulate(r, n_scenarios=80_000, horizon_periods=1, seed=5, sampling="mc", moment_match=False) + model = cm.estimate(r).values + sc = np.cov(scen.log_returns, rowvar=False) + np.testing.assert_allclose(np.sqrt(np.diag(sc)), np.sqrt(np.diag(model)), rtol=0.03) + assert scen.extra["standardize"] == mode and scen.extra["innovation_variance_median"] > 0 + with pytest.raises(ValueError): + EmpiricalInnovationSimulator(cm, standardize="bogus") + + +def test_auto_shrinkage_intensity_properties(): + from quantcore.measurement.estimators import shrinkage_intensity + + rng = np.random.default_rng(11) + assert shrinkage_intensity(rng.standard_normal((80, 40))) == 1.0 # independent -> full shrinkage + L = np.linalg.cholesky(0.4 * np.eye(10) + 0.6) + strong = shrinkage_intensity(rng.standard_normal((2000, 10)) @ L.T) + assert 0.0 <= strong < 0.05 # strongly correlated, long sample -> little shrinkage + cm = CovarianceModel("rolling", window=100, shrinkage="auto") + x = pd.DataFrame(rng.standard_normal((300, 20)) @ np.linalg.cholesky(0.5 * np.eye(20) + 0.5).T) + assert 0 < cm.intensity(x) < 1 and cm.describe()["shrinkage"] == "auto" + + +def test_importance_sampling_tilt_both_models(r): + + from quantcore.models.simulation import TiltSpec + + cm = CovarianceModel("ewm", lam=0.97) + exposure = np.array([30_000.0, 50_000.0, 20_000.0]) + beta = 0.99 + for sim in (GBMSimulator(cm), EmpiricalInnovationSimulator(cm)): + plain = sim.simulate(r, n_scenarios=4096, horizon_periods=5, seed=9) + tilted = sim.simulate(r, n_scenarios=4096, horizon_periods=5, seed=9, + tilt=TiltSpec.at_confidence(exposure, beta, mixture=0.5)) + assert plain.probabilities is None and tilted.probabilities is not None + assert abs(tilted.probabilities.sum() - 1) < 1e-12 and tilted.describe()["effective_sample_size"] > 2000 + # exact first two moments in both cases (weighted for the tilted set) + w = tilted.weights + np.testing.assert_allclose(w @ tilted.log_returns, plain.log_returns.mean(axis=0), atol=1e-12) + cov_t = (tilted.log_returns - w @ tilted.log_returns).T @ ((tilted.log_returns - w @ tilted.log_returns) * w[:, None]) + np.testing.assert_allclose(cov_t, np.cov(plain.log_returns, rowvar=False, ddof=0), rtol=1e-8, atol=1e-14) + # many more scenarios beyond the reference portfolio's VaR, with the right total probability + loss_p = -(plain.simple_returns() @ exposure) + loss_t = -(tilted.simple_returns() @ exposure) + var = np.quantile(loss_p, beta) + assert (loss_t > var).sum() > 5 * (loss_p > var).sum() + assert abs(w[loss_t > var].sum() - (1 - beta)) < 0.01 + assert "importance_sampling" in tilted.extra and tilted.hash != plain.hash + # the default tilt mixes z_0.95 and z_0.99 blocks, so one set serves both levels + both = sim.simulate(r, n_scenarios=4096, horizon_periods=5, seed=9, tilt=TiltSpec(exposure=exposure)) + w2 = both.weights + loss_b = -(both.simple_returns() @ exposure) + for level in (0.95, 0.99): + var_l = np.quantile(loss_p, level) + assert (loss_b > var_l).sum() > 3 * (loss_p > var_l).sum() + assert abs(w2[loss_b > var_l].sum() - (1 - level)) < 0.01 + # determinism: same seed -> same hash, other seed -> different + a = GBMSimulator(cm).simulate(r, n_scenarios=512, seed=1) + b = GBMSimulator(cm).simulate(r, n_scenarios=512, seed=1) + c = GBMSimulator(cm).simulate(r, n_scenarios=512, seed=2) + assert a.hash == b.hash != c.hash + + +def test_scenario_distribution_reports_the_model_quantities(r): + sim = GBMSimulator(CovarianceModel("ewm", lam=0.97, shrinkage=0.0)) + scen = sim.simulate(r, n_scenarios=8192, horizon_periods=5, mu=pd.Series(0.0004, index=r.columns), seed=5) + v = np.array([400_000.0, 300_000.0, 300_000.0]) + d = scen.distribution(v, bins=40, confidence=(0.95, 0.99)) + cov_h = CovarianceModel("ewm", lam=0.97, shrinkage=0.0).estimate(r).values * 5 + sigma = np.sqrt(v @ cov_h @ v) + assert d["volatility"] == pytest.approx(sigma, rel=0.02) + # the reported mean is the mean of the *displayed* sample; moment matching is exact in log + # space, so it equals the model's expectation only up to the convexity of exp + assert d["expected_pnl"] == pytest.approx(float(scen.expected_simple @ v), rel=1e-4) + assert sum(d["histogram"]["mass"]) == pytest.approx(1.0) + assert len(d["histogram"]["edges"]) == 41 + # Gaussian reference: VaR = z sigma - mean, CVaR = phi(z)/(1-beta) sigma - mean + from scipy.stats import norm + for beta in (0.95, 0.99): + z = norm.ppf(beta) + assert d["risk"][f"{beta:g}"]["var"] == pytest.approx(z * sigma - d["expected_pnl"], rel=0.03) + assert d["risk"][f"{beta:g}"]["cvar"] == pytest.approx( + norm.pdf(z) / (1 - beta) * sigma - d["expected_pnl"], rel=0.03) + assert abs(d["skewness"]) < 0.25 and d["kurtosis"] == pytest.approx(3.0, abs=0.4) + + +def test_a_tilted_set_reports_the_same_distribution_as_an_untilted_one(r): + from quantcore.models.simulation import TiltSpec + sim = GBMSimulator(CovarianceModel("ewm", lam=0.97, shrinkage=0.0)) + v = np.array([400_000.0, 300_000.0, 300_000.0]) + kw = dict(n_scenarios=8192, horizon_periods=5, mu=pd.Series(0.0004, index=r.columns), seed=11) + plain = sim.simulate(r, **kw).distribution(v) + tilted = sim.simulate(r, **kw, tilt=TiltSpec(exposure=v)).distribution(v) + assert tilted["weighted"] and tilted["effective_sample_size"] < 8192 + assert tilted["expected_pnl"] == pytest.approx(plain["expected_pnl"], rel=1e-6) + assert tilted["volatility"] == pytest.approx(plain["volatility"], rel=0.02) + assert tilted["risk"]["0.99"]["cvar"] == pytest.approx(plain["risk"]["0.99"]["cvar"], rel=0.03) + for q in ("0.05", "0.5"): + assert tilted["quantiles"][q] == pytest.approx(plain["quantiles"][q], rel=0.05) + + +def test_summary_reports_price_returns_that_cannot_fall_below_minus_one(r): + """A long horizon makes the two spaces look very different: a log-return of -2 is -86 % of the + price. The table must carry both, and the price column must be bounded below by -100 %.""" + sim = GBMSimulator(CovarianceModel("ewm", lam=0.97, shrinkage=0.0)) + scen = sim.simulate(r, n_scenarios=4096, horizon_periods=63, mu=pd.Series(0.0004, index=r.columns), seed=3) + t = scen.summary() + for col in ("q01", "q05", "q50", "q95", "min", "max"): + np.testing.assert_allclose(t[f"{col}_simple"], np.expm1(t[col]), rtol=1e-12) # a monotone map + assert (t["min_simple"] > -1).all() and (t["min"] < t["min_simple"]).all() + assert (t["max_simple"] > t["max"]).all() + # the scenario mean in price space is above the log mean (Jensen) and matches the model's + assert (t["mean_simple"] > t["mean"]).all() + assert np.allclose(t["mean_simple"], t["expected_simple"], rtol=0.05) diff --git a/tests/test_solvers.py b/tests/test_solvers.py new file mode 100644 index 0000000..308ba39 --- /dev/null +++ b/tests/test_solvers.py @@ -0,0 +1,165 @@ +import numpy as np +import pytest +import scipy.sparse as sp + +from quantcore.config import Settings +from quantcore.optimization.problem import MILPModel +from quantcore.optimization.solvers import CuOptSolver, HighsSolver, solve_with_fallback, to_cuopt_payload + + +def readme_model() -> MILPModel: + # maximise 3x + 2y s.t. 2x + 3y <= 12, 4x + y <= 10, x,y >= 0 integer -> x = 2, y = 2, obj 10 + A = sp.csr_matrix(np.array([[2.0, 3.0], [4.0, 1.0]])) + return MILPModel(c=np.array([3.0, 2.0]), A=A, row_lb=np.array([-np.inf, -np.inf]), row_ub=np.array([12.0, 10.0]), + var_lb=np.zeros(2), var_ub=np.array([np.inf, np.inf]), var_types=["I", "I"], var_names=["x", "y"], + row_names=["c1", "c2"], maximize=True) + + +def test_payload_format(): + payload = to_cuopt_payload(readme_model(), time_limit=5) + assert payload["csr_constraint_matrix"] == {"offsets": [0, 2, 4], "indices": [0, 1, 0, 1], "values": [2.0, 3.0, 4.0, 1.0]} + assert payload["constraint_bounds"]["lower_bounds"] == ["ninf", "ninf"] + assert payload["variable_bounds"]["upper_bounds"] == ["inf", "inf"] + assert payload["variable_types"] == ["I", "I"] and payload["maximize"] is True + assert payload["solver_config"]["time_limit"] == 5.0 + + +def test_highs_solves_readme_example(): + res = HighsSolver().solve(readme_model()) + assert res.ok and res.status == "Optimal" + np.testing.assert_allclose(res.x, [2.0, 2.0]) + assert res.objective == pytest.approx(10.0) + + +def test_fallback_when_cuopt_unreachable(): + settings = Settings(cuopt_url="http://127.0.0.1:9", solver_fallback=True, cuopt_request_timeout=2) + res = solve_with_fallback(readme_model(), settings, solver="auto") + assert res.solver == "highs" and "fallback" in res.message and res.objective == pytest.approx(10.0) + with pytest.raises(ConnectionError): + solve_with_fallback(readme_model(), settings, solver="cuopt") + assert CuOptSolver(settings).health()["reachable"] is False + + +def test_fallback_when_cuopt_dies_during_the_solve(monkeypatch): + """cuOpt accepted the job, then its server went away while the client polled for the solution + (seen with a read-only container whose solver process crashed): 'auto' must still answer, on HiGHS.""" + import requests + + from quantcore.optimization import solvers as S + + settings = Settings(cuopt_url="http://127.0.0.1:9", solver_fallback=True, cuopt_poll_interval=0.01) + monkeypatch.setattr(S.CuOptSolver, "reachable", lambda self, ttl=30.0: True) + monkeypatch.setattr(S.CuOptSolver, "_post_with_retry", lambda self, payload: "job-1") + + def lost(*args, **kwargs): + raise requests.ConnectionError("Max retries exceeded") + + monkeypatch.setattr(S.requests, "get", lost) + monkeypatch.setattr(S.requests, "delete", lost) + res = solve_with_fallback(readme_model(), settings, solver="auto") + assert res.solver == "highs" and "connection lost" in res.message and res.objective == pytest.approx(10.0) + with pytest.raises(ConnectionError): + solve_with_fallback(readme_model(), settings, solver="cuopt") + + +@pytest.mark.skipif(not CuOptSolver(Settings()).health().get("reachable"), reason="cuOpt server not reachable") +def test_cuopt_solves_readme_example(): + res = CuOptSolver(Settings()).solve(readme_model(), time_limit=10) + assert res.ok and res.solver == "cuopt" + np.testing.assert_allclose(res.x, [2.0, 2.0]) + assert res.objective == pytest.approx(10.0) + + +def test_cuopt_retries_down_the_method_ladder_after_an_abort(monkeypatch): + """cuOpt's default method aborts on some CVaR-objective models; the client works down + ``CUOPT_RETRY_METHODS`` and reports which method finally solved it.""" + from quantcore.optimization import solvers as S + + calls = [] + + class Response: + def __init__(self, payload, status=200): + self._payload = payload + self.status_code = status + self.text = str(payload) + + def json(self): + return self._payload + + def raise_for_status(self): + pass + + def fake_post(url, data=None, headers=None, timeout=None, **kw): # noqa: ARG001 + import msgpack + + calls.append(msgpack.unpackb(data)["solver_config"]) + return Response({"reqId": f"req{len(calls)}"}) + + def fake_get(url, headers=None, timeout=None, **kw): + if len(calls) == 1: # first attempt: aborted + return Response({"error_result": True, "error": "req1 was aborted"}) + return Response({"response": {"solver_response": { + "status": "Optimal", + "solution": {"primal_solution": [1.0, 0.0], "primal_objective": 1.0, "solver_time": 0.1, + "milp_statistics": {"mip_gap": 0.0}}, + }}}) + + class FakeSession: # the client opens a fresh session per submit attempt + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + post = staticmethod(fake_post) + + monkeypatch.setattr(S.requests, "Session", FakeSession) + monkeypatch.setattr(S.requests, "get", fake_get) + monkeypatch.setattr(S.requests, "delete", lambda *a, **k: Response({})) + + model = readme_model() + res = S.CuOptSolver(Settings(cuopt_url="http://example.invalid")).solve(model, time_limit=5) + first = S.CUOPT_RETRY_METHODS[0] + assert res.status == "Optimal" and f"method={first}" in res.message + assert calls[0].get("method") is None and calls[1]["method"] == first # default, then the ladder + + +def test_cuopt_reports_when_every_method_aborts(monkeypatch): + """All rungs exhausted: the caller is told that, not just that the first attempt aborted.""" + from quantcore.optimization import solvers as S + + calls = [] + + class Response: + def __init__(self, payload): + self._payload, self.status_code, self.text = payload, 200, str(payload) + + def json(self): + return self._payload + + def raise_for_status(self): + pass + + class FakeSession: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + @staticmethod + def post(url, data=None, headers=None, timeout=None, **kw): + import msgpack + + calls.append(msgpack.unpackb(data)["solver_config"]) + return Response({"reqId": f"req{len(calls)}"}) + + monkeypatch.setattr(S.requests, "Session", FakeSession) + monkeypatch.setattr(S.requests, "get", + lambda *a, **k: Response({"error_result": True, "error": "req was aborted"})) + monkeypatch.setattr(S.requests, "delete", lambda *a, **k: Response({})) + + res = S.CuOptSolver(Settings(cuopt_url="http://example.invalid")).solve(readme_model(), time_limit=5) + assert res.status == "Error" + assert str(S.CUOPT_RETRY_METHODS) in res.message + assert len(calls) == 1 + len(S.CUOPT_RETRY_METHODS) # the default plus every rung