From 356017e689d0134d471df02fa78b11cd51cccf66 Mon Sep 17 00:00:00 2001 From: herzogflorian Date: Wed, 4 Nov 2026 10:03:30 +0100 Subject: [PATCH] Large-scale driver is the default strategy for scenario problems strategy='auto' now routes every scenario-mode CVaR optimisation through quantcore.optimization.large_scale (LP relaxation on the tail rows, screening, round-and-repair, certified against the relaxation bound, escalation to the integer MILP otherwise); strategy='milp' is the explicit opt-out, and row_generation applies to that path only. When the escalated MILP proves the rounded portfolio optimal within its gap, the result is Optimal even where the relaxation bound is loose (small portfolios with a real integrality gap). UI: a Strategy selector and the driver's bound, gap, screened size, rows and solver calls in the provenance card. Deploy env examples document the new cuOpt settings (presolve off, dual simplex for LPs, 50 ms polling). The service-equivalence test asks for strategy='milp' where it checks the integer row-generation path. Co-Authored-By: Claude Fable 5.1 --- .env.example | 3 +++ README.md | 2 +- deploy/apptainer/quantcore.env.example | 3 +++ docs/METHODOLOGY.md | 8 +++---- docs/USAGE.md | 10 ++++---- quantcore/api/schemas.py | 9 ++++--- quantcore/api/ui/js/core.js | 5 ++-- quantcore/api/ui/js/tabs/optimize.js | 18 +++++++++++--- quantcore/optimization/large_scale.py | 7 +++++- quantcore/optimization/service.py | 33 ++++++++++++-------------- tests/test_large_scale.py | 4 ++-- tests/test_service_equivalence.py | 4 +++- 12 files changed, 64 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index 21df0b0..2a313bf 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,9 @@ QUANTCORE_DATA_DIR=./data # where the setup/download services persist everything QUANTCORE_CUOPT_URL=http://127.0.0.1:5000 # cuOpt on this machine (Apptainer setup); a remote one: http://:5000 QUANTCORE_CUOPT_TIME_LIMIT=120 # seconds, MILP time limit +QUANTCORE_CUOPT_PRESOLVE=false # off: no 5-25 s GPU-heuristic wait per MILP (cuOpt 26.08) +QUANTCORE_CUOPT_LP_METHOD=2 # pure LPs on cuOpt's dual simplex (exact; concurrent may return PDLP's point) +QUANTCORE_CUOPT_POLL_INTERVAL=0.05 # seconds between polls of a submitted job (a latency floor per solve) QUANTCORE_BASE_CURRENCY=CHF QUANTCORE_RISK_FREE_RATE=0.0 # annual, CHF QUANTCORE_HISTORY_START=2004-12-01 # first date requested from Yahoo diff --git a/README.md b/README.md index 9cc64d8..f6a28c3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ never computes a number itself. | **PerformanceService** | `quantcore.measurement.performance` | Annualised return, volatility, Sharpe, Sortino, drawdown, Calmar, skew/kurtosis, hit ratio; rolling and EWM series; portfolios in shares. | | **Stock-price models** | `quantcore.models.simulation` | Geometric Brownian motion (multivariate normal) and the empirical-innovation model `eps_t = Sigma_{t-1}^{-1/2} r_t`, `r(j|T) = Sigma_T^{1/2} eps_j` with the latest rolling or EWM covariance. | | **Expected returns** | `quantcore.models.black_litterman` | Market-cap equilibrium returns (broad-market average, market weights), Black–Litterman on a 10-year weekly covariance with **Idzorek confidence** views (diagonal Ω), sentiment → view helper. | -| **OptimizationService** | `quantcore.optimization` | Mean–CVaR MILP in integer share numbers: **new portfolio** (from cash) or **rebalancing** (existing holdings, transaction costs); constraints: max 10 % per stock, max per sector, no shorts, cost/turnover limits, optional cardinality; scenario sets with variance reduction (Sobol', moment matching, market-direction importance sampling) that stay identical when risk limits, constraints, costs or views change, tail-row generation for large sets, a relax-screen-round driver for large universes (`strategy`), and an exact Gaussian (mean–variance-equivalent) mode; solved on cuOpt (GPU, over HTTP) with a HiGHS fallback (CPU, compiled into SciPy — runs in-process, nothing to deploy). | +| **OptimizationService** | `quantcore.optimization` | Mean–CVaR MILP in integer share numbers: **new portfolio** (from cash) or **rebalancing** (existing holdings, transaction costs); constraints: max 10 % per stock, max per sector, no shorts, cost/turnover limits, optional cardinality; scenario sets with variance reduction (Sobol', moment matching, market-direction importance sampling) that stay identical when risk limits, constraints, costs or views change, solved by default through a relax-screen-round driver certified against the relaxation bound (`strategy`; the integer MILP with tail-row generation on request), and an exact Gaussian (mean–variance-equivalent) mode; solved on cuOpt (GPU, over HTTP) with a HiGHS fallback (CPU, compiled into SciPy — runs in-process, nothing to deploy). | | **HTTP API** | `quantcore.api` | FastAPI app (`/docs`) exposing all of the above; setup/update as background jobs. Runs as a container on the DGX Spark next to cuOpt, or as a Kubernetes Deployment. | Documentation: **[docs/USAGE.md](docs/USAGE.md) (start here if you are building on this package: the API, the diff --git a/deploy/apptainer/quantcore.env.example b/deploy/apptainer/quantcore.env.example index d92303c..4ec4b21 100644 --- a/deploy/apptainer/quantcore.env.example +++ b/deploy/apptainer/quantcore.env.example @@ -33,6 +33,9 @@ QC_IMAGE=build # --- passed unchanged into the quantcore container (see quantcore/config.py) ------------------- QUANTCORE_CUOPT_TIME_LIMIT=300 # seconds per MILP +QUANTCORE_CUOPT_PRESOLVE=false # off: no 5-25 s GPU-heuristic wait per MILP (cuOpt 26.08) +QUANTCORE_CUOPT_LP_METHOD=2 # pure LPs on cuOpt's dual simplex (exact) +QUANTCORE_CUOPT_POLL_INTERVAL=0.05 # seconds between polls of a submitted job QUANTCORE_ENABLE_UI=true # false: JSON API and /docs only, no test bench at /ui/ QUANTCORE_LOG_LEVEL=INFO #QUANTCORE_BASE_CURRENCY=CHF diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index abfb215..7d26174 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -264,15 +264,15 @@ Sizes: `S + N(+2N) + 1` variables, `S + N + K + O(1)` rows, `≈ S·(N+2)` non-z linearly in `S` (HiGHS LP ≈ `S^1.9`, cuOpt ≈ `S^1.2–1.6`): 2 000 → 100 000 scenarios costs ≈ 45× on cuOpt. -**Tail-row generation** (`quantcore.optimization.rowgen`, `row_generation="auto"` from 8 192 -scenarios on): only tail scenarios have `u_s > 0` at the optimum, so the MILP is solved on a working +**Tail-row generation** (`quantcore.optimization.rowgen`; with `strategy="milp"`, +`row_generation="auto"` from 8 192 scenarios on; the default strategy below always works on tail rows): only tail scenarios have `u_s > 0` at the optimum, so the MILP is solved on a working set (the `2(1−β)S` worst scenarios of a reference portfolio), the losses of all scenarios are checked, violated ones (`L_s > α*`) are added, and the loop stops when none is violated — the restricted problem is a relaxation, so the final solution is optimal for the full set (within the MIP gap). Measured: working sets ≈ 11 % of `S`, 2–3 iterations, 6–7× faster at 32 k–100 k scenarios. -**Large-scale strategy** (`quantcore.optimization.large_scale`, `strategy="auto"` from 200 candidates -or 32 768 scenarios on, `"large_scale"` to force it): the solver benchmark of November 2026 +**Large-scale strategy** (`quantcore.optimization.large_scale`; `strategy="auto"`, the default for +every scenario problem; `"milp"` opts out): the solver benchmark of November 2026 (`GPU_Optimization/cuopt_benchmark`) showed that HiGHS closes every mean–CVaR MILP at the root node — share counts in the hundreds round with a loss far below the MIP gap, so the branch-and-bound is paid for on every call and almost never used — and that the only hard combinatorics is the cardinality diff --git a/docs/USAGE.md b/docs/USAGE.md index 99b6978..8fa15a8 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -302,12 +302,12 @@ that must be identical next week, freeze the data with a snapshot (`POST /snapsh for hours. Set it. 300 s is a sane default. * **The GPU solver is optional.** `solver="auto"` uses cuOpt if it answers and HiGHS otherwise; the result says which, and adds a warning. A cuOpt outage degrades speed, never correctness. -* **Large universes do not need the integer MILP.** `strategy="auto"` switches to the large-scale - driver (LP relaxation on the tail rows, screening for a cardinality limit, round-and-repair, - certified against the relaxation bound) from 200 candidates or 32 768 scenarios on; - `strategy="large_scale"` forces it, `"milp"` keeps the integer MILP. The result's +* **The CVaR problem is solved by the large-scale driver by default**, not by the integer MILP: + LP relaxation on the tail rows, screening for a cardinality limit, round-and-repair, certified + against the relaxation bound and escalated to the MILP only when the gap is above `mip_rel_gap`. + `strategy="milp"` restores the integer MILP (then `row_generation` applies). The result's `solver_info["large_scale"]["gap"]` is the proven gap; a `FeasibleFound` status there means the gap - is above `mip_rel_gap`, not that a time limit was hit. + stayed above `mip_rel_gap`, not that a time limit was hit. * **`risk_drift="model"` (the default) measures the CVaR on scenarios that include the expected return.** At long horizons the drift grows with *h* while the spread grows with *√h*, so a bullish forecast can push the whole loss tail above zero and the risk limit stops binding — the optimiser diff --git a/quantcore/api/schemas.py b/quantcore/api/schemas.py index ef86ba1..7968f9a 100644 --- a/quantcore/api/schemas.py +++ b/quantcore/api/schemas.py @@ -204,12 +204,11 @@ class OptimizeRequest(BaseModel): "return; 'zero': on the centred scenarios, so a high return forecast " "cannot relax the risk limit (the usual market-risk convention)") row_generation: str = Field("auto", pattern="^(auto|on|off)$", - description="solve on the tail rows only (large scenario sets)") + description="strategy 'milp' only: solve on the tail rows only (large scenario sets)") strategy: str = Field("auto", pattern="^(auto|milp|large_scale)$", - description="'large_scale': relaxation, tail rows, screening for cardinality, " - "round-and-repair certified against the relaxation bound (large universes " - "or scenario sets); 'milp': the integer MILP; 'auto': large_scale from " - "200 candidates or 32 768 scenarios on") + description="'auto' (= 'large_scale', the default): LP relaxation on the tail rows, " + "screening for a cardinality limit, round-and-repair, certified against the " + "relaxation bound; 'milp': the integer MILP (row_generation applies)") covariance: dict = Field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) expected_returns: str = Field("black_litterman", pattern="^(black_litterman|historical|zero)$") views: list[ViewSpec] = Field(default_factory=list) diff --git a/quantcore/api/ui/js/core.js b/quantcore/api/ui/js/core.js index 9414aec..0dc7a47 100644 --- a/quantcore/api/ui/js/core.js +++ b/quantcore/api/ui/js/core.js @@ -50,7 +50,7 @@ export const DEFAULT_STATE = { opt: { objective: "max_return", confidence: 0.95, cvar_mode: "fraction", cvar_limit_fraction: 0.05, cvar_limit: null, risk_aversion: 1, target_return: null, solver: "auto", time_limit: 300, - row_generation: "auto", auto_scenarios: true, risk_drift: "model", + strategy: "auto", row_generation: "auto", auto_scenarios: true, risk_drift: "model", }, data: { start: "", end: "", freq: "daily" }, }; @@ -151,7 +151,8 @@ export function optimizePayload(extra = {}) { objective: o.objective, risk_aversion: Number(o.risk_aversion), scenario_model: m.scenario_model, seed: Number(m.seed), sampling: m.sampling, moment_match: !!m.moment_match, importance_sampling: m.importance_sampling, - drift_convention: m.drift_convention, row_generation: o.row_generation, risk_drift: o.risk_drift, + drift_convention: m.drift_convention, strategy: o.strategy || "auto", row_generation: o.row_generation, + risk_drift: o.risk_drift, covariance: covarianceSpec(), expected_returns: m.expected_returns, views: viewPayload(), bl_tau: Number(m.bl_tau), bl_years: Number(m.bl_years), bl_omega_method: m.bl_omega_method || "idzorek", constraints: constraintPayload(), costs: { diff --git a/quantcore/api/ui/js/tabs/optimize.js b/quantcore/api/ui/js/tabs/optimize.js index c43971a..6e96bb1 100644 --- a/quantcore/api/ui/js/tabs/optimize.js +++ b/quantcore/api/ui/js/tabs/optimize.js @@ -157,8 +157,13 @@ function inputsCard(out, rerender) { o.risk_drift === "model" ? "over a long horizon a large forecast can push the whole tail into profit" : "the risk figure does not depend on the return forecast"), + field("Strategy", select([["auto", "relax, round and certify (default)"], ["milp", "integer MILP"]], + o.strategy || "auto", set("strategy")), + (o.strategy || "auto") === "milp" + ? "branch-and-bound on the full integer model; tail rows per the setting below" + : "LP relaxation on the tail rows, round and repair, certified against the relaxation bound"), field("Tail rows", select([["auto", "auto"], ["on", "always"], ["off", "never"]], o.row_generation, set("row_generation")), - "solve on the worst scenarios only, then verify"), + "integer MILP only: solve on the worst scenarios only, then verify"), el("div", { class: "field" }, el("label", { text: "Scenario count" }), segmented([[true, "automatic"], [false, `fixed (${fmt.int(state.model.n_scenarios)})`]], o.auto_scenarios, @@ -414,7 +419,7 @@ function positionsTable(positions, rebalancing, wealthValue) { function provenanceCard(r) { const m = r.model || {}, s = r.solver_info || {}; const scen = m.scenarios || {}; - const rg = s.row_generation; + const rg = s.row_generation, ls = s.large_scale; return card("How this number was produced", "everything needed to reproduce the run", el("div", { class: "grid three" }, el("div", {}, el("h3", { text: "Scenarios" }), @@ -433,7 +438,14 @@ function provenanceCard(r) { ["solver", r.solver], ["status", r.status], ["objective", Number.isFinite(s.objective) ? fmt.money2(s.objective) : null], ["MIP gap", Number.isFinite(s.mip_gap) ? s.mip_gap.toExponential(1) : s.mip_gap], ["variables", fmt.int(m.milp?.n_vars)], - ["rows solved", rg ? `${fmt.int(rg.rows_final)} of ${fmt.int(scen.n_scenarios)}` : null], + ["strategy", ls ? (ls.certified ? "relax + round, certified" : ls.escalated ? "relax + round, then MILP" : "relax + round") + : s.outer_approximation ? "Gaussian cuts" : "integer MILP"], + ["relaxation bound", Number.isFinite(ls?.bound) ? fmt.money2(ls.bound) : null], + ["gap to bound", Number.isFinite(ls?.gap) ? ls.gap.toExponential(1) : null], + ["screened to", ls?.screened_to ? `${fmt.int(ls.screened_to)} names` : null], + ["rows solved", ls ? `${fmt.int(ls.working_set)} of ${fmt.int(scen.n_scenarios)}` + : rg ? `${fmt.int(rg.rows_final)} of ${fmt.int(scen.n_scenarios)}` : null], + ["solver calls", ls?.solver_calls], ["row-generation passes", rg?.iterations], ["cuts (Gaussian)", s.outer_approximation?.iterations], ["solve time", fmt.secs(s.solve_time_s)], ["total", fmt.secs(r.timings?.total_s)], ["reproducibility", r.reproducibility], diff --git a/quantcore/optimization/large_scale.py b/quantcore/optimization/large_scale.py index 31ff594..35c3bda 100644 --- a/quantcore/optimization/large_scale.py +++ b/quantcore/optimization/large_scale.py @@ -364,7 +364,12 @@ def solve_large_scale(spec: ProblemSpec, solve_small: SolveFn, solve_large: Solv return finish(res, problem_r, idx, status=status, message=f"integer row generation ({rg.iterations} passes) after rounding") if rep is not None: - return finish(rep, problem_rep, idx, message="rounded; the escalated MILP did not improve on it") + # the MILP found nothing better: its optimality proof (within its own gap) covers the rounded + # portfolio too, so the result is certified by the MILP even where the relaxation bound is loose + proven = res.ok and res.status == "Optimal" + return finish(rep, problem_rep, idx, status=("Optimal" if proven else None), + message=("rounded; the escalated MILP proved it optimal within its gap" if proven + else "rounded; the escalated MILP did not improve on it")) info.total_time = time.monotonic() - t0 return (replace(res, x=(embed(full, MeanCVaRProblem(restricted_spec(sub_spec, np.arange(sub_spec.n_scenarios))), res.x, idx) if res.x is not None else None), diff --git a/quantcore/optimization/service.py b/quantcore/optimization/service.py index 1ac096b..4be08c0 100644 --- a/quantcore/optimization/service.py +++ b/quantcore/optimization/service.py @@ -13,9 +13,10 @@ Pipeline (deterministic given the seed; the result carries the scenario hash and solution (``importance_sampling``), or -- ``scenario_model='gaussian_exact'`` -- no scenarios at all but the exact Gaussian (mean-variance-equivalent) problem by outer approximation; 4. the mean-CVaR MILP (:mod:`quantcore.optimization.problem`) in "new" or "rebalancing" mode, - for large scenario sets by tail-row generation (:mod:`quantcore.optimization.rowgen`), and for - large universes or scenario sets by relaxation, screening and rounding - (:mod:`quantcore.optimization.large_scale`, ``strategy``); + solved by default through the large-scale driver (:mod:`quantcore.optimization.large_scale`: + LP relaxation on the tail rows, screening, round-and-repair, certified against the relaxation + bound) or, with ``strategy='milp'``, as the integer MILP with tail-row generation for large + scenario sets (:mod:`quantcore.optimization.rowgen`); 5. solved on cuOpt (HiGHS fallback), decoded into shares, trades, cash, costs, risk figures. """ @@ -60,12 +61,11 @@ log = logging.getLogger(__name__) SCENARIO_MODELS = ("gbm", "empirical", "gaussian_exact") IMPORTANCE_SAMPLING_MODES = ("auto", "market", "adaptive", "off") -ROW_GENERATION_MIN_SCENARIOS = 8192 # 'auto': row generation from this size on -# strategy 'auto': the relax / round-and-repair driver (quantcore.optimization.large_scale) from this -# many candidates or scenarios on; below, the integer MILP as before. Measured (cuopt_benchmark, 2026-11): -# HiGHS closes every CVaR MILP at the root, so the driver loses nothing and saves the tree search. -LARGE_SCALE_MIN_ASSETS = 200 -LARGE_SCALE_MIN_SCENARIOS = 32768 +ROW_GENERATION_MIN_SCENARIOS = 8192 # strategy 'milp', row_generation 'auto': row generation from this size on +# strategy 'auto' is the relax / round-and-repair driver (quantcore.optimization.large_scale) for every +# scenario problem. Measured (cuopt_benchmark, 2026-11): HiGHS closes every CVaR MILP at the root, so the +# driver loses nothing (it certifies against the relaxation bound and escalates to the MILP otherwise) and +# saves the tree search: 45 s -> 2.9 s at N=300, S=16384. LARGE_SCALE_CUOPT_ROWS = 16_000 # the driver sends models with at least this many rows to cuOpt IMPORTANCE_SAMPLING_MIN_TAIL = 128 # 'auto': tilt when fewer than this many scenarios would be in the tail STAGE1_SCENARIOS = 2048 # 'adaptive': first-pass solve that provides the tilt direction @@ -127,12 +127,11 @@ class OptimizationRequest: # stops binding. 'zero': the risk rows use the centred (driftless) scenarios while the # objective keeps the expected return, which is the usual market-risk convention. risk_drift: str = "model" # 'model' | 'zero' - row_generation: str = "auto" # 'auto' | 'on' | 'off': solve on the tail rows only - # 'large_scale': relaxation, tail rows, screening for cardinality, round and repair, certified - # against the relaxation bound (quantcore.optimization.large_scale); 'milp': the integer MILP - # (with row generation per `row_generation`); 'auto': 'large_scale' from LARGE_SCALE_MIN_ASSETS - # candidates or LARGE_SCALE_MIN_SCENARIOS scenarios on + # 'auto' (= 'large_scale'): relaxation, tail rows, screening for cardinality, round and repair, + # certified against the relaxation bound (quantcore.optimization.large_scale) -- the default for + # every scenario problem; 'milp': the integer MILP, with row generation per `row_generation` strategy: str = "auto" # 'auto' | 'milp' | 'large_scale' + row_generation: str = "auto" # strategy 'milp' only -- 'auto' | 'on' | 'off': solve on the tail rows only covariance: dict = field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) expected_returns: str = "black_litterman" # 'black_litterman' | 'historical' | 'zero' views: list[dict] = field(default_factory=list) @@ -186,10 +185,8 @@ class OptimizationRequest: return self.row_generation == "auto" and n_scenarios >= ROW_GENERATION_MIN_SCENARIOS def uses_large_scale(self, n_assets: int, n_scenarios: int) -> bool: - if self.strategy == "large_scale": - return True - return self.strategy == "auto" and (n_assets >= LARGE_SCALE_MIN_ASSETS - or n_scenarios >= LARGE_SCALE_MIN_SCENARIOS) + """The large-scale driver is the default for every scenario problem; 'milp' opts out.""" + return self.strategy != "milp" def tilt_mode(self, n_scenarios: int) -> str: """``'market'``, ``'adaptive'`` or ``'off'`` for this request. diff --git a/tests/test_large_scale.py b/tests/test_large_scale.py index 3e93eea..d8fe175 100644 --- a/tests/test_large_scale.py +++ b/tests/test_large_scale.py @@ -179,7 +179,7 @@ def test_service_routes_the_strategy(settings, md): assert ls.status == "Optimal" and ls.reproducible assert abs(ls.expected_pnl - milp.expected_pnl) <= 1e-4 * abs(milp.expected_pnl) + 1e-6 assert ls.cvar <= 12_000.0 * (1 + 1e-7) - req = OptimizationRequest(symbols=SYMBOLS, **BASE) - assert not req.uses_large_scale(12, 2048) and req.uses_large_scale(200, 2048) and req.uses_large_scale(12, 32768) + assert OptimizationRequest(symbols=SYMBOLS, **BASE).uses_large_scale(12, 2048) # the default path + assert not OptimizationRequest(symbols=SYMBOLS, strategy="milp", **BASE).uses_large_scale(750, 65536) with pytest.raises(ValueError): OptimizationRequest(symbols=SYMBOLS, strategy="bogus", **BASE) diff --git a/tests/test_service_equivalence.py b/tests/test_service_equivalence.py index 2f7ee46..b53710b 100644 --- a/tests/test_service_equivalence.py +++ b/tests/test_service_equivalence.py @@ -72,7 +72,9 @@ def test_importance_sampling_and_row_generation_keep_the_optimum(svc): assert abs(tilted.expected_pnl - plain.expected_pnl) <= 0.02 * plain.expected_pnl exact_req = OptimizationRequest(scenario_model="gaussian_exact", **BASE) assert gaussian_cvar_of(svc, exact_req, tilted)[1] <= BASE["cvar_limit"] * 1.05 - rg = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **{**BASE, "row_generation": "on"})) + # the integer MILP path with tail-row generation (the default strategy is the large-scale driver) + rg = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, + **{**BASE, "strategy": "milp", "row_generation": "on"})) assert rg.status == "Optimal" and rg.solver_info["row_generation"]["converged"] assert rg.scenario_hash == plain.scenario_hash # same scenario set ... assert abs(rg.expected_pnl - plain.expected_pnl) <= 1e-4 * plain.expected_pnl # ... same optimum