diff --git a/.env.example b/.env.example index 21df0b0..2a313bf 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,9 @@ QUANTCORE_DATA_DIR=./data # where the setup/download services persist everything QUANTCORE_CUOPT_URL=http://127.0.0.1:5000 # cuOpt on this machine (Apptainer setup); a remote one: http://:5000 QUANTCORE_CUOPT_TIME_LIMIT=120 # seconds, MILP time limit +QUANTCORE_CUOPT_PRESOLVE=false # off: no 5-25 s GPU-heuristic wait per MILP (cuOpt 26.08) +QUANTCORE_CUOPT_LP_METHOD=2 # pure LPs on cuOpt's dual simplex (exact; concurrent may return PDLP's point) +QUANTCORE_CUOPT_POLL_INTERVAL=0.05 # seconds between polls of a submitted job (a latency floor per solve) QUANTCORE_BASE_CURRENCY=CHF QUANTCORE_RISK_FREE_RATE=0.0 # annual, CHF QUANTCORE_HISTORY_START=2004-12-01 # first date requested from Yahoo diff --git a/README.md b/README.md index 9cc64d8..f6a28c3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ never computes a number itself. | **PerformanceService** | `quantcore.measurement.performance` | Annualised return, volatility, Sharpe, Sortino, drawdown, Calmar, skew/kurtosis, hit ratio; rolling and EWM series; portfolios in shares. | | **Stock-price models** | `quantcore.models.simulation` | Geometric Brownian motion (multivariate normal) and the empirical-innovation model `eps_t = Sigma_{t-1}^{-1/2} r_t`, `r(j|T) = Sigma_T^{1/2} eps_j` with the latest rolling or EWM covariance. | | **Expected returns** | `quantcore.models.black_litterman` | Market-cap equilibrium returns (broad-market average, market weights), Black–Litterman on a 10-year weekly covariance with **Idzorek confidence** views (diagonal Ω), sentiment → view helper. | -| **OptimizationService** | `quantcore.optimization` | Mean–CVaR MILP in integer share numbers: **new portfolio** (from cash) or **rebalancing** (existing holdings, transaction costs); constraints: max 10 % per stock, max per sector, no shorts, cost/turnover limits, optional cardinality; scenario sets with variance reduction (Sobol', moment matching, market-direction importance sampling) that stay identical when risk limits, constraints, costs or views change, tail-row generation for large sets, a relax-screen-round driver for large universes (`strategy`), and an exact Gaussian (mean–variance-equivalent) mode; solved on cuOpt (GPU, over HTTP) with a HiGHS fallback (CPU, compiled into SciPy — runs in-process, nothing to deploy). | +| **OptimizationService** | `quantcore.optimization` | Mean–CVaR MILP in integer share numbers: **new portfolio** (from cash) or **rebalancing** (existing holdings, transaction costs); constraints: max 10 % per stock, max per sector, no shorts, cost/turnover limits, optional cardinality; scenario sets with variance reduction (Sobol', moment matching, market-direction importance sampling) that stay identical when risk limits, constraints, costs or views change, solved by default through a relax-screen-round driver certified against the relaxation bound (`strategy`; the integer MILP with tail-row generation on request), and an exact Gaussian (mean–variance-equivalent) mode; solved on cuOpt (GPU, over HTTP) with a HiGHS fallback (CPU, compiled into SciPy — runs in-process, nothing to deploy). | | **HTTP API** | `quantcore.api` | FastAPI app (`/docs`) exposing all of the above; setup/update as background jobs. Runs as a container on the DGX Spark next to cuOpt, or as a Kubernetes Deployment. | Documentation: **[docs/USAGE.md](docs/USAGE.md) (start here if you are building on this package: the API, the diff --git a/deploy/apptainer/quantcore.env.example b/deploy/apptainer/quantcore.env.example index d92303c..4ec4b21 100644 --- a/deploy/apptainer/quantcore.env.example +++ b/deploy/apptainer/quantcore.env.example @@ -33,6 +33,9 @@ QC_IMAGE=build # --- passed unchanged into the quantcore container (see quantcore/config.py) ------------------- QUANTCORE_CUOPT_TIME_LIMIT=300 # seconds per MILP +QUANTCORE_CUOPT_PRESOLVE=false # off: no 5-25 s GPU-heuristic wait per MILP (cuOpt 26.08) +QUANTCORE_CUOPT_LP_METHOD=2 # pure LPs on cuOpt's dual simplex (exact) +QUANTCORE_CUOPT_POLL_INTERVAL=0.05 # seconds between polls of a submitted job QUANTCORE_ENABLE_UI=true # false: JSON API and /docs only, no test bench at /ui/ QUANTCORE_LOG_LEVEL=INFO #QUANTCORE_BASE_CURRENCY=CHF diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md index abfb215..7d26174 100644 --- a/docs/METHODOLOGY.md +++ b/docs/METHODOLOGY.md @@ -264,15 +264,15 @@ Sizes: `S + N(+2N) + 1` variables, `S + N + K + O(1)` rows, `≈ S·(N+2)` non-z linearly in `S` (HiGHS LP ≈ `S^1.9`, cuOpt ≈ `S^1.2–1.6`): 2 000 → 100 000 scenarios costs ≈ 45× on cuOpt. -**Tail-row generation** (`quantcore.optimization.rowgen`, `row_generation="auto"` from 8 192 -scenarios on): only tail scenarios have `u_s > 0` at the optimum, so the MILP is solved on a working +**Tail-row generation** (`quantcore.optimization.rowgen`; with `strategy="milp"`, +`row_generation="auto"` from 8 192 scenarios on; the default strategy below always works on tail rows): only tail scenarios have `u_s > 0` at the optimum, so the MILP is solved on a working set (the `2(1−β)S` worst scenarios of a reference portfolio), the losses of all scenarios are checked, violated ones (`L_s > α*`) are added, and the loop stops when none is violated — the restricted problem is a relaxation, so the final solution is optimal for the full set (within the MIP gap). Measured: working sets ≈ 11 % of `S`, 2–3 iterations, 6–7× faster at 32 k–100 k scenarios. -**Large-scale strategy** (`quantcore.optimization.large_scale`, `strategy="auto"` from 200 candidates -or 32 768 scenarios on, `"large_scale"` to force it): the solver benchmark of November 2026 +**Large-scale strategy** (`quantcore.optimization.large_scale`; `strategy="auto"`, the default for +every scenario problem; `"milp"` opts out): the solver benchmark of November 2026 (`GPU_Optimization/cuopt_benchmark`) showed that HiGHS closes every mean–CVaR MILP at the root node — share counts in the hundreds round with a loss far below the MIP gap, so the branch-and-bound is paid for on every call and almost never used — and that the only hard combinatorics is the cardinality diff --git a/docs/USAGE.md b/docs/USAGE.md index 99b6978..8fa15a8 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -302,12 +302,12 @@ that must be identical next week, freeze the data with a snapshot (`POST /snapsh for hours. Set it. 300 s is a sane default. * **The GPU solver is optional.** `solver="auto"` uses cuOpt if it answers and HiGHS otherwise; the result says which, and adds a warning. A cuOpt outage degrades speed, never correctness. -* **Large universes do not need the integer MILP.** `strategy="auto"` switches to the large-scale - driver (LP relaxation on the tail rows, screening for a cardinality limit, round-and-repair, - certified against the relaxation bound) from 200 candidates or 32 768 scenarios on; - `strategy="large_scale"` forces it, `"milp"` keeps the integer MILP. The result's +* **The CVaR problem is solved by the large-scale driver by default**, not by the integer MILP: + LP relaxation on the tail rows, screening for a cardinality limit, round-and-repair, certified + against the relaxation bound and escalated to the MILP only when the gap is above `mip_rel_gap`. + `strategy="milp"` restores the integer MILP (then `row_generation` applies). The result's `solver_info["large_scale"]["gap"]` is the proven gap; a `FeasibleFound` status there means the gap - is above `mip_rel_gap`, not that a time limit was hit. + stayed above `mip_rel_gap`, not that a time limit was hit. * **`risk_drift="model"` (the default) measures the CVaR on scenarios that include the expected return.** At long horizons the drift grows with *h* while the spread grows with *√h*, so a bullish forecast can push the whole loss tail above zero and the risk limit stops binding — the optimiser diff --git a/quantcore/api/schemas.py b/quantcore/api/schemas.py index ef86ba1..7968f9a 100644 --- a/quantcore/api/schemas.py +++ b/quantcore/api/schemas.py @@ -204,12 +204,11 @@ class OptimizeRequest(BaseModel): "return; 'zero': on the centred scenarios, so a high return forecast " "cannot relax the risk limit (the usual market-risk convention)") row_generation: str = Field("auto", pattern="^(auto|on|off)$", - description="solve on the tail rows only (large scenario sets)") + description="strategy 'milp' only: solve on the tail rows only (large scenario sets)") strategy: str = Field("auto", pattern="^(auto|milp|large_scale)$", - description="'large_scale': relaxation, tail rows, screening for cardinality, " - "round-and-repair certified against the relaxation bound (large universes " - "or scenario sets); 'milp': the integer MILP; 'auto': large_scale from " - "200 candidates or 32 768 scenarios on") + description="'auto' (= 'large_scale', the default): LP relaxation on the tail rows, " + "screening for a cardinality limit, round-and-repair, certified against the " + "relaxation bound; 'milp': the integer MILP (row_generation applies)") covariance: dict = Field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) expected_returns: str = Field("black_litterman", pattern="^(black_litterman|historical|zero)$") views: list[ViewSpec] = Field(default_factory=list) diff --git a/quantcore/api/ui/js/core.js b/quantcore/api/ui/js/core.js index 9414aec..0dc7a47 100644 --- a/quantcore/api/ui/js/core.js +++ b/quantcore/api/ui/js/core.js @@ -50,7 +50,7 @@ export const DEFAULT_STATE = { opt: { objective: "max_return", confidence: 0.95, cvar_mode: "fraction", cvar_limit_fraction: 0.05, cvar_limit: null, risk_aversion: 1, target_return: null, solver: "auto", time_limit: 300, - row_generation: "auto", auto_scenarios: true, risk_drift: "model", + strategy: "auto", row_generation: "auto", auto_scenarios: true, risk_drift: "model", }, data: { start: "", end: "", freq: "daily" }, }; @@ -151,7 +151,8 @@ export function optimizePayload(extra = {}) { objective: o.objective, risk_aversion: Number(o.risk_aversion), scenario_model: m.scenario_model, seed: Number(m.seed), sampling: m.sampling, moment_match: !!m.moment_match, importance_sampling: m.importance_sampling, - drift_convention: m.drift_convention, row_generation: o.row_generation, risk_drift: o.risk_drift, + drift_convention: m.drift_convention, strategy: o.strategy || "auto", row_generation: o.row_generation, + risk_drift: o.risk_drift, covariance: covarianceSpec(), expected_returns: m.expected_returns, views: viewPayload(), bl_tau: Number(m.bl_tau), bl_years: Number(m.bl_years), bl_omega_method: m.bl_omega_method || "idzorek", constraints: constraintPayload(), costs: { diff --git a/quantcore/api/ui/js/tabs/optimize.js b/quantcore/api/ui/js/tabs/optimize.js index c43971a..6e96bb1 100644 --- a/quantcore/api/ui/js/tabs/optimize.js +++ b/quantcore/api/ui/js/tabs/optimize.js @@ -157,8 +157,13 @@ function inputsCard(out, rerender) { o.risk_drift === "model" ? "over a long horizon a large forecast can push the whole tail into profit" : "the risk figure does not depend on the return forecast"), + field("Strategy", select([["auto", "relax, round and certify (default)"], ["milp", "integer MILP"]], + o.strategy || "auto", set("strategy")), + (o.strategy || "auto") === "milp" + ? "branch-and-bound on the full integer model; tail rows per the setting below" + : "LP relaxation on the tail rows, round and repair, certified against the relaxation bound"), field("Tail rows", select([["auto", "auto"], ["on", "always"], ["off", "never"]], o.row_generation, set("row_generation")), - "solve on the worst scenarios only, then verify"), + "integer MILP only: solve on the worst scenarios only, then verify"), el("div", { class: "field" }, el("label", { text: "Scenario count" }), segmented([[true, "automatic"], [false, `fixed (${fmt.int(state.model.n_scenarios)})`]], o.auto_scenarios, @@ -414,7 +419,7 @@ function positionsTable(positions, rebalancing, wealthValue) { function provenanceCard(r) { const m = r.model || {}, s = r.solver_info || {}; const scen = m.scenarios || {}; - const rg = s.row_generation; + const rg = s.row_generation, ls = s.large_scale; return card("How this number was produced", "everything needed to reproduce the run", el("div", { class: "grid three" }, el("div", {}, el("h3", { text: "Scenarios" }), @@ -433,7 +438,14 @@ function provenanceCard(r) { ["solver", r.solver], ["status", r.status], ["objective", Number.isFinite(s.objective) ? fmt.money2(s.objective) : null], ["MIP gap", Number.isFinite(s.mip_gap) ? s.mip_gap.toExponential(1) : s.mip_gap], ["variables", fmt.int(m.milp?.n_vars)], - ["rows solved", rg ? `${fmt.int(rg.rows_final)} of ${fmt.int(scen.n_scenarios)}` : null], + ["strategy", ls ? (ls.certified ? "relax + round, certified" : ls.escalated ? "relax + round, then MILP" : "relax + round") + : s.outer_approximation ? "Gaussian cuts" : "integer MILP"], + ["relaxation bound", Number.isFinite(ls?.bound) ? fmt.money2(ls.bound) : null], + ["gap to bound", Number.isFinite(ls?.gap) ? ls.gap.toExponential(1) : null], + ["screened to", ls?.screened_to ? `${fmt.int(ls.screened_to)} names` : null], + ["rows solved", ls ? `${fmt.int(ls.working_set)} of ${fmt.int(scen.n_scenarios)}` + : rg ? `${fmt.int(rg.rows_final)} of ${fmt.int(scen.n_scenarios)}` : null], + ["solver calls", ls?.solver_calls], ["row-generation passes", rg?.iterations], ["cuts (Gaussian)", s.outer_approximation?.iterations], ["solve time", fmt.secs(s.solve_time_s)], ["total", fmt.secs(r.timings?.total_s)], ["reproducibility", r.reproducibility], diff --git a/quantcore/optimization/large_scale.py b/quantcore/optimization/large_scale.py index 31ff594..35c3bda 100644 --- a/quantcore/optimization/large_scale.py +++ b/quantcore/optimization/large_scale.py @@ -364,7 +364,12 @@ def solve_large_scale(spec: ProblemSpec, solve_small: SolveFn, solve_large: Solv return finish(res, problem_r, idx, status=status, message=f"integer row generation ({rg.iterations} passes) after rounding") if rep is not None: - return finish(rep, problem_rep, idx, message="rounded; the escalated MILP did not improve on it") + # the MILP found nothing better: its optimality proof (within its own gap) covers the rounded + # portfolio too, so the result is certified by the MILP even where the relaxation bound is loose + proven = res.ok and res.status == "Optimal" + return finish(rep, problem_rep, idx, status=("Optimal" if proven else None), + message=("rounded; the escalated MILP proved it optimal within its gap" if proven + else "rounded; the escalated MILP did not improve on it")) info.total_time = time.monotonic() - t0 return (replace(res, x=(embed(full, MeanCVaRProblem(restricted_spec(sub_spec, np.arange(sub_spec.n_scenarios))), res.x, idx) if res.x is not None else None), diff --git a/quantcore/optimization/service.py b/quantcore/optimization/service.py index 1ac096b..4be08c0 100644 --- a/quantcore/optimization/service.py +++ b/quantcore/optimization/service.py @@ -13,9 +13,10 @@ Pipeline (deterministic given the seed; the result carries the scenario hash and solution (``importance_sampling``), or -- ``scenario_model='gaussian_exact'`` -- no scenarios at all but the exact Gaussian (mean-variance-equivalent) problem by outer approximation; 4. the mean-CVaR MILP (:mod:`quantcore.optimization.problem`) in "new" or "rebalancing" mode, - for large scenario sets by tail-row generation (:mod:`quantcore.optimization.rowgen`), and for - large universes or scenario sets by relaxation, screening and rounding - (:mod:`quantcore.optimization.large_scale`, ``strategy``); + solved by default through the large-scale driver (:mod:`quantcore.optimization.large_scale`: + LP relaxation on the tail rows, screening, round-and-repair, certified against the relaxation + bound) or, with ``strategy='milp'``, as the integer MILP with tail-row generation for large + scenario sets (:mod:`quantcore.optimization.rowgen`); 5. solved on cuOpt (HiGHS fallback), decoded into shares, trades, cash, costs, risk figures. """ @@ -60,12 +61,11 @@ log = logging.getLogger(__name__) SCENARIO_MODELS = ("gbm", "empirical", "gaussian_exact") IMPORTANCE_SAMPLING_MODES = ("auto", "market", "adaptive", "off") -ROW_GENERATION_MIN_SCENARIOS = 8192 # 'auto': row generation from this size on -# strategy 'auto': the relax / round-and-repair driver (quantcore.optimization.large_scale) from this -# many candidates or scenarios on; below, the integer MILP as before. Measured (cuopt_benchmark, 2026-11): -# HiGHS closes every CVaR MILP at the root, so the driver loses nothing and saves the tree search. -LARGE_SCALE_MIN_ASSETS = 200 -LARGE_SCALE_MIN_SCENARIOS = 32768 +ROW_GENERATION_MIN_SCENARIOS = 8192 # strategy 'milp', row_generation 'auto': row generation from this size on +# strategy 'auto' is the relax / round-and-repair driver (quantcore.optimization.large_scale) for every +# scenario problem. Measured (cuopt_benchmark, 2026-11): HiGHS closes every CVaR MILP at the root, so the +# driver loses nothing (it certifies against the relaxation bound and escalates to the MILP otherwise) and +# saves the tree search: 45 s -> 2.9 s at N=300, S=16384. LARGE_SCALE_CUOPT_ROWS = 16_000 # the driver sends models with at least this many rows to cuOpt IMPORTANCE_SAMPLING_MIN_TAIL = 128 # 'auto': tilt when fewer than this many scenarios would be in the tail STAGE1_SCENARIOS = 2048 # 'adaptive': first-pass solve that provides the tilt direction @@ -127,12 +127,11 @@ class OptimizationRequest: # stops binding. 'zero': the risk rows use the centred (driftless) scenarios while the # objective keeps the expected return, which is the usual market-risk convention. risk_drift: str = "model" # 'model' | 'zero' - row_generation: str = "auto" # 'auto' | 'on' | 'off': solve on the tail rows only - # 'large_scale': relaxation, tail rows, screening for cardinality, round and repair, certified - # against the relaxation bound (quantcore.optimization.large_scale); 'milp': the integer MILP - # (with row generation per `row_generation`); 'auto': 'large_scale' from LARGE_SCALE_MIN_ASSETS - # candidates or LARGE_SCALE_MIN_SCENARIOS scenarios on + # 'auto' (= 'large_scale'): relaxation, tail rows, screening for cardinality, round and repair, + # certified against the relaxation bound (quantcore.optimization.large_scale) -- the default for + # every scenario problem; 'milp': the integer MILP, with row generation per `row_generation` strategy: str = "auto" # 'auto' | 'milp' | 'large_scale' + row_generation: str = "auto" # strategy 'milp' only -- 'auto' | 'on' | 'off': solve on the tail rows only covariance: dict = field(default_factory=lambda: {"method": "ewm", "lam": 0.99, "shrinkage": "auto"}) expected_returns: str = "black_litterman" # 'black_litterman' | 'historical' | 'zero' views: list[dict] = field(default_factory=list) @@ -186,10 +185,8 @@ class OptimizationRequest: return self.row_generation == "auto" and n_scenarios >= ROW_GENERATION_MIN_SCENARIOS def uses_large_scale(self, n_assets: int, n_scenarios: int) -> bool: - if self.strategy == "large_scale": - return True - return self.strategy == "auto" and (n_assets >= LARGE_SCALE_MIN_ASSETS - or n_scenarios >= LARGE_SCALE_MIN_SCENARIOS) + """The large-scale driver is the default for every scenario problem; 'milp' opts out.""" + return self.strategy != "milp" def tilt_mode(self, n_scenarios: int) -> str: """``'market'``, ``'adaptive'`` or ``'off'`` for this request. diff --git a/tests/test_large_scale.py b/tests/test_large_scale.py index 3e93eea..d8fe175 100644 --- a/tests/test_large_scale.py +++ b/tests/test_large_scale.py @@ -179,7 +179,7 @@ def test_service_routes_the_strategy(settings, md): assert ls.status == "Optimal" and ls.reproducible assert abs(ls.expected_pnl - milp.expected_pnl) <= 1e-4 * abs(milp.expected_pnl) + 1e-6 assert ls.cvar <= 12_000.0 * (1 + 1e-7) - req = OptimizationRequest(symbols=SYMBOLS, **BASE) - assert not req.uses_large_scale(12, 2048) and req.uses_large_scale(200, 2048) and req.uses_large_scale(12, 32768) + assert OptimizationRequest(symbols=SYMBOLS, **BASE).uses_large_scale(12, 2048) # the default path + assert not OptimizationRequest(symbols=SYMBOLS, strategy="milp", **BASE).uses_large_scale(750, 65536) with pytest.raises(ValueError): OptimizationRequest(symbols=SYMBOLS, strategy="bogus", **BASE) diff --git a/tests/test_service_equivalence.py b/tests/test_service_equivalence.py index 2f7ee46..b53710b 100644 --- a/tests/test_service_equivalence.py +++ b/tests/test_service_equivalence.py @@ -72,7 +72,9 @@ def test_importance_sampling_and_row_generation_keep_the_optimum(svc): assert abs(tilted.expected_pnl - plain.expected_pnl) <= 0.02 * plain.expected_pnl exact_req = OptimizationRequest(scenario_model="gaussian_exact", **BASE) assert gaussian_cvar_of(svc, exact_req, tilted)[1] <= BASE["cvar_limit"] * 1.05 - rg = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, **{**BASE, "row_generation": "on"})) + # the integer MILP path with tail-row generation (the default strategy is the large-scale driver) + rg = svc.optimize(OptimizationRequest(scenario_model="gbm", n_scenarios=4096, + **{**BASE, "strategy": "milp", "row_generation": "on"})) assert rg.status == "Optimal" and rg.solver_info["row_generation"]["converged"] assert rg.scenario_hash == plain.scenario_hash # same scenario set ... assert abs(rg.expected_pnl - plain.expected_pnl) <= 1e-4 * plain.expected_pnl # ... same optimum