diff --git a/deploy/apptainer/qc.sh b/deploy/apptainer/qc.sh index 588b118..17d87b1 100755 --- a/deploy/apptainer/qc.sh +++ b/deploy/apptainer/qc.sh @@ -329,9 +329,8 @@ get_quantcore_image() { # force(0|1) say " present: virtualenv $QC_VENV (no-build fallback; reinstall: $0 build)"; return 0 fi build_sif || { - say "" - say " The image build is not possible on this Spark without root. Falling back to the no-build" - say " setup: a plain Python image plus a virtualenv in $QC_VENV (same libraries, same results)." + say " -> using the no-build setup instead: a plain Python image plus a virtualenv in $QC_VENV" + say " (same libraries, same results)" build_venv } ;; venv) @@ -348,15 +347,18 @@ get_quantcore_image() { # force(0|1) } build_sif() { # build quantcore.sif from the definition; non-zero when this Apptainer cannot build - local dst="$QC_IMAGES/quantcore.sif" + local dst="$QC_IMAGES/quantcore.sif" log="$QC_HOME/build.log" reason mkdir -p "$QC_IMAGES" - say " building $dst from $REPO_DIR (a few minutes)" + say " building $dst from $REPO_DIR (a few minutes; log: $log)" prepare_apptainer_dirs rm -f "$dst.part" # %files paths in the definition are relative to the repository root. Without root Apptainer - # builds through --fakeroot (user namespaces); a Spark may not allow that. - if ! (cd "$REPO_DIR" && "$APPTAINER" build "$dst.part" deploy/apptainer/quantcore.def); then + # builds through --fakeroot (user namespaces); many Sparks do not allow that. The output goes to a + # log: on such a Spark Apptainer's red ERROR line is expected, and the caller falls back. + if ! (cd "$REPO_DIR" && "$APPTAINER" build "$dst.part" deploy/apptainer/quantcore.def) >"$log" 2>&1; then rm -f "$dst.part" + reason="$(grep -E 'FATAL|ERROR' "$log" | tail -n 1 | sed -e 's/\x1b\[[0-9;]*m//g' -e 's/^[A-Z]* *: *//')" + say " no image build on this machine${reason:+ ($reason)}" return 1 fi mv -f "$dst.part" "$dst" @@ -391,8 +393,8 @@ cmd_build() { # rebuild after a code change, in the mode in use if [[ "$mode" == venv ]]; then build_venv elif [[ "$QC_IMAGE" == build ]]; then - build_sif || die "the image build failed (see above). Without root it needs Apptainer's --fakeroot. Use the - no-build fallback instead: set QC_IMAGE=venv in $CONFIG and run: $0 build" + build_sif || die "the image build failed (log: $QC_HOME/build.log). Without root it needs Apptainer's + --fakeroot. Use the no-build setup instead: set QC_IMAGE=venv in $CONFIG and run: $0 build" else die "QC_IMAGE=$QC_IMAGE is a prebuilt image; nothing to build" fi diff --git a/docs/SPARK_APPTAINER.md b/docs/SPARK_APPTAINER.md index 187387f..e79ad53 100644 --- a/docs/SPARK_APPTAINER.md +++ b/docs/SPARK_APPTAINER.md @@ -45,7 +45,9 @@ The installer takes 20 to 30 minutes the first time and prints each step: 2. It downloads the code to `~/Portfolio_Risk_Management`. If the Spark has no git, it uses git from a small container. 3. It gets the cuOpt image and the quantcore image. -4. It builds the market data from Yahoo Finance, which takes about 15 minutes. +4. It builds the market data from Yahoo Finance, which takes about 15 minutes. The candidate + stocks come from the list shipped in the repository, so every student gets the same universe. + Companies that Yahoo no longer lists are left out, and setup names them in one summary line. 5. It starts the service, runs the self-test and installs the command `qc`. With a prepared data set from your lecturer, the data step takes seconds instead. Pass the file @@ -78,6 +80,7 @@ qc status # what runs, whether cuOpt is reachable, how much data qc logs # API log; `qc logs cuopt` for the solver, add -f to follow qc update # the newest prices and FX rates from Yahoo Finance qc upgrade # the newest version of the code: download, rebuild, restart +~/Portfolio_Risk_Management/install.sh --rebuild-data # set the data aside and build it anew qc stop # stop both containers ``` @@ -215,6 +218,7 @@ and restarts the service. | Yahoo errors during `qc setup` or `qc update` | Yahoo limits how fast it answers. Run the command again later; stored data stays intact. | | `FATAL: ... failed to create build parent dir: stat /scratch/...` | Your account sets `APPTAINER_TMPDIR` to a folder that does not exist. The installer and `qc` now detect this, use a folder that works, and save the choice in `~/quantcore/quantcore.env`. Run the installer again. | | `git: command not found` | Use the one-line installer. It needs no git. | +| Hundreds of red `Quote not found for symbol` lines during setup | Your installation predates the fix. Its setup used guessed symbols, so many European companies are missing from your data. Run `qc upgrade`, then `~/Portfolio_Risk_Management/install.sh --rebuild-data`. | | `qc: command not found` right after the installation | Open a new terminal, or run `export PATH="$HOME/.local/bin:$PATH"`. | | `qc install` stops with "no 'instance run'" | The Spark's Apptainer is older than 1.1. Ask the administrator to update it. | | The cuOpt pull fails with "no space left on device" | Apptainer unpacks images in `/tmp`. Point it at a larger directory: `APPTAINER_TMPDIR=~/quantcore/tmp qc pull-cuopt`. | @@ -278,6 +282,10 @@ code with git from a container, installed, started and passed the self-test. `qc a new commit, rebuilt quantcore and restarted it in under a minute. `qc setup` built a data set from scratch inside the container. -`qc setup` takes the STOXX 600 candidates from the iShares holdings file. In October 2026 that link -answered 404, and setup fell back to Wikipedia's list of 465 names. A data set built by a student can -therefore have fewer European stocks than the prepared one. +**Where the candidate stocks come from.** The installer runs `qc setup --offline`. It takes the +candidates from the lists shipped in `quantcore/data/static/`: the S&P 500 and the iShares STOXX +Europe 600 holdings of September 2026, from which the course data set was built. `qc setup` without +`--offline` fetches the current lists instead. In October 2026 iShares moved the German download, so +setup now reads the same file from iShares' Swiss site. If that fails too, setup uses the shipped +list. It no longer uses Wikipedia's STOXX 600 table, whose codes mostly do not exist on Yahoo, for +example `BNPP.PA` instead of `BNP.PA`. diff --git a/install.sh b/install.sh index 49c8bc0..2bdc041 100755 --- a/install.sh +++ b/install.sh @@ -9,7 +9,8 @@ # # Options go after `bash -s --`, e.g. ... | bash -s -- --data ~/Downloads/quantcore_data.tar.gz # --data FILE|URL import this prepared data set instead of building one from Yahoo Finance -# --no-data skip the data step (later: qc setup, or qc data-import FILE) +# --no-data skip the data step (later: qc setup --offline, or qc data-import FILE) +# --rebuild-data move the existing data set aside (to ~/quantcore/data.) and build a new one # --no-start install only, do not start the services # --dir DIR where the code goes (default ~/Portfolio_Risk_Management) # @@ -18,7 +19,10 @@ # point to a directory that does not exist (seen on managed machines: /scratch/$USER/...) # 2. downloads the code with git -- or, without git on the machine, with git in a small container # 3. deploy/apptainer/qc.sh install: the cuOpt image and the quantcore image (or its no-build fallback) -# 4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none +# 4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none. +# The candidate stocks come from the list shipped in the repository (the iShares STOXX Europe 600 +# and S&P 500 constituents the course data was built from), so every installation gets the same +# universe; prices and company data come from Yahoo. # 5. starts cuOpt and the API, runs the self-test, and installs the command `qc` # # Everything is wrapped in main(), which runs only after bash has read the whole script: with @@ -29,7 +33,7 @@ main() { local repo_url="${QC_REPO_URL:-https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git}" local branch="${QC_BRANCH:-main}" local dir="${QC_DIR:-$HOME/Portfolio_Risk_Management}" - local data="" do_data=1 do_start=1 + local data="" do_data=1 do_start=1 rebuild_data=0 QC_HOME="${QC_HOME:-$HOME/quantcore}" APPTAINER="${APPTAINER:-apptainer}" GIT_IMAGE="${QC_GIT_IMAGE:-docker://alpine/git:2.54.0}" @@ -43,7 +47,8 @@ main() { --no-start) do_start=0 ;; --dir) dir="${2:?--dir needs a directory}"; shift ;; --dir=*) dir="${1#--dir=}" ;; - -h|--help) sed -n '2,26p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;; + --rebuild-data) rebuild_data=1 ;; + -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;; *) die "unknown option: $1 (see the top of install.sh)" ;; esac shift @@ -88,6 +93,14 @@ main() { fi step "4/5 Market data" + if [[ "$rebuild_data" == 1 ]] && "$qc" has-data; then + local data_dir aside + data_dir="$("$qc" env | sed -n 's|^QC_DATA=||p')" + aside="$data_dir.$(date +%Y%m%d-%H%M%S)" + "$qc" stop >/dev/null 2>&1 || true + mv "$data_dir" "$aside" + say " moved the existing data set to $aside" + fi if [[ -n "$data" ]]; then local file="$data" if [[ "$data" == http://* || "$data" == https://* ]]; then @@ -100,7 +113,8 @@ main() { say " present -- newest prices later with: qc update" elif [[ "$do_data" == 1 ]]; then say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)" - "$qc" setup + say " candidates: the list shipped in the repository; companies Yahoo no longer knows are left out" + "$qc" setup --offline else say " skipped (--no-data). Later: qc setup or qc data-import FILE" fi diff --git a/quantcore/data/yahoo.py b/quantcore/data/yahoo.py index 8696f66..ac22bc5 100644 --- a/quantcore/data/yahoo.py +++ b/quantcore/data/yahoo.py @@ -141,6 +141,9 @@ class YahooClient: # -- metadata -------------------------------------------------------------------------- def ticker_info(self, symbol: str) -> dict: """Selected ``Ticker.info`` fields for one symbol (``{}`` when Yahoo has nothing).""" + return normalise_info(symbol, self._info_fields(symbol)) + + def _info_fields(self, symbol: str) -> dict: import yfinance as yf def _info(): @@ -155,15 +158,32 @@ class YahooClient: log.warning("no info for %s: %s", symbol, exc) data = {} time.sleep(self.settings.yahoo_info_pause) - return normalise_info(symbol, data) + return data def ticker_infos(self, symbols: list[str], progress=None) -> pd.DataFrame: - """``ticker_info`` for many symbols as a DataFrame indexed by symbol.""" - rows = [] - for i, s in enumerate(symbols, 1): - rows.append(self.ticker_info(s)) - if progress and (i % 25 == 0 or i == len(symbols)): - progress(i, len(symbols)) + """``ticker_info`` for many symbols as a DataFrame indexed by symbol. + + yfinance logs an ERROR line for every symbol Yahoo does not know -- a delisted, renamed or + merged company. They are expected among ~1100 candidates and the selection drops them, so + they are collected and reported in one line instead.""" + rows, unknown = [], [] + yf_log = logging.getLogger("yfinance") + previous = yf_log.level + yf_log.setLevel(logging.CRITICAL) + try: + for i, s in enumerate(symbols, 1): + data = self._info_fields(s) + if not data: + unknown.append(s) + rows.append(normalise_info(s, data)) + if progress and (i % 25 == 0 or i == len(symbols)): + progress(i, len(symbols)) + finally: + yf_log.setLevel(previous) + if unknown: + shown = ", ".join(unknown[:12]) + (f" and {len(unknown) - 12} more" if len(unknown) > 12 else "") + log.info("Yahoo has no quote for %d of %d candidates (delisted, renamed or merged); " + "the selection leaves them out: %s", len(unknown), len(symbols), shown) df = pd.DataFrame(rows).set_index("symbol") return df diff --git a/quantcore/universe/sources.py b/quantcore/universe/sources.py index 561e767..7ecc8bc 100644 --- a/quantcore/universe/sources.py +++ b/quantcore/universe/sources.py @@ -47,12 +47,17 @@ USER_AGENT = "Mozilla/5.0 (compatible; quantcore-setup/0.1; +https://fhgr.ch)" SP500_URL = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies" STOXX600_WIKI_URL = "https://en.wikipedia.org/wiki/STOXX_Europe_600" -# iShares STOXX Europe 600 UCITS ETF (DE), holdings download of the German retail site -ISHARES_STOXX600_URL = ( - "https://www.ishares.com/de/privatanleger/de/produkte/251931/" - "ishares-stoxx-europe-600-ucits-etf-de-fund/1478358465952.ajax" - "?fileType=csv&fileName=EXSA_holdings&dataType=fund" +# iShares STOXX Europe 600 UCITS ETF (DE), holdings file in German, tried in this order. The Swiss +# site serves it; the German site's download answered 404 from October 2026 on and stays as a spare. +ISHARES_STOXX600_URLS = ( + ("https://www.ishares.com/ch/privatkunden/de/produkte/251931/" + "ishares-stoxx-europe-600-ucits-etf-de-fund/1495092304805.ajax" + "?fileType=csv&fileName=EXSA_holdings&dataType=fund"), + ("https://www.ishares.com/de/privatanleger/de/produkte/251931/" + "ishares-stoxx-europe-600-ucits-etf-de-fund/1478358465952.ajax" + "?fileType=csv&fileName=EXSA_holdings&dataType=fund"), ) +ISHARES_STOXX600_URL = ISHARES_STOXX600_URLS[0] # German labels of the iShares file -> English ISHARES_SECTOR_EN = { @@ -209,15 +214,30 @@ def parse_ishares_holdings(text: str, source_date: str | None = None) -> pd.Data def fetch_stoxx600_ishares(timeout: float = 30.0) -> pd.DataFrame: - """STOXX Europe 600 constituents from the iShares ETF holdings (complete, with weights).""" - raw = _get(ISHARES_STOXX600_URL, timeout).decode("utf-8-sig", errors="replace") - df = parse_ishares_holdings(raw) - log.info("STOXX 600 candidates from iShares: %d", len(df)) - return df + """STOXX 600 constituents from the iShares holdings file: exchange and ticker per line, so the + Yahoo symbols are exact. Tries every address in :data:`ISHARES_STOXX600_URLS`.""" + last: Exception | None = None + for url in ISHARES_STOXX600_URLS: + try: + raw = _get(url, timeout).decode("utf-8-sig", errors="replace") + df = parse_ishares_holdings(raw) + except Exception as exc: # noqa: BLE001 - next address + log.info("iShares holdings not available at %s: %s", url.split("?")[0], exc) + last = exc + continue + if len(df) >= 100: + log.info("STOXX 600 candidates from iShares: %d (as of %s)", len(df), df["source_date"].iloc[0]) + return df + last = ValueError(f"only {len(df)} rows in the iShares file at {url.split('?')[0]}") + raise RuntimeError(f"iShares holdings file not available: {last}") def fetch_stoxx600_wikipedia(timeout: float = 30.0) -> pd.DataFrame: - """Partial STOXX 600 list from Wikipedia (fallback; no exchange, so the suffix is by country).""" + """Partial STOXX 600 list from Wikipedia -- the last resort, used only when no snapshot ships. + + The table gives Reuters-style codes without the exchange, so the Yahoo suffix is guessed from the + country; for a large share of the index the guess does not exist on Yahoo (BNPP.PA for BNP.PA, + NOV-N.SW for NOVN.SW, VESTAS.CO for VWS.CO, UC.MI for UCG.MI).""" html = _get(STOXX600_WIKI_URL, timeout) tables = pd.read_html(io.BytesIO(html)) table = next(t for t in tables if "Ticker" in t.columns and "Country" in t.columns) @@ -277,26 +297,43 @@ def save_static_candidates(df: pd.DataFrame, region: str, path: Path | None = No def fetch_candidates(region: str, allow_web: bool = True, timeout: float = 30.0) -> pd.DataFrame: - """Best-effort candidate list: web sources first, then the static snapshot. + """Candidate list of a region: the primary web source, else the snapshot shipped with the package. - Every web source is wrapped: a failing source is logged and the next one is tried - (timeout, retry and fallback -- the resilience the platform requires on external calls). + US: the S&P 500 table on Wikipedia. Europe: the iShares STOXX Europe 600 holdings. When the + primary source fails, the shipped snapshot (taken from the same sources) is used -- not the + Wikipedia STOXX 600 table, whose guessed symbols miss a large share of the index; that table is + the last resort when no snapshot ships. ``allow_web=False`` (``quantcore setup --offline``) uses + the snapshot directly, so every installation builds its universe from the same candidates. + + Only a primary source refreshes the snapshot. Every web call is wrapped: a failure is logged + and the next option is used (timeout, retry and fallback, as the platform requires). """ region = "US" if region.upper() == "US" else "Europe" - fetchers = [fetch_sp500_wikipedia] if region == "US" else [fetch_stoxx600_ishares, fetch_stoxx600_wikipedia] + primary = fetch_sp500_wikipedia if region == "US" else fetch_stoxx600_ishares if allow_web: - for fetch in fetchers: + try: + df = primary(timeout) + except Exception as exc: # noqa: BLE001 - any failure -> the snapshot + log.warning("Candidate source %s failed: %s", primary.__name__, exc) + df = _empty() + if len(df) >= 100: + if region == "Europe": + df = df[df["country"].isin(EUROPE_COUNTRIES)].reset_index(drop=True) try: - df = fetch(timeout) - except Exception as exc: # noqa: BLE001 - any failure -> next source - log.warning("Candidate source %s failed: %s", fetch.__name__, exc) - continue - if len(df) >= 100: - if region == "Europe": - df = df[df["country"].isin(EUROPE_COUNTRIES)].reset_index(drop=True) - try: - save_static_candidates(df, region) - except OSError as exc: # read-only install - log.debug("Could not refresh static snapshot: %s", exc) - return df - return load_static_candidates(region) + save_static_candidates(df, region) + except OSError as exc: # read-only install + log.debug("Could not refresh static snapshot: %s", exc) + return df + static = load_static_candidates(region) + if len(static): + if allow_web: + log.warning("Using the %s candidates shipped with the package instead (%d, as of %s)", + region, len(static), static["source_date"].iloc[0]) + return static + if allow_web and region == "Europe": + try: + df = fetch_stoxx600_wikipedia(timeout) + return df[df["country"].isin(EUROPE_COUNTRIES)].reset_index(drop=True) + except Exception as exc: # noqa: BLE001 + log.warning("Candidate source fetch_stoxx600_wikipedia failed: %s", exc) + return static diff --git a/tests/test_data.py b/tests/test_data.py index eb78305..a40d5f5 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -79,3 +79,74 @@ def test_parse_ishares_holdings(): assert df.iloc[0]["gics_sector"] == "Information Technology" and df.iloc[0]["country"] == "Netherlands" assert df.iloc[0]["index_weight"] == pytest.approx(4.23) assert df.iloc[0]["source_date"] == "2026-09-18" + + +# -- candidate sources: which list a setup builds its universe from --------------------------------- +def _frame(symbols, source): + from quantcore.universe.sources import CANDIDATE_COLUMNS + + rows = [{c: "" for c in CANDIDATE_COLUMNS} | {"symbol": s, "country": "Switzerland", "region": "Europe", + "source": source, "source_date": "2026-10-06"} for s in symbols] + return pd.DataFrame(rows, columns=CANDIDATE_COLUMNS) + + +def test_europe_falls_back_to_the_shipped_list_not_to_wikipedia(monkeypatch): + """iShares down (it answered 404 in October 2026): the shipped snapshot, with exact symbols, is used. + Wikipedia's table yields guesses such as BNPP.PA or NOV-N.SW that Yahoo does not know.""" + from quantcore.universe import sources as S + + calls = [] + monkeypatch.setattr(S, "fetch_stoxx600_ishares", lambda timeout=30.0: (_ for _ in ()).throw(RuntimeError("404"))) + monkeypatch.setattr(S, "fetch_stoxx600_wikipedia", lambda timeout=30.0: calls.append("wiki") or _frame(["X"] * 200, "w")) + monkeypatch.setattr(S, "save_static_candidates", lambda *a, **k: calls.append("save")) + df = S.fetch_candidates("Europe") + assert calls == [] and len(df) > 500 and set(df["source"]) == {"ishares_stoxx600"} + assert {"BNP.PA", "NOVN.SW", "VWS.CO", "UCG.MI"} <= set(df["symbol"]) + + +def test_offline_setup_uses_the_shipped_lists_without_the_web(monkeypatch): + from quantcore.universe import sources as S + + def no_web(*a, **k): + raise AssertionError("no web access with allow_web=False") + + monkeypatch.setattr(S, "_get", no_web) + assert len(S.fetch_candidates("US", allow_web=False)) > 400 + assert len(S.fetch_candidates("Europe", allow_web=False)) > 500 + + +def test_ishares_tries_the_next_address(monkeypatch): + from quantcore.universe import sources as S + + tried = [] + + def fake_get(url, timeout=30.0): + tried.append(url) + if len(tried) == 1: + raise RuntimeError("404 Client Error") + return b"csv" + + monkeypatch.setattr(S, "_get", fake_get) + monkeypatch.setattr(S, "parse_ishares_holdings", lambda raw: _frame([f"S{i}.SW" for i in range(150)], "ishares")) + df = S.fetch_stoxx600_ishares() + assert tried == list(S.ISHARES_STOXX600_URLS[:2]) and len(df) == 150 + + +def test_unknown_symbols_are_summarised_not_logged_one_by_one(monkeypatch, caplog): + import logging + + from quantcore.config import Settings + from quantcore.data.yahoo import YahooClient + + client = YahooClient(Settings(yahoo_info_pause=0.0)) + known = {"quoteType": "EQUITY", "currency": "CHF", "longName": "Known AG", "exchange": "EBS"} + monkeypatch.setattr(YahooClient, "_info_fields", + lambda self, s: logging.getLogger("yfinance").error("HTTP Error 404 %s", s) or + ({} if s.startswith("GONE") else known)) + with caplog.at_level(logging.INFO): + df = client.ticker_infos(["KNOWN.SW", "GONE1.PA", "GONE2.SW"]) + assert list(df.index) == ["KNOWN.SW", "GONE1.PA", "GONE2.SW"] and df.loc["KNOWN.SW", "currency"] == "CHF" + assert not [r for r in caplog.records if r.name == "yfinance"] # no per-symbol ERROR lines + summary = [r.getMessage() for r in caplog.records if "no quote for 2 of 3" in r.getMessage()] + assert summary and "GONE1.PA, GONE2.SW" in summary[0] + assert logging.getLogger("yfinance").level == logging.NOTSET # restored afterwards