diff --git a/README.md b/README.md index 5a7faab..2e0d0cd 100644 --- a/README.md +++ b/README.md @@ -39,23 +39,52 @@ Apptainer, no root), [docs/SPARK_DEPLOYMENT.md](docs/SPARK_DEPLOYMENT.md) (the D ## Quick start on your DGX Spark (students) The service and cuOpt run as two Apptainer containers on your Spark and talk over `127.0.0.1`. -No root rights, no Docker, no git, nothing exposed to the network. One line in a terminal on the -Spark installs everything, builds the market data from Yahoo Finance and starts the service: +No root rights, no Docker, no git, nothing exposed to the network. + +**1. Install.** One line in a terminal on the Spark installs everything, builds the market data +from Yahoo Finance (~15 min) and starts the service: ```bash wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash ``` -Without `wget`, use `curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash`. With a prepared data set, add -`bash -s -- --data /path/to/quantcore_data.tar.gz` in place of the final `bash`. Afterwards the command `qc` -runs the service (`qc status`, `qc stop`, `qc start`, `qc upgrade`), the test bench is at -`http://127.0.0.1:5001/ui/`, and your platform calls the API at `http://127.0.0.1:5001`. The full guide, with settings, -several users on one Spark and troubleshooting, is [docs/SPARK_APPTAINER.md](docs/SPARK_APPTAINER.md). +Without `wget`: `curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash`. +With a prepared data set: `wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash -s -- --data /path/to/quantcore_data.tar.gz`. -## Quick start (local) +**2. Open a new terminal.** The installer adds the commands `qc`, `quantcore` and +`quantcore-api` to `~/.local/bin`; a terminal that was open during the installation does not know +them yet. + +**3. Use it.** The test bench is at `http://127.0.0.1:5001/ui/` in the Spark's browser, the API +documentation at `http://127.0.0.1:5001/docs`. Your platform calls the API at +`http://127.0.0.1:5001`. On the command line: ```bash -pip install -e ".[dev]" # Python 3.11+ +qc status # what runs, cuOpt, how much data +quantcore risk NESN.SW AAPL ASML.AS --horizon week --window 200 --lam 0.94 +quantcore performance NESN.SW AAPL --freq weekly +quantcore optimize --cash 1000000 --cvar-limit 40000 --confidence 0.95 --horizon week --max-assets 100 +quantcore-api health # the same service through its HTTP API +quantcore update # newest prices / FX +qc stop / qc start # the service runs in the background; after a reboot: qc start +qc upgrade # newest version of this repository +``` + +On the Spark, `quantcore` and `quantcore-api` run inside the container. Do **not** run `pip install` +there: the software is already installed in the container, and Ubuntu refuses system-wide +`pip install` anyway ("externally-managed-environment"). The full guide, with settings, several +users on one Spark and troubleshooting, is [docs/SPARK_APPTAINER.md](docs/SPARK_APPTAINER.md). + +## Quick start on your own computer (not the Spark) + +With Python 3.11+ on a laptop or workstation, install the package into a virtual environment. Not +needed on a Spark that ran the installer above. + +```bash +git clone https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git +cd Portfolio_Risk_Management +python3 -m venv .venv && source .venv/bin/activate +pip install -e ".[dev]" cp .env.example .env # optional: data dir, cuOpt URL, ... quantcore setup # once: universe + full history (~15 min, ~1100 Yahoo tickers) diff --git a/deploy/apptainer/qc.sh b/deploy/apptainer/qc.sh index 17d87b1..3fca445 100755 --- a/deploy/apptainer/qc.sh +++ b/deploy/apptainer/qc.sh @@ -532,6 +532,10 @@ print(" cuOpt seen by the API:", "reachable" if c.get("reachable") else "NOT r step "Files" say " settings: $CONFIG $([[ -f "$CONFIG" ]] || echo '(missing: defaults in use)')" say " data: $QC_DATA ($(du -sh "$QC_DATA" 2>/dev/null | cut -f1 || echo -))" + if grep -q ',wikipedia_stoxx600,' "$QC_DATA/universe/universe.csv" 2>/dev/null; then + say " NOTE: this data set comes from an earlier version's guessed European symbols and misses many" + say " companies. Build it anew (~10 min): $REPO_DIR/install.sh --rebuild-data" + fi say " quantcore: $(qc_mode) mode, $(qc_image)$([[ $(qc_mode) == venv ]] && echo " + $QC_VENV")" say " cuOpt: $(cuopt_sif)" } diff --git a/docs/SPARK_APPTAINER.md b/docs/SPARK_APPTAINER.md index e79ad53..0944588 100644 --- a/docs/SPARK_APPTAINER.md +++ b/docs/SPARK_APPTAINER.md @@ -48,7 +48,8 @@ The installer takes 20 to 30 minutes the first time and prints each step: 4. It builds the market data from Yahoo Finance, which takes about 15 minutes. The candidate stocks come from the list shipped in the repository, so every student gets the same universe. Companies that Yahoo no longer lists are left out, and setup names them in one summary line. -5. It starts the service, runs the self-test and installs the command `qc`. +5. It starts the service, runs the self-test and installs the commands `qc`, `quantcore` and + `quantcore-api`. With a prepared data set from your lecturer, the data step takes seconds instead. Pass the file with `--data`: @@ -69,8 +70,11 @@ the laptop's browser. Nothing else on the network can reach the service: ssh -L 5001:127.0.0.1:5001 @ ``` -The installer puts the command `qc` into `~/.local/bin`. Open a new terminal once so that the -shell finds it. `qc` is the same as `deploy/apptainer/qc.sh` in the code folder. +The installer puts the commands `qc`, `quantcore` and `quantcore-api` into `~/.local/bin`. **Open a +new terminal** after the installation; a terminal that was already open does not find them. `qc` is +the same as `deploy/apptainer/qc.sh` in the code folder. `quantcore` and `quantcore-api` are the +commands of the README and of [USAGE.md](USAGE.md); on the Spark they run inside the container, so +nothing needs to be installed with `pip`. ## Daily use @@ -105,29 +109,35 @@ print(qc.health()["cuopt"]["reachable"]) risk = qc.asset_risk(["NESN.SW", "AAPL"], horizon="week") ``` -Install the client into your own environment with -`pip install "git+https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git"`. -[USAGE.md](USAGE.md) describes every endpoint and the request fields. +Install the client into your platform's own virtual environment with +`pip install "git+https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git"`. Ubuntu refuses +`pip install` outside a virtual environment. A quick script can also run with the container's Python, +where quantcore is already installed: `qc python my_script.py`. [USAGE.md](USAGE.md) describes every +endpoint and the request fields. -**From the command line, through the API.** `qc api` runs the API client against your -service: +**From the command line, through the API.** `quantcore-api` is the API client. It talks to your +running service: ```bash -qc api health -qc api selftest # calls every endpoint, pass/fail table -qc api risk NESN.SW AAPL --horizon week -qc api optimize --cash 1000000 --cvar-limit 40000 --horizon week +quantcore-api health +quantcore-api selftest # calls every endpoint, pass/fail table +quantcore-api risk NESN.SW AAPL --horizon week +quantcore-api optimize --cash 1000000 --cvar-limit 40000 --horizon week ``` -**From the command line, without the API.** `qc cli` runs the services directly in the +**From the command line, without the API.** `quantcore` runs the services directly in the container. The API does not need to be running: ```bash -qc cli status -qc cli risk NESN.SW AAPL --horizon week -qc cli optimize --cash 1000000 --cvar-limit 40000 --horizon week --max-assets 50 +quantcore status +quantcore risk NESN.SW AAPL --horizon week +quantcore optimize --cash 1000000 --cvar-limit 40000 --horizon week --max-assets 50 ``` +`quantcore serve` starts the background service, like `qc start`. `quantcore setup` and +`quantcore update` run through the service when it is running, so it picks up the new data. +`qc cli …` and `qc api …` are the same commands under their `qc` names. + ## Settings `qc install` writes the settings file `~/quantcore/quantcore.env`. Edit it, then run @@ -219,7 +229,8 @@ and restarts the service. | `FATAL: ... failed to create build parent dir: stat /scratch/...` | Your account sets `APPTAINER_TMPDIR` to a folder that does not exist. The installer and `qc` now detect this, use a folder that works, and save the choice in `~/quantcore/quantcore.env`. Run the installer again. | | `git: command not found` | Use the one-line installer. It needs no git. | | Hundreds of red `Quote not found for symbol` lines during setup | Your installation predates the fix. Its setup used guessed symbols, so many European companies are missing from your data. Run `qc upgrade`, then `~/Portfolio_Risk_Management/install.sh --rebuild-data`. | -| `qc: command not found` right after the installation | Open a new terminal, or run `export PATH="$HOME/.local/bin:$PATH"`. | +| `qc: command not found` or `quantcore: command not found` right after the installation | Open a new terminal, or run `export PATH="$HOME/.local/bin:$PATH"`. | +| `error: externally-managed-environment` from `pip install` | That is the laptop route of the README. On the Spark nothing needs to be installed with `pip`: `quantcore` and `quantcore-api` already work. For your own code, use a virtual environment. | | `qc install` stops with "no 'instance run'" | The Spark's Apptainer is older than 1.1. Ask the administrator to update it. | | The cuOpt pull fails with "no space left on device" | Apptainer unpacks images in `/tmp`. Point it at a larger directory: `APPTAINER_TMPDIR=~/quantcore/tmp qc pull-cuopt`. | | Model status `FeasibleFound` instead of `Optimal` | The time limit was reached. Raise `time_limit` in the request or `QUANTCORE_CUOPT_TIME_LIMIT`, or reduce `n_scenarios` or `max_assets`. | diff --git a/docs/USAGE.md b/docs/USAGE.md index 350a711..e8ed3de 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -27,16 +27,18 @@ documented in `README.md` and in `docs/ARCHITECTURE.md` if you need them. ## 2. Install +**On a DGX Spark, skip this section.** The installer of [SPARK_APPTAINER.md](SPARK_APPTAINER.md) runs +the service in containers and provides both commands below, `quantcore` and `quantcore-api`; they +run inside the container. Install the package yourself only where your own code imports it, for +example for `quantcore.client`, and then in a virtual environment: + ```bash git clone https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git cd Portfolio_Risk_Management -pip install -e . # Python 3.11+ +python3 -m venv .venv && source .venv/bin/activate # Python 3.11+ +pip install -e . ``` -On a DGX Spark you do not need this step to run the service: `deploy/apptainer/qc.sh` runs it in -containers (see §3 and [SPARK_APPTAINER.md](SPARK_APPTAINER.md)). Install the package only where your -own code imports it, for example for `quantcore.client`. - This gives you two commands: | command | what it is | diff --git a/install.sh b/install.sh index 2bdc041..b7cba34 100755 --- a/install.sh +++ b/install.sh @@ -111,6 +111,14 @@ main() { if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi elif "$qc" has-data; then say " present -- newest prices later with: qc update" + local universe_csv + universe_csv="$("$qc" env | sed -n 's|^QC_DATA=||p')/universe/universe.csv" + if grep -q ',wikipedia_stoxx600,' "$universe_csv" 2>/dev/null; then + say "" + say " NOTE: this data set was built by an earlier version from Wikipedia's STOXX 600 list, whose" + say " guessed Yahoo symbols left out many European companies. Build it anew (~10 min):" + say " $dir/install.sh --rebuild-data" + fi elif [[ "$do_data" == 1 ]]; then say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)" say " candidates: the list shipped in the repository; companies Yahoo no longer knows are left out" @@ -119,14 +127,17 @@ main() { say " skipped (--no-data). Later: qc setup or qc data-import FILE" fi - step "5/5 The command 'qc'" - mkdir -p "$HOME/.local/bin" - ln -sf "$qc" "$HOME/.local/bin/qc" - say " $HOME/.local/bin/qc -> $qc" - case ":$PATH:" in - *":$HOME/.local/bin:"*) ;; - *) say " open a new terminal to use it (or run: export PATH=\"\$HOME/.local/bin:\$PATH\")" ;; - esac + step "5/5 Commands: qc, quantcore, quantcore-api" + local bin="$HOME/.local/bin" path_ok=0 + mkdir -p "$bin" + ln -sf "$qc" "$bin/qc" + # `quantcore` and `quantcore-api` as on a laptop -- here they run inside the container, so the + # commands in the README work on the Spark as written + write_wrapper "$bin/quantcore" "$qc" quantcore + write_wrapper "$bin/quantcore-api" "$qc" quantcore-api + say " $bin/qc, quantcore, quantcore-api -> the containers ($qc)" + case ":$PATH:" in *":$bin:"*) path_ok=1 ;; esac + ensure_path_in_bashrc "$bin" if [[ "$do_start" == 1 ]]; then if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi @@ -136,10 +147,20 @@ main() { fi fi + local port + port="$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')" step "Done" - say " Test bench: http://127.0.0.1:$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')/ui/ (on this Spark)" - say " Commands: qc status | qc stop | qc start | qc logs | qc update | qc upgrade | qc help" - say " Guide: $dir/docs/SPARK_APPTAINER.md" + say " Test bench: http://127.0.0.1:$port/ui/ API docs: http://127.0.0.1:$port/docs (in the Spark's browser)" + if [[ "$path_ok" == 0 ]]; then + say " This terminal does not know the new commands yet. Open a NEW terminal, or run here:" + say " export PATH=\"\$HOME/.local/bin:\$PATH\"" + fi + say " Then try:" + say " qc status # what runs, cuOpt, data" + say " quantcore risk NESN.SW AAPL --horizon week # the CLI, inside the container" + say " quantcore-api optimize --cash 1000000 --cvar-limit 40000 --horizon week" + say " The service keeps running; after a reboot: qc start. All commands: qc help" + say " Guide: $dir/docs/SPARK_APPTAINER.md" } say() { printf '%s\n' "$*"; } @@ -200,6 +221,45 @@ run_git() { "$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" /dev/null; then + say " kept your own $path (not replaced)" + return 0 + fi + rm -f "$path" + { + printf '#!/usr/bin/env bash\n%s, written by install.sh\n' "$marker" + printf '# %s on this DGX Spark: runs inside the quantcore container via deploy/apptainer/qc.sh\n' "$name" + printf 'QC=%q\n' "$qcpath" + if [[ "$name" == quantcore ]]; then + cat <<'EOF' +case "${1:-}" in + serve) echo "On the Spark the API runs as a background service: qc start (stop: qc stop)" >&2 + exec "$QC" start ;; + setup|update) exec "$QC" "$@" ;; # through the running API, which reloads the data afterwards + *) exec "$QC" cli "$@" ;; +esac +EOF + else + printf 'exec "$QC" api "$@"\n' + fi + } > "$path" + chmod +x "$path" +} + +# New terminals must find ~/.local/bin. Ubuntu's ~/.profile adds it only when it existed at login, +# so a terminal opened right after the installation would not -- hence one guarded line in ~/.bashrc. +ensure_path_in_bashrc() { # bin + local rc="$HOME/.bashrc" marker="# added by Portfolio_Risk_Management install.sh" + grep -q "$marker" "$rc" 2>/dev/null && return 0 + printf '\n%s: the commands qc, quantcore, quantcore-api\ncase ":$PATH:" in *":%s:"*) ;; *) export PATH="%s:$PATH" ;; esac\n' \ + "$marker" "$1" "$1" >> "$rc" + say " added $1 to PATH in $rc" +} + fetch() { # url file -- with whatever this machine has if command -v curl >/dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1" elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1" diff --git a/quantcore/cli.py b/quantcore/cli.py index 3dd245b..576a6a8 100644 --- a/quantcore/cli.py +++ b/quantcore/cli.py @@ -72,6 +72,14 @@ def cmd_clean(args) -> int: return 0 +def _rounded(table: pd.DataFrame, digits: int) -> pd.DataFrame: + """Round the numeric columns only: pandas warns when asked to round the date columns too.""" + out = table.copy() + numeric = out.select_dtypes("number").columns + out[numeric] = out[numeric].round(digits) + return out + + def cmd_snapshot(args) -> int: from quantcore.data.store import DataStore @@ -98,7 +106,7 @@ def cmd_risk(args) -> int: table = RiskService(md).asset_risk(args.symbols, freq=args.freq, window=args.window, lam=args.lam, horizon=args.horizon, currency=args.currency) with pd.option_context("display.width", 200, "display.max_columns", 30): - print(table.round(5).T) + print(_rounded(table, 5).T) return 0 @@ -111,7 +119,7 @@ def cmd_performance(args) -> int: table = PerformanceService(md, s.risk_free_rate).asset_performance(args.symbols, freq=args.freq, currency=args.currency, start=args.start) with pd.option_context("display.width", 200, "display.max_columns", 30): - print(table.round(4).T) + print(_rounded(table, 4).T) return 0