diff --git a/README.md b/README.md index 2e0d0cd..b96d0e6 100644 --- a/README.md +++ b/README.md @@ -65,11 +65,19 @@ quantcore risk NESN.SW AAPL ASML.AS --horizon week --window 200 --lam 0.94 quantcore performance NESN.SW AAPL --freq weekly quantcore optimize --cash 1000000 --cvar-limit 40000 --confidence 0.95 --horizon week --max-assets 100 quantcore-api health # the same service through its HTTP API -quantcore update # newest prices / FX -qc stop / qc start # the service runs in the background; after a reboot: qc start -qc upgrade # newest version of this repository ``` +The block above only reads, so it is safe to paste as a whole. The service runs in the background; +these commands manage it, one at a time: + +| Command | What it does | +|---|---| +| `qc start` | Starts cuOpt and the API. Needed again after a reboot. | +| `qc stop` | Stops both. | +| `quantcore update` | Fetches the newest prices and FX rates, then restarts the API. | +| `qc upgrade` | Installs the newest version of this repository, then restarts. | +| `~/Portfolio_Risk_Management/install.sh --rebuild-data` | Sets the data aside and builds it anew (~10 min). | + On the Spark, `quantcore` and `quantcore-api` run inside the container. Do **not** run `pip install` there: the software is already installed in the container, and Ubuntu refuses system-wide `pip install` anyway ("externally-managed-environment"). The full guide, with settings, several diff --git a/deploy/apptainer/qc.sh b/deploy/apptainer/qc.sh index 3fca445..40fb1c8 100755 --- a/deploy/apptainer/qc.sh +++ b/deploy/apptainer/qc.sh @@ -243,6 +243,20 @@ run_qc() { # command [args]: quantcore | quantcore-api | python ... exec "$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin "$cmd")" "$@" } +restore_hint() { # list earlier data sets next to $QC_DATA (from --rebuild-data) that can be restored + local d n=0 when stocks + for d in $(ls -1dt "$QC_DATA".* 2>/dev/null); do + [[ -f "$d/universe/universe.csv" ]] || continue + if [[ $n == 0 ]]; then say " earlier data sets found (newest first):"; fi + when="$(grep -o '"setup_at": *"[^"]*"' "$d/manifest.json" 2>/dev/null | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/' || true)" + stocks=$(( $(wc -l < "$d/universe/universe.csv") - 1 )) + say " $d ($stocks stocks, built ${when:-?})" + n=$((n + 1)) + if [[ $n -ge 4 ]]; then break; fi + done + if [[ $n -gt 0 ]]; then say " restore one with: $0 data-import "; fi +} + data_is_empty() { [[ -z "$(ls -A "$QC_DATA" 2>/dev/null | grep -v -x -e jobs -e '.*\.log')" ]]; } # --- install ---------------------------------------------------------------------------------- @@ -488,7 +502,11 @@ cmd_start() { step "Starting quantcore (user $ME, settings $([[ -f "$CONFIG" ]] && echo "$CONFIG" || echo "defaults"))" start_cuopt start_api - data_is_empty && warn "no market data yet: $0 data-import or $0 setup" + if data_is_empty; then + warn "no market data in $QC_DATA -- the API answers 503 until there is some." + restore_hint + say " build it (~10 min): $0 setup --offline or import a prepared one: $0 data-import FILE" + fi local ui=""; [[ "${QUANTCORE_ENABLE_UI:-true}" != "false" ]] && ui="$QC_API_URL/ui/ (test bench)" step "Ready -- on this Spark:" [[ -n "$ui" ]] && say " $ui" @@ -501,7 +519,7 @@ cmd_stop() { local name stopped=0 for name in "$API_INSTANCE" "$CUOPT_INSTANCE"; do if instance_running "$name"; then - "$APPTAINER" instance stop "$name" >/dev/null && say "stopped $name" && stopped=1 + stop_instance "$name"; say "stopped $name"; stopped=1 fi done [[ $stopped == 1 ]] || say "nothing running" @@ -516,6 +534,14 @@ cmd_status() { if h="$(http_get "$CUOPT_URL/cuopt/health")"; then say " up: $h"; else say " not answering"; fi step "API $QC_API_URL" if h="$(http_get "$QC_API_URL/health")"; then + local api_setup host_setup + # `|| true`: under pipefail a grep without a match would end the script silently + api_setup="$(printf '%s' "$h" | grep -o '"setup_at":"[^"]*"' | head -n 1 || true)" + host_setup="$(grep -o '"setup_at": *"[^"]*"' "$QC_DATA/manifest.json" 2>/dev/null | head -n 1 | tr -d ' ' || true)" + if [[ "$api_setup" != "$host_setup" ]]; then + warn "the running API serves a different data set than $QC_DATA (API ${api_setup:-none}, folder ${host_setup:-none}). + Run: $0 restart" + fi if command -v python3 >/dev/null 2>&1; then printf '%s' "$h" | python3 -c ' import json, sys @@ -532,6 +558,7 @@ print(" cuOpt seen by the API:", "reachable" if c.get("reachable") else "NOT r step "Files" say " settings: $CONFIG $([[ -f "$CONFIG" ]] || echo '(missing: defaults in use)')" say " data: $QC_DATA ($(du -sh "$QC_DATA" 2>/dev/null | cut -f1 || echo -))" + data_is_empty && restore_hint if grep -q ',wikipedia_stoxx600,' "$QC_DATA/universe/universe.csv" 2>/dev/null; then say " NOTE: this data set comes from an earlier version's guessed European symbols and misses many" say " companies. Build it anew (~10 min): $REPO_DIR/install.sh --rebuild-data" @@ -553,14 +580,39 @@ cmd_logs() { } # --- data ------------------------------------------------------------------------------------- -data_job() { # setup|update [args]: through the running API (which then reloads), else in-process +data_job() { # setup|update [args] + # Always run in a fresh container on $QC_DATA, never inside the running API: the API holds the + # directory it was started with, and if that is no longer $QC_DATA its writes would end up somewhere + # the CLI never looks. A running API is restarted afterwards so that it serves the new data; for a + # full setup it is stopped first (setup rewrites every file it reads). local job="$1"; shift - if http_get "$QC_API_URL/health" >/dev/null; then - say "the API is running: the $job job runs there, and the API reloads the data when it is done" - run_qc quantcore-api --url "$QC_API_URL" "$job" --wait "$@" - else - run_qc quantcore "$job" "$@" + local api_was_running=0 rc=0 + instance_running "$API_INSTANCE" && api_was_running=1 + if [[ "$job" == setup && "$api_was_running" == 1 ]]; then + say "stopping the API while the data set is rebuilt" + stop_instance "$API_INSTANCE" fi + ( run_qc quantcore "$job" "$@" ) || rc=$? + if [[ "$api_was_running" == 1 ]]; then + say "restarting the API so that it serves the new data" + restart_api + fi + return "$rc" +} + +stop_instance() { # name -- stop and wait until it is gone + local i + instance_running "$1" || return 0 + "$APPTAINER" instance stop "$1" >/dev/null 2>&1 || "$APPTAINER" instance stop --force "$1" >/dev/null 2>&1 || true + for i in $(seq 1 30); do instance_running "$1" || return 0; sleep 1; done + die "could not stop the instance $1 (apptainer instance stop --force $1)" +} + +restart_api() { + stop_instance "$API_INSTANCE" + local i + for i in $(seq 1 15); do port_busy "$QC_API_PORT" || break; sleep 1; done + start_api } cmd_data_import() { @@ -577,7 +629,7 @@ cmd_data_import() { tar -xzf "$src" -C "$QC_DATA" fi say "imported into $QC_DATA ($(du -sh "$QC_DATA" | cut -f1))" - if instance_running "$API_INSTANCE"; then say "the running API still holds the old data: $0 restart"; fi + if instance_running "$API_INSTANCE"; then say "restarting the API so that it serves the imported data"; restart_api; fi } cmd_data_export() { @@ -653,6 +705,7 @@ main() { data-import) cmd_data_import "$@" ;; data-export) cmd_data_export "$@" ;; has-data) ! data_is_empty ;; + is-running) need_apptainer; instance_running "$API_INSTANCE" || instance_running "$CUOPT_INSTANCE" ;; upgrade) cmd_upgrade "$@" ;; env) cmd_env ;; clean-cache) need_apptainer; "$APPTAINER" cache clean --force ;; diff --git a/docs/SPARK_APPTAINER.md b/docs/SPARK_APPTAINER.md index 0944588..8254841 100644 --- a/docs/SPARK_APPTAINER.md +++ b/docs/SPARK_APPTAINER.md @@ -78,15 +78,16 @@ nothing needs to be installed with `pip`. ## Daily use -```bash -qc start # cuOpt + API; after a reboot, run it again -qc status # what runs, whether cuOpt is reachable, how much data there is -qc logs # API log; `qc logs cuopt` for the solver, add -f to follow -qc update # the newest prices and FX rates from Yahoo Finance -qc upgrade # the newest version of the code: download, rebuild, restart -~/Portfolio_Risk_Management/install.sh --rebuild-data # set the data aside and build it anew -qc stop # stop both containers -``` +| Command | What it does | +|---|---| +| `qc status` | Shows what runs, whether cuOpt is reachable, and how much data there is. It warns when the API serves a different data set than the data folder. | +| `qc logs` | Shows the API log. `qc logs cuopt` shows the solver's log; add `-f` to follow. | +| `qc start` | Starts cuOpt and the API. Run it again after a reboot. | +| `qc stop` | Stops both containers. | +| `qc restart` | Stops and starts both. | +| `qc update` | Fetches the newest prices and FX rates from Yahoo Finance, then restarts the API. | +| `qc upgrade` | Downloads the newest version of the code, rebuilds and restarts. | +| `~/Portfolio_Risk_Management/install.sh --rebuild-data` | Sets the data aside in `~/quantcore/data.` and builds it anew (~10 min). | The containers keep running after you log out. They stop when the Spark reboots. @@ -224,7 +225,8 @@ and restarts the service. | `port 5001 is taken` | Another user or program uses the port. Set the suggested `QC_API_PORT` in `~/quantcore/quantcore.env`, then run `qc start`. | | `qc status` shows cuOpt as not answering | Run `qc logs cuopt`. Check that `nvidia-smi` works. Then `qc restart`. Optimisations keep working on HiGHS in the meantime. | | The cuOpt log says `Read-only file system: '/opt/cuopt/.cupy'` | cuOpt was started without `qc.sh`. Start it with `qc start`, or give it a writable home directory as described in "Running cuOpt by hand". | -| `/optimize` answers 503, "no market data yet" | Run `qc data-import ` or `qc setup`. | +| The test bench or `/optimize` answers 503, "no market data yet" | The data folder is empty. Run `qc status`: it lists earlier data sets in `~/quantcore/data.` and the command to restore one, `qc data-import `. Otherwise build the data with `qc setup --offline` (~10 min). | +| `qc status` warns that the API serves a different data set | Run `qc restart`. | | Yahoo errors during `qc setup` or `qc update` | Yahoo limits how fast it answers. Run the command again later; stored data stays intact. | | `FATAL: ... failed to create build parent dir: stat /scratch/...` | Your account sets `APPTAINER_TMPDIR` to a folder that does not exist. The installer and `qc` now detect this, use a folder that works, and save the choice in `~/quantcore/quantcore.env`. Run the installer again. | | `git: command not found` | Use the one-line installer. It needs no git. | diff --git a/install.sh b/install.sh index b7cba34..e2c4836 100755 --- a/install.sh +++ b/install.sh @@ -97,8 +97,12 @@ main() { local data_dir aside data_dir="$("$qc" env | sed -n 's|^QC_DATA=||p')" aside="$data_dir.$(date +%Y%m%d-%H%M%S)" - "$qc" stop >/dev/null 2>&1 || true - mv "$data_dir" "$aside" + "$qc" stop + ! "$qc" is-running || die "the service is still running; stop it (qc stop) and run this again" + # move the contents, never the folder itself: a container that still had the folder mounted would + # keep reading and writing the moved copy, while every new command looks at the empty new folder + mkdir -p "$aside" + find "$data_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$aside/" \; say " moved the existing data set to $aside" fi if [[ -n "$data" ]]; then @@ -124,7 +128,7 @@ main() { say " candidates: the list shipped in the repository; companies Yahoo no longer knows are left out" "$qc" setup --offline else - say " skipped (--no-data). Later: qc setup or qc data-import FILE" + say " no data yet. Build it (~10 min): qc setup --offline or import a prepared one: qc data-import FILE" fi step "5/5 Commands: qc, quantcore, quantcore-api" diff --git a/quantcore/cli.py b/quantcore/cli.py index 576a6a8..3eec322 100644 --- a/quantcore/cli.py +++ b/quantcore/cli.py @@ -254,7 +254,16 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) - return args.func(args) + try: + return args.func(args) + except FileNotFoundError as exc: + if "run the setup service first" not in str(exc): + raise + # no traceback for the commonest first-day mistake: a command before the data exists + print(f"error: no market data yet ({str(exc).split(';')[0]}).\n" + " Build it first: quantcore setup (on a DGX Spark: qc setup --offline),\n" + " or import a prepared data set: qc data-import FILE", file=sys.stderr) + return 2 if __name__ == "__main__":