54721801ba
On a student Spark the API answered with 746 stocks while `quantcore risk` found no data, and after a restart the test bench answered 503 "no market data yet". Cause: `install.sh --rebuild-data` renamed the data folder; a container that still has a folder mounted keeps using it under its new name, and the data jobs ran inside the API -- so the rebuilt data went into ~/quantcore/data.<date> while the commands read the new, empty ~/quantcore/data. Pasting the README's command block made it visible: it contained `qc stop / qc start`, which stopped the service. - install.sh --rebuild-data stops the service, checks that it stopped, and moves the folder's contents, never the folder itself - data jobs (setup, update) always run in a fresh container on the data folder; a running API is restarted afterwards (stopped during setup); `qc data-import` restarts it too; stop waits and forces if needed - `qc status` warns when the API serves a different data set than the folder, and lists earlier data sets with the command to restore one; `qc start` does the same when the folder is empty - the CLI prints three lines instead of a traceback when there is no data - README and guide: command blocks only read, so they can be pasted whole; start, stop, update, upgrade and --rebuild-data are in tables - lookups that may find nothing are guarded under pipefail (one ended `qc status` silently) Tested on a stripped-down machine: the README block pasted whole keeps the service running; `quantcore update` restarts the API and status agrees; the broken state is detected and restored with `qc data-import`; `install.sh --rebuild-data` with the service running rebuilt 745 stocks, and API, CLI and folder agree afterwards. 142 tests pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
718 lines
31 KiB
Bash
Executable File
718 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# qc.sh -- quantcore standalone on a DGX Spark with Apptainer: no root, no Docker, no Tailscale.
|
|
#
|
|
# cuOpt and quantcore each run in their own Apptainer container on this machine and talk over
|
|
# 127.0.0.1. The API and the browser test bench listen on 127.0.0.1 too, so they are used from this
|
|
# Spark (its desktop browser, a terminal, a notebook) and are not reachable from the network.
|
|
#
|
|
# qc.sh install once: config, cuOpt image, quantcore image, data directory
|
|
# qc.sh data-import FILE copy a prepared data set (.tar.gz or directory) ...
|
|
# qc.sh setup ... or build it from Yahoo Finance (~15 min)
|
|
# qc.sh start cuOpt + API -> http://127.0.0.1:5001/ui/ and /docs
|
|
# qc.sh status | logs [api|cuopt] [-f] | stop | restart
|
|
# qc.sh cli <command> ... the quantcore CLI in the container, no API needed
|
|
# (e.g. qc.sh cli optimize --cash 1e6 --cvar-limit 40000)
|
|
# qc.sh api <command> ... quantcore-api against the local API (e.g. qc.sh api selftest)
|
|
# qc.sh help all commands
|
|
#
|
|
# Settings: $QC_HOME/quantcore.env (QC_HOME defaults to ~/quantcore); see quantcore.env.example.
|
|
# Guide: docs/SPARK_APPTAINER.md.
|
|
set -euo pipefail
|
|
|
|
# resolve the link ~/.local/bin/qc (made by install.sh) to the script in the repository
|
|
SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
|
|
REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
QC_HOME="${QC_HOME:-$HOME/quantcore}"
|
|
CONFIG="$QC_HOME/quantcore.env"
|
|
APPTAINER="${APPTAINER:-apptainer}"
|
|
API_INSTANCE="quantcore-api"
|
|
CUOPT_INSTANCE="quantcore-cuopt"
|
|
ME="$(id -un)"
|
|
|
|
# --- settings: defaults < config file < environment ------------------------------------------
|
|
_env_overrides="$(declare -p $(compgen -v | grep -E '^(QC_|QUANTCORE_)' || true) 2>/dev/null || true)"
|
|
QC_DATA="$QC_HOME/data"
|
|
QC_IMAGES="$QC_HOME/images"
|
|
QC_API_PORT=5001
|
|
QC_CUOPT_PORT=5000
|
|
QC_CUOPT_START=auto
|
|
QC_CUOPT_IMAGE="docker://nvidia/cuopt:26.8.0-cuda13.3-py3.14"
|
|
QC_IMAGE=build
|
|
QC_PYTHON_IMAGE="docker://python:3.12-slim" # base of the no-build fallback (same as quantcore.def)
|
|
if [[ -f "$CONFIG" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$CONFIG"
|
|
set +a
|
|
fi
|
|
[[ -n "$_env_overrides" ]] && eval "$_env_overrides"
|
|
unset _env_overrides
|
|
QC_API_URL="http://127.0.0.1:$QC_API_PORT"
|
|
QC_VENV="$QC_HOME/venv"
|
|
PY_SIF="$QC_IMAGES/python.sif"
|
|
CUOPT_URL="${QUANTCORE_CUOPT_URL:-http://127.0.0.1:$QC_CUOPT_PORT}"
|
|
CUOPT_EXTERNAL=0 # 1: QUANTCORE_CUOPT_URL names a cuOpt somewhere else -- never start our own
|
|
[[ -n "${QUANTCORE_CUOPT_URL:-}" ]] && CUOPT_EXTERNAL=1
|
|
|
|
# --- helpers ----------------------------------------------------------------------------------
|
|
say() { printf '%s\n' "$*"; }
|
|
step() { printf '\n>> %s\n' "$*"; }
|
|
warn() { printf 'warning: %s\n' "$*" >&2; }
|
|
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
|
|
|
|
quantcore_sif() { if [[ "$QC_IMAGE" == *.sif && -f "$QC_IMAGE" ]]; then say "$QC_IMAGE"; else say "$QC_IMAGES/quantcore.sif"; fi; }
|
|
|
|
# How quantcore runs: "sif" = the quantcore image (built or prebuilt); "venv" = the no-build fallback,
|
|
# a plain Python image plus a virtualenv in $QC_HOME/venv (for a Spark that allows no image builds).
|
|
qc_mode() {
|
|
if [[ "$QC_IMAGE" == "venv" ]]; then say venv; return; fi
|
|
if [[ -f "$(quantcore_sif)" ]]; then say sif; return; fi
|
|
if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" ]]; then say venv; return; fi
|
|
say none
|
|
}
|
|
qc_image() { if [[ "$(qc_mode)" == venv ]]; then say "$PY_SIF"; else quantcore_sif; fi; }
|
|
qc_bin() { if [[ "$(qc_mode)" == venv ]]; then say "$QC_VENV/bin/$1"; else say "$1"; fi; }
|
|
cuopt_sif() { if [[ -f "$QC_CUOPT_IMAGE" ]]; then say "$QC_CUOPT_IMAGE"; else say "$QC_IMAGES/cuopt.sif"; fi; }
|
|
|
|
need_apptainer() {
|
|
command -v "$APPTAINER" >/dev/null 2>&1 \
|
|
|| die "'$APPTAINER' not found. This Spark needs Apptainer (ask the administrator), or set APPTAINER=singularity."
|
|
}
|
|
|
|
need_image() { # path what
|
|
[[ -f "$1" ]] || die "$2 image missing ($1) -- run: $0 install"
|
|
}
|
|
|
|
# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3, else plain bash:
|
|
# a barebone Spark may have neither of the first two.
|
|
http_get() {
|
|
local url="$1" timeout="${2:-3}"
|
|
if command -v curl >/dev/null 2>&1; then
|
|
curl -sf --max-time "$timeout" "$url"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
python3 -c 'import sys, urllib.request as u; sys.stdout.write(u.urlopen(sys.argv[1], timeout=float(sys.argv[2])).read().decode())' \
|
|
"$url" "$timeout" 2>/dev/null
|
|
else
|
|
http_get_bash "$url" "$timeout"
|
|
fi
|
|
}
|
|
|
|
# HTTP/1.0 GET over bash's /dev/tcp, for the http://host:port/path addresses qc.sh checks.
|
|
http_get_bash() {
|
|
local rest="${1#http://}" t="${2:-3}" hostport path host port
|
|
hostport="${rest%%/*}"; path="${rest#"$hostport"}"; [[ -n "$path" ]] || path="/"
|
|
host="${hostport%%:*}"; port=80; [[ "$hostport" == *:* ]] && port="${hostport##*:}"
|
|
(
|
|
exec 3<>"/dev/tcp/$host/$port" || exit 1
|
|
printf 'GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' "$path" "$host" >&3
|
|
IFS= read -r -t "$t" status <&3 || exit 1
|
|
[[ "$status" =~ ^HTTP/[0-9.]+\ 2[0-9][0-9] ]] || exit 1
|
|
while IFS= read -r -t "$t" line <&3; do line="${line%$'\r'}"; [[ -z "$line" ]] && break; done
|
|
while IFS= read -r -t "$t" line <&3 || [[ -n "$line" ]]; do printf '%s\n' "$line"; line=""; done
|
|
) 2>/dev/null
|
|
}
|
|
|
|
port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
|
|
|
|
free_port() { # first free port from $1 upwards
|
|
local p="$1"
|
|
while port_busy "$p"; do p=$((p + 1)); done
|
|
say "$p"
|
|
}
|
|
|
|
instance_running() {
|
|
"$APPTAINER" instance list 2>/dev/null | awk 'NR > 1 {print $1}' | grep -qx "$1"
|
|
}
|
|
|
|
instance_log() { # name out|err -> path of the instance log (Apptainer keeps them per host and user)
|
|
# this host first: a home directory on a network drive holds the logs of every machine; the user
|
|
# directory may be spelt "name" or "name@domain", so match any
|
|
local f
|
|
for f in "$HOME/.apptainer/instances/logs/$(hostname)/"*"/$1.$2" "$HOME/.apptainer/instances/logs/"*/*"/$1.$2"; do
|
|
[[ -f "$f" ]] && { say "$f"; return 0; }
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Apptainer's working and cache directories. Some installations export APPTAINER_TMPDIR or
|
|
# APPTAINER_CACHEDIR for a convention such as /scratch/$USER that this machine does not have; every
|
|
# pull then dies with "failed to create build parent dir". Create the directory, or fall back.
|
|
APPTAINER_DIRS_READY=0
|
|
usable_dir() { mkdir -p "$1" 2>/dev/null && [[ -d "$1" && -w "$1" ]]; }
|
|
|
|
prepare_apptainer_dirs() {
|
|
[[ "$APPTAINER_DIRS_READY" == 1 ]] && return 0
|
|
local name var legacy dir fallback tmp_free_gb
|
|
for name in TMPDIR CACHEDIR; do
|
|
var="APPTAINER_$name"; legacy="SINGULARITY_$name"
|
|
dir="${!var:-${!legacy:-}}"
|
|
[[ -n "$dir" ]] || continue # not set: Apptainer's defaults (/tmp, ~/.apptainer/cache) apply
|
|
usable_dir "$dir" && continue
|
|
if [[ "$name" == TMPDIR ]]; then
|
|
# unpacking the cuOpt image needs ~12 GB: local /tmp if it has the room, else our own directory
|
|
tmp_free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 ))
|
|
fallback="$QC_HOME/apptainer-tmp"
|
|
(( tmp_free_gb >= 15 )) && fallback="/tmp/$ME-apptainer-tmp"
|
|
else
|
|
fallback="$QC_HOME/apptainer-cache"
|
|
fi
|
|
usable_dir "$fallback" || die "$var=$dir does not exist, and neither it nor $fallback can be created"
|
|
export "$var=$fallback"
|
|
unset "$legacy"
|
|
if [[ -f "$CONFIG" ]] && ! grep -q "^$var=" "$CONFIG"; then
|
|
printf '\n# set by qc.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \
|
|
"$var=$fallback" >> "$CONFIG"
|
|
warn "$var=$dir (set in your environment) does not exist and cannot be created.
|
|
Using $fallback instead; saved in $CONFIG."
|
|
else
|
|
warn "$var=$dir does not exist and cannot be created; using $fallback for this run."
|
|
fi
|
|
done
|
|
APPTAINER_DIRS_READY=1
|
|
}
|
|
|
|
pull_image() { # destination source -- download an image to a .sif file, with a useful error
|
|
prepare_apptainer_dirs
|
|
rm -f "$1.part"
|
|
if ! "$APPTAINER" pull "$1.part" "$2"; then
|
|
rm -f "$1.part"
|
|
die "could not download $2. Check that this machine reaches the registry (a proxy needs
|
|
https_proxy set), and that ${APPTAINER_TMPDIR:-/tmp} has ~12 GB free (APPTAINER_TMPDIR chooses another
|
|
directory). Then run the command again."
|
|
fi
|
|
mv -f "$1.part" "$1"
|
|
}
|
|
|
|
show_log_tail() { # name
|
|
local f
|
|
for kind in err out; do
|
|
if f="$(instance_log "$1" "$kind")"; then
|
|
say "--- last lines of $f"
|
|
tail -n 25 "$f" >&2 || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
wait_for() { # url seconds label
|
|
local url="$1" limit="$2" label="$3" waited=0
|
|
printf ' waiting for %s ' "$label"
|
|
until http_get "$url" 2 >/dev/null; do
|
|
if (( waited >= limit )); then printf ' timed out after %ss\n' "$limit"; return 1; fi
|
|
printf '.'; sleep 2; waited=$((waited + 2))
|
|
done
|
|
printf ' up (%ss)\n' "$waited"
|
|
}
|
|
|
|
# Hand the settings to a container through APPTAINERENV_* (unlike --env these survive commas,
|
|
# e.g. in no_proxy). The containers start with --cleanenv, so nothing else from the host leaks in:
|
|
# no PYTHONPATH, no activated virtualenv or conda, no LD_LIBRARY_PATH.
|
|
export_quantcore_env() {
|
|
local v
|
|
for v in $(compgen -v | grep '^QUANTCORE_' || true); do
|
|
export "APPTAINERENV_$v=${!v}"
|
|
done
|
|
export APPTAINERENV_QUANTCORE_DATA_DIR=/data
|
|
export APPTAINERENV_QUANTCORE_CUOPT_URL="$CUOPT_URL"
|
|
export APPTAINERENV_QUANTCORE_API_HOST=127.0.0.1
|
|
export APPTAINERENV_QUANTCORE_API_PORT="$QC_API_PORT"
|
|
export APPTAINERENV_QUANTCORE_API_URL="$QC_API_URL"
|
|
# what quantcore.def's %environment sets, for the venv fallback (an explicit value wins)
|
|
export APPTAINERENV_PYTHONNOUSERSITE=1 APPTAINERENV_PYTHONUNBUFFERED=1 APPTAINERENV_PYTHONDONTWRITEBYTECODE=1
|
|
export APPTAINERENV_OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-4}" APPTAINERENV_OMP_NUM_THREADS="${OMP_NUM_THREADS:-4}"
|
|
export APPTAINERENV_MKL_NUM_THREADS="${MKL_NUM_THREADS:-4}"
|
|
for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY TZ; do
|
|
[[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# Run a command in the quantcore container (the CLI needs no running API).
|
|
qc_binds() { # bind options for the quantcore container
|
|
local b=(--bind "$QC_DATA:/data")
|
|
[[ "$(qc_mode)" == venv ]] && b+=(--bind "$QC_VENV")
|
|
printf '%s\n' "${b[@]}"
|
|
}
|
|
|
|
run_qc() { # command [args]: quantcore | quantcore-api | python ...
|
|
need_apptainer
|
|
[[ "$(qc_mode)" != none ]] || die "no quantcore image yet -- run: $0 install"
|
|
mkdir -p "$QC_DATA"
|
|
export_quantcore_env
|
|
local cmd="$1"; shift
|
|
local binds; mapfile -t binds < <(qc_binds)
|
|
exec "$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin "$cmd")" "$@"
|
|
}
|
|
|
|
restore_hint() { # list earlier data sets next to $QC_DATA (from --rebuild-data) that can be restored
|
|
local d n=0 when stocks
|
|
for d in $(ls -1dt "$QC_DATA".* 2>/dev/null); do
|
|
[[ -f "$d/universe/universe.csv" ]] || continue
|
|
if [[ $n == 0 ]]; then say " earlier data sets found (newest first):"; fi
|
|
when="$(grep -o '"setup_at": *"[^"]*"' "$d/manifest.json" 2>/dev/null | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/' || true)"
|
|
stocks=$(( $(wc -l < "$d/universe/universe.csv") - 1 ))
|
|
say " $d ($stocks stocks, built ${when:-?})"
|
|
n=$((n + 1))
|
|
if [[ $n -ge 4 ]]; then break; fi
|
|
done
|
|
if [[ $n -gt 0 ]]; then say " restore one with: $0 data-import <folder>"; fi
|
|
}
|
|
|
|
data_is_empty() { [[ -z "$(ls -A "$QC_DATA" 2>/dev/null | grep -v -x -e jobs -e '.*\.log')" ]]; }
|
|
|
|
# --- install ----------------------------------------------------------------------------------
|
|
cmd_install() {
|
|
need_apptainer
|
|
local force=0; [[ "${1:-}" == "--force" ]] && force=1
|
|
step "Apptainer"
|
|
say " $("$APPTAINER" --version)"
|
|
"$APPTAINER" instance run --help >/dev/null 2>&1 \
|
|
|| die "this Apptainer has no 'instance run' (needs Apptainer >= 1.1); ask the administrator to update it"
|
|
|
|
step "Directories and settings ($QC_HOME)"
|
|
mkdir -p "$QC_HOME" "$QC_DATA" "$QC_IMAGES"
|
|
if [[ -f "$CONFIG" ]]; then
|
|
say " keeping $CONFIG"
|
|
else
|
|
cp "$SCRIPT_DIR/quantcore.env.example" "$CONFIG"
|
|
say " wrote $CONFIG"
|
|
fi
|
|
|
|
prepare_apptainer_dirs
|
|
step "GPU"
|
|
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
|
|
say " $(nvidia-smi -L | head -n 1)"
|
|
else
|
|
warn "no GPU visible (nvidia-smi). cuOpt will not start; optimisations will use HiGHS on the CPU."
|
|
fi
|
|
local free_gb; free_gb=$(( $(df -Pk "$QC_IMAGES" | awk 'NR == 2 {print $4}') / 1024 / 1024 ))
|
|
say " free disk under $QC_IMAGES: ${free_gb} GB (the cuOpt pull needs ~12 GB while it runs, 3.3 GB after)"
|
|
|
|
if [[ "$QC_CUOPT_START" != "no" && "$CUOPT_EXTERNAL" == 0 ]]; then
|
|
cmd_pull_cuopt $([[ $force == 1 ]] && echo --force)
|
|
fi
|
|
step "quantcore image"
|
|
get_quantcore_image "$force"
|
|
step "Self-test of the quantcore container ($(qc_mode) mode)"
|
|
qc_selftest
|
|
|
|
[[ "${QC_FROM_INSTALLER:-0}" == 1 ]] && return 0 # install.sh does the data and the start itself
|
|
step "Data ($QC_DATA)"
|
|
if data_is_empty; then
|
|
say " empty. Either copy a prepared data set: $0 data-import quantcore_data.tar.gz"
|
|
say " or build it from Yahoo Finance (~15 min): $0 setup"
|
|
else
|
|
say " present ($(du -sh "$QC_DATA" | cut -f1))"
|
|
fi
|
|
step "Done. Next: $0 start"
|
|
}
|
|
|
|
cmd_pull_cuopt() {
|
|
need_apptainer
|
|
local dst="$QC_IMAGES/cuopt.sif"
|
|
if [[ -f "$QC_CUOPT_IMAGE" ]]; then
|
|
step "cuOpt image: $QC_CUOPT_IMAGE (provided on this machine, not pulled)"
|
|
return 0
|
|
fi
|
|
if [[ -f "$dst" && "${1:-}" != "--force" ]]; then
|
|
step "cuOpt image present: $dst"
|
|
return 0
|
|
fi
|
|
step "Pulling cuOpt: $QC_CUOPT_IMAGE -> $dst (~3.3 GB, several minutes)"
|
|
mkdir -p "$QC_IMAGES"
|
|
pull_image "$dst" "$QC_CUOPT_IMAGE"
|
|
}
|
|
|
|
qc_selftest() {
|
|
# from /: `python -c` puts the working directory first on sys.path, and from the repository root
|
|
# it would import the sources there instead of what was installed into the container
|
|
( cd / && export_quantcore_env
|
|
local binds; mapfile -t binds < <(qc_binds)
|
|
"$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin python)" -c '
|
|
import importlib.resources as r, scipy, quantcore, quantcore.api.app as app
|
|
from scipy.optimize import milp
|
|
assert (r.files("quantcore.api") / "ui" / "index.html").is_file(), "test bench missing"
|
|
print(" quantcore", quantcore.__version__, "| scipy", scipy.__version__, "| UI mounted:", app.UI_ENABLED)' )
|
|
}
|
|
|
|
get_quantcore_image() { # force(0|1)
|
|
local dst="$QC_IMAGES/quantcore.sif"
|
|
case "$QC_IMAGE" in
|
|
build)
|
|
if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst (rebuild: $0 build)"; return 0; fi
|
|
if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" && "$1" != 1 ]]; then
|
|
say " present: virtualenv $QC_VENV (no-build fallback; reinstall: $0 build)"; return 0
|
|
fi
|
|
build_sif || {
|
|
say " -> using the no-build setup instead: a plain Python image plus a virtualenv in $QC_VENV"
|
|
say " (same libraries, same results)"
|
|
build_venv
|
|
} ;;
|
|
venv)
|
|
if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" && "$1" != 1 ]]; then say " present: $QC_VENV"; else build_venv; fi ;;
|
|
docker://*|oras://*|library://*|http://*|https://*)
|
|
if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst"; else
|
|
say " pulling $QC_IMAGE -> $dst"
|
|
pull_image "$dst" "$QC_IMAGE"
|
|
fi ;;
|
|
*)
|
|
[[ -f "$QC_IMAGE" ]] || die "QC_IMAGE=$QC_IMAGE is neither 'build', a URL, nor an existing .sif file"
|
|
say " using $QC_IMAGE" ;;
|
|
esac
|
|
}
|
|
|
|
build_sif() { # build quantcore.sif from the definition; non-zero when this Apptainer cannot build
|
|
local dst="$QC_IMAGES/quantcore.sif" log="$QC_HOME/build.log" reason
|
|
mkdir -p "$QC_IMAGES"
|
|
say " building $dst from $REPO_DIR (a few minutes; log: $log)"
|
|
prepare_apptainer_dirs
|
|
rm -f "$dst.part"
|
|
# %files paths in the definition are relative to the repository root. Without root Apptainer
|
|
# builds through --fakeroot (user namespaces); many Sparks do not allow that. The output goes to a
|
|
# log: on such a Spark Apptainer's red ERROR line is expected, and the caller falls back.
|
|
if ! (cd "$REPO_DIR" && "$APPTAINER" build "$dst.part" deploy/apptainer/quantcore.def) >"$log" 2>&1; then
|
|
rm -f "$dst.part"
|
|
reason="$(grep -E 'FATAL|ERROR' "$log" | tail -n 1 | sed -e 's/\x1b\[[0-9;]*m//g' -e 's/^[A-Z]* *: *//')"
|
|
say " no image build on this machine${reason:+ ($reason)}"
|
|
return 1
|
|
fi
|
|
mv -f "$dst.part" "$dst"
|
|
say " built $dst ($(du -h "$dst" | cut -f1))"
|
|
}
|
|
|
|
build_venv() { # the no-build fallback: pull a plain Python image, install quantcore into a virtualenv
|
|
mkdir -p "$QC_IMAGES" "$QC_HOME"
|
|
if [[ ! -f "$PY_SIF" ]]; then
|
|
say " pulling $QC_PYTHON_IMAGE -> $PY_SIF"
|
|
pull_image "$PY_SIF" "$QC_PYTHON_IMAGE"
|
|
fi
|
|
say " installing quantcore into $QC_VENV (pinned versions from deploy/apptainer/constraints.txt)"
|
|
# a copy of the sources under QC_HOME (not $TMPDIR, which may point nowhere on a managed machine)
|
|
local work; work="$(mktemp -d "$QC_HOME/.quantcore-src.XXXXXX")"
|
|
# install from a copy: pip writes build/ and *.egg-info next to the sources
|
|
cp -r "$REPO_DIR/pyproject.toml" "$REPO_DIR/README.md" "$REPO_DIR/quantcore" "$work/"
|
|
"$APPTAINER" exec --cleanenv --bind "$QC_HOME" --bind "$work" "$PY_SIF" python -m venv --clear "$QC_VENV"
|
|
"$APPTAINER" exec --cleanenv --env PIP_DISABLE_PIP_VERSION_CHECK=1 --bind "$QC_HOME" --bind "$work" \
|
|
--bind "$REPO_DIR/deploy/apptainer/constraints.txt:/tmp/qc-constraints.txt" "$PY_SIF" \
|
|
"$QC_VENV/bin/pip" install --quiet --no-cache-dir -c /tmp/qc-constraints.txt "$work"
|
|
rm -rf "$work"
|
|
( cd / && "$APPTAINER" exec --cleanenv --bind "$QC_VENV" "$PY_SIF" "$QC_VENV/bin/quantcore" --help >/dev/null ) \
|
|
|| die "quantcore was installed into $QC_VENV but does not start (see the error above)"
|
|
say " installed"
|
|
}
|
|
|
|
cmd_build() { # rebuild after a code change, in the mode in use
|
|
need_apptainer
|
|
local mode; mode="$(qc_mode)"
|
|
step "Rebuilding quantcore ($([[ $mode == venv ]] && echo "virtualenv $QC_VENV" || echo "image $QC_IMAGES/quantcore.sif"))"
|
|
if [[ "$mode" == venv ]]; then
|
|
build_venv
|
|
elif [[ "$QC_IMAGE" == build ]]; then
|
|
build_sif || die "the image build failed (log: $QC_HOME/build.log). Without root it needs Apptainer's
|
|
--fakeroot. Use the no-build setup instead: set QC_IMAGE=venv in $CONFIG and run: $0 build"
|
|
else
|
|
die "QC_IMAGE=$QC_IMAGE is a prebuilt image; nothing to build"
|
|
fi
|
|
if instance_running "$API_INSTANCE"; then say " the running API still uses the old code: $0 restart"; fi
|
|
}
|
|
|
|
# --- run --------------------------------------------------------------------------------------
|
|
start_cuopt() {
|
|
if [[ "$CUOPT_EXTERNAL" == 1 ]]; then
|
|
if http_get "$CUOPT_URL/cuopt/health" >/dev/null; then say "cuOpt: using $CUOPT_URL (QUANTCORE_CUOPT_URL)"
|
|
else warn "cuOpt at $CUOPT_URL (QUANTCORE_CUOPT_URL) does not answer -- optimisations will use HiGHS"; fi
|
|
return 0
|
|
fi
|
|
if [[ "$QC_CUOPT_START" == "no" ]]; then
|
|
say "cuOpt: not started (QC_CUOPT_START=no) -- optimisations use HiGHS on the CPU"
|
|
return 0
|
|
fi
|
|
if http_get "$CUOPT_URL/cuopt/health" >/dev/null; then
|
|
if instance_running "$CUOPT_INSTANCE"; then
|
|
say "cuOpt: running ($CUOPT_INSTANCE, $CUOPT_URL)"
|
|
elif [[ "$QC_CUOPT_START" == "yes" ]]; then
|
|
die "QC_CUOPT_START=yes, but another cuOpt already serves port $QC_CUOPT_PORT. Use it (QC_CUOPT_START=auto)
|
|
or pick another port: QC_CUOPT_PORT=$(free_port $((QC_CUOPT_PORT + 100))) in $CONFIG"
|
|
else
|
|
say "cuOpt: using the server that already answers at $CUOPT_URL"
|
|
fi
|
|
return 0
|
|
fi
|
|
if instance_running "$CUOPT_INSTANCE"; then # still starting, or hung
|
|
wait_for "$CUOPT_URL/cuopt/health" 180 "cuOpt ($CUOPT_INSTANCE)" && return 0
|
|
show_log_tail "$CUOPT_INSTANCE"
|
|
die "cuOpt instance runs but does not answer; try: $0 restart"
|
|
fi
|
|
if port_busy "$QC_CUOPT_PORT"; then
|
|
die "port $QC_CUOPT_PORT is taken by something that is not cuOpt.
|
|
Set QC_CUOPT_PORT=$(free_port $((QC_CUOPT_PORT + 100))) in $CONFIG and run: $0 start"
|
|
fi
|
|
local sif; sif="$(cuopt_sif)"
|
|
need_image "$sif" cuOpt
|
|
say "cuOpt: starting instance $CUOPT_INSTANCE on 127.0.0.1:$QC_CUOPT_PORT (GPU)"
|
|
# NVIDIA's entrypoint sets HOME=/opt/cuopt. That is writable under Docker but read-only in a SIF,
|
|
# and the solver process dies on its first job when CuPy cannot write its kernel cache there.
|
|
# So the server runs with HOME on a writable directory of ours (CuPy, Numba and CUDA caches).
|
|
local cuopt_home="$QC_HOME/cuopt-home"
|
|
mkdir -p "$cuopt_home"
|
|
( [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]] && export APPTAINERENV_CUDA_VISIBLE_DEVICES="$CUDA_VISIBLE_DEVICES"
|
|
cd /
|
|
# the image's entrypoint runs its arguments: the cuOpt REST server, on localhost only
|
|
"$APPTAINER" instance run --nv --cleanenv --bind "$cuopt_home" "$sif" "$CUOPT_INSTANCE" \
|
|
env HOME="$cuopt_home" python -m cuopt_server.cuopt_service \
|
|
-i 127.0.0.1 -p "$QC_CUOPT_PORT" -l "${QC_CUOPT_LOG_LEVEL:-info}" >/dev/null )
|
|
if ! wait_for "$CUOPT_URL/cuopt/health" 180 "cuOpt"; then
|
|
show_log_tail "$CUOPT_INSTANCE"
|
|
warn "cuOpt did not come up -- the API starts anyway and optimises with HiGHS"
|
|
fi
|
|
}
|
|
|
|
start_api() {
|
|
if instance_running "$API_INSTANCE"; then
|
|
if http_get "$QC_API_URL/health" >/dev/null; then say "API: running ($API_INSTANCE, $QC_API_URL)"; return 0; fi
|
|
wait_for "$QC_API_URL/health" 60 "API ($API_INSTANCE)" && return 0
|
|
show_log_tail "$API_INSTANCE"
|
|
die "the API instance runs but does not answer; try: $0 restart"
|
|
fi
|
|
if port_busy "$QC_API_PORT"; then
|
|
die "port $QC_API_PORT is taken (another user's quantcore?).
|
|
Set QC_API_PORT=$(free_port $((QC_API_PORT + 10))) in $CONFIG and run: $0 start"
|
|
fi
|
|
local mode; mode="$(qc_mode)"
|
|
[[ "$mode" != none ]] || die "no quantcore image yet -- run: $0 install"
|
|
mkdir -p "$QC_DATA"
|
|
say "API: starting instance $API_INSTANCE on $QC_API_URL ($mode mode)"
|
|
( export_quantcore_env
|
|
local binds; mapfile -t binds < <(qc_binds)
|
|
# start from /: pydantic-settings reads a .env from the working directory, and the container's
|
|
# / has none (instances take the caller's directory; they have no --cwd option)
|
|
cd /
|
|
if [[ "$mode" == venv ]]; then # the Python image's runscript executes its arguments
|
|
"$APPTAINER" instance run --cleanenv "${binds[@]}" "$PY_SIF" "$API_INSTANCE" "$QC_VENV/bin/quantcore" serve >/dev/null
|
|
else # quantcore.def's startscript: quantcore serve
|
|
"$APPTAINER" instance start --cleanenv "${binds[@]}" "$(quantcore_sif)" "$API_INSTANCE" >/dev/null
|
|
fi )
|
|
if ! wait_for "$QC_API_URL/health" 90 "API"; then
|
|
show_log_tail "$API_INSTANCE"
|
|
die "the API did not come up"
|
|
fi
|
|
}
|
|
|
|
cmd_start() {
|
|
need_apptainer
|
|
step "Starting quantcore (user $ME, settings $([[ -f "$CONFIG" ]] && echo "$CONFIG" || echo "defaults"))"
|
|
start_cuopt
|
|
start_api
|
|
if data_is_empty; then
|
|
warn "no market data in $QC_DATA -- the API answers 503 until there is some."
|
|
restore_hint
|
|
say " build it (~10 min): $0 setup --offline or import a prepared one: $0 data-import FILE"
|
|
fi
|
|
local ui=""; [[ "${QUANTCORE_ENABLE_UI:-true}" != "false" ]] && ui="$QC_API_URL/ui/ (test bench)"
|
|
step "Ready -- on this Spark:"
|
|
[[ -n "$ui" ]] && say " $ui"
|
|
say " $QC_API_URL/docs (API, Swagger)"
|
|
say " Python: QuantcoreClient(\"$QC_API_URL\") CLI: $0 api health"
|
|
}
|
|
|
|
cmd_stop() {
|
|
need_apptainer
|
|
local name stopped=0
|
|
for name in "$API_INSTANCE" "$CUOPT_INSTANCE"; do
|
|
if instance_running "$name"; then
|
|
stop_instance "$name"; say "stopped $name"; stopped=1
|
|
fi
|
|
done
|
|
[[ $stopped == 1 ]] || say "nothing running"
|
|
}
|
|
|
|
cmd_status() {
|
|
need_apptainer
|
|
step "Instances of $ME"
|
|
"$APPTAINER" instance list
|
|
step "cuOpt $CUOPT_URL"
|
|
local h
|
|
if h="$(http_get "$CUOPT_URL/cuopt/health")"; then say " up: $h"; else say " not answering"; fi
|
|
step "API $QC_API_URL"
|
|
if h="$(http_get "$QC_API_URL/health")"; then
|
|
local api_setup host_setup
|
|
# `|| true`: under pipefail a grep without a match would end the script silently
|
|
api_setup="$(printf '%s' "$h" | grep -o '"setup_at":"[^"]*"' | head -n 1 || true)"
|
|
host_setup="$(grep -o '"setup_at": *"[^"]*"' "$QC_DATA/manifest.json" 2>/dev/null | head -n 1 | tr -d ' ' || true)"
|
|
if [[ "$api_setup" != "$host_setup" ]]; then
|
|
warn "the running API serves a different data set than $QC_DATA (API ${api_setup:-none}, folder ${host_setup:-none}).
|
|
Run: $0 restart"
|
|
fi
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
printf '%s' "$h" | python3 -c '
|
|
import json, sys
|
|
h = json.load(sys.stdin); s = h.get("store", {}); c = h.get("cuopt", {})
|
|
print(" up: version", h.get("version"), "| base currency", h.get("base_currency"))
|
|
print(" data:", {k: s[k] for k in list(s)[:6]})
|
|
print(" cuOpt seen by the API:", "reachable" if c.get("reachable") else "NOT reachable -> HiGHS")'
|
|
else
|
|
say " up: $h"
|
|
fi
|
|
else
|
|
say " not answering"
|
|
fi
|
|
step "Files"
|
|
say " settings: $CONFIG $([[ -f "$CONFIG" ]] || echo '(missing: defaults in use)')"
|
|
say " data: $QC_DATA ($(du -sh "$QC_DATA" 2>/dev/null | cut -f1 || echo -))"
|
|
data_is_empty && restore_hint
|
|
if grep -q ',wikipedia_stoxx600,' "$QC_DATA/universe/universe.csv" 2>/dev/null; then
|
|
say " NOTE: this data set comes from an earlier version's guessed European symbols and misses many"
|
|
say " companies. Build it anew (~10 min): $REPO_DIR/install.sh --rebuild-data"
|
|
fi
|
|
say " quantcore: $(qc_mode) mode, $(qc_image)$([[ $(qc_mode) == venv ]] && echo " + $QC_VENV")"
|
|
say " cuOpt: $(cuopt_sif)"
|
|
}
|
|
|
|
cmd_logs() {
|
|
local which="${1:-api}" follow=""
|
|
[[ "${2:-}" == "-f" || "${1:-}" == "-f" ]] && follow=1
|
|
[[ "$which" == "-f" ]] && which=api
|
|
local name="$API_INSTANCE"; [[ "$which" == "cuopt" ]] && name="$CUOPT_INSTANCE"
|
|
local out err
|
|
out="$(instance_log "$name" out || true)"; err="$(instance_log "$name" err || true)"
|
|
[[ -n "$out$err" ]] || die "no logs for $name yet"
|
|
if [[ -n "$follow" ]]; then exec tail -n 50 -F ${out:+"$out"} ${err:+"$err"}; fi
|
|
tail -n 50 ${out:+"$out"} ${err:+"$err"}
|
|
}
|
|
|
|
# --- data -------------------------------------------------------------------------------------
|
|
data_job() { # setup|update [args]
|
|
# Always run in a fresh container on $QC_DATA, never inside the running API: the API holds the
|
|
# directory it was started with, and if that is no longer $QC_DATA its writes would end up somewhere
|
|
# the CLI never looks. A running API is restarted afterwards so that it serves the new data; for a
|
|
# full setup it is stopped first (setup rewrites every file it reads).
|
|
local job="$1"; shift
|
|
local api_was_running=0 rc=0
|
|
instance_running "$API_INSTANCE" && api_was_running=1
|
|
if [[ "$job" == setup && "$api_was_running" == 1 ]]; then
|
|
say "stopping the API while the data set is rebuilt"
|
|
stop_instance "$API_INSTANCE"
|
|
fi
|
|
( run_qc quantcore "$job" "$@" ) || rc=$?
|
|
if [[ "$api_was_running" == 1 ]]; then
|
|
say "restarting the API so that it serves the new data"
|
|
restart_api
|
|
fi
|
|
return "$rc"
|
|
}
|
|
|
|
stop_instance() { # name -- stop and wait until it is gone
|
|
local i
|
|
instance_running "$1" || return 0
|
|
"$APPTAINER" instance stop "$1" >/dev/null 2>&1 || "$APPTAINER" instance stop --force "$1" >/dev/null 2>&1 || true
|
|
for i in $(seq 1 30); do instance_running "$1" || return 0; sleep 1; done
|
|
die "could not stop the instance $1 (apptainer instance stop --force $1)"
|
|
}
|
|
|
|
restart_api() {
|
|
stop_instance "$API_INSTANCE"
|
|
local i
|
|
for i in $(seq 1 15); do port_busy "$QC_API_PORT" || break; sleep 1; done
|
|
start_api
|
|
}
|
|
|
|
cmd_data_import() {
|
|
local src="${1:-}" force="${2:-}"
|
|
[[ -n "$src" ]] || die "usage: $0 data-import <quantcore_data.tar.gz | directory> [--force]"
|
|
[[ -e "$src" ]] || die "$src not found"
|
|
mkdir -p "$QC_DATA"
|
|
if ! data_is_empty && [[ "$force" != "--force" ]]; then
|
|
die "$QC_DATA already holds data; add --force to overwrite it"
|
|
fi
|
|
if [[ -d "$src" ]]; then
|
|
cp -a "$src/." "$QC_DATA/"
|
|
else
|
|
tar -xzf "$src" -C "$QC_DATA"
|
|
fi
|
|
say "imported into $QC_DATA ($(du -sh "$QC_DATA" | cut -f1))"
|
|
if instance_running "$API_INSTANCE"; then say "restarting the API so that it serves the imported data"; restart_api; fi
|
|
}
|
|
|
|
cmd_data_export() {
|
|
local dst="${1:-quantcore_data.tar.gz}"
|
|
[[ "$dst" = /* ]] || dst="$PWD/$dst"
|
|
tar -czf "$dst" -C "$QC_DATA" --exclude './jobs' --exclude '*.log' .
|
|
say "wrote $dst ($(du -h "$dst" | cut -f1))"
|
|
}
|
|
|
|
cmd_upgrade() { # newest code from the repository, rebuild, restart -- install.sh does all of it
|
|
[[ -f "$REPO_DIR/install.sh" ]] || die "$REPO_DIR/install.sh missing; download the code again"
|
|
exec bash "$REPO_DIR/install.sh" --dir "$REPO_DIR" --no-data "$@"
|
|
}
|
|
|
|
cmd_env() {
|
|
cat <<EOF
|
|
QC_HOME=$QC_HOME
|
|
config=$CONFIG $([[ -f "$CONFIG" ]] || echo '(missing)')
|
|
QC_DATA=$QC_DATA
|
|
QC_IMAGES=$QC_IMAGES
|
|
quantcore=$(qc_mode) mode: $(qc_image) (QC_IMAGE=$QC_IMAGE)
|
|
cuOpt image=$(cuopt_sif) (QC_CUOPT_IMAGE=$QC_CUOPT_IMAGE)
|
|
API=$QC_API_URL
|
|
cuOpt=$CUOPT_URL (QC_CUOPT_START=$QC_CUOPT_START)
|
|
EOF
|
|
compgen -v | grep '^QUANTCORE_' | while read -r v; do say "$v=${!v}"; done
|
|
}
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
quantcore on this DGX Spark (Apptainer, localhost only). Settings: $CONFIG
|
|
|
|
Setup
|
|
install [--force] config, cuOpt image (pull), quantcore image (build or prebuilt), checks
|
|
build rebuild quantcore from this repository (after a code change / git pull)
|
|
pull-cuopt [--force] download the cuOpt image again
|
|
data-import SRC [--force] copy a prepared data set (.tar.gz or directory) into $QC_DATA
|
|
data-export [FILE] pack the data directory (default quantcore_data.tar.gz)
|
|
setup [--offline] build the data set from Yahoo Finance (~15 min, needs internet)
|
|
update [--full] fetch the newest prices / FX
|
|
upgrade newest code from the repository, rebuild, restart (runs install.sh)
|
|
Run
|
|
start | stop | restart cuOpt + API instances
|
|
status instances, health, data, images
|
|
logs [api|cuopt] [-f] instance logs
|
|
Use
|
|
cli COMMAND ... quantcore CLI in the container: status, risk, performance, optimize, snapshot ...
|
|
api COMMAND ... quantcore-api against $QC_API_URL: health, selftest, optimize ...
|
|
python [ARGS] Python with quantcore installed (QUANTCORE_* set as for the API)
|
|
shell a shell in the quantcore container
|
|
env the effective settings
|
|
clean-cache remove Apptainer's download cache (layers of pulled images)
|
|
EOF
|
|
}
|
|
|
|
main() {
|
|
local cmd="${1:-help}"; [[ $# -gt 0 ]] && shift
|
|
case "$cmd" in
|
|
install) cmd_install "$@" ;;
|
|
build) cmd_build ;;
|
|
pull-cuopt) cmd_pull_cuopt "$@" ;;
|
|
start) cmd_start ;;
|
|
stop) cmd_stop ;;
|
|
restart) cmd_stop; cmd_start ;;
|
|
status) cmd_status ;;
|
|
logs) cmd_logs "$@" ;;
|
|
cli) run_qc quantcore "$@" ;;
|
|
api) run_qc quantcore-api --url "$QC_API_URL" "$@" ;;
|
|
python) run_qc python "$@" ;;
|
|
shell) need_apptainer; export_quantcore_env; mapfile -t binds < <(qc_binds)
|
|
exec "$APPTAINER" shell --cleanenv "${binds[@]}" "$(qc_image)" ;;
|
|
setup|update) data_job "$cmd" "$@" ;;
|
|
data-import) cmd_data_import "$@" ;;
|
|
data-export) cmd_data_export "$@" ;;
|
|
has-data) ! data_is_empty ;;
|
|
is-running) need_apptainer; instance_running "$API_INSTANCE" || instance_running "$CUOPT_INSTANCE" ;;
|
|
upgrade) cmd_upgrade "$@" ;;
|
|
env) cmd_env ;;
|
|
clean-cache) need_apptainer; "$APPTAINER" cache clean --force ;;
|
|
help|-h|--help) usage ;;
|
|
*) usage; die "unknown command: $cmd" ;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|