WIP: one-step installer (test branch)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
herzogflorian 2026-10-07 12:07:59 +02:00
parent 3cd3854b2f
commit 862aaf1b86
2 changed files with 284 additions and 10 deletions

View File

@ -19,7 +19,8 @@
# Guide: docs/SPARK_APPTAINER.md.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# resolve the link ~/.local/bin/qc (made by install.sh) to the script in the repository
SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
QC_HOME="${QC_HOME:-$HOME/quantcore}"
CONFIG="$QC_HOME/quantcore.env"
@ -82,7 +83,8 @@ need_image() { # path what
[[ -f "$1" ]] || die "$2 image missing ($1) -- run: $0 install"
}
# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3 (both on DGX OS).
# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3, else plain bash:
# a barebone Spark may have neither of the first two.
http_get() {
local url="$1" timeout="${2:-3}"
if command -v curl >/dev/null 2>&1; then
@ -91,10 +93,25 @@ http_get() {
python3 -c 'import sys, urllib.request as u; sys.stdout.write(u.urlopen(sys.argv[1], timeout=float(sys.argv[2])).read().decode())' \
"$url" "$timeout" 2>/dev/null
else
return 2
http_get_bash "$url" "$timeout"
fi
}
# HTTP/1.0 GET over bash's /dev/tcp, for the http://host:port/path addresses qc.sh checks.
http_get_bash() {
local rest="${1#http://}" t="${2:-3}" hostport path host port
hostport="${rest%%/*}"; path="${rest#"$hostport"}"; [[ -n "$path" ]] || path="/"
host="${hostport%%:*}"; port=80; [[ "$hostport" == *:* ]] && port="${hostport##*:}"
(
exec 3<>"/dev/tcp/$host/$port" || exit 1
printf 'GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' "$path" "$host" >&3
IFS= read -r -t "$t" status <&3 || exit 1
[[ "$status" =~ ^HTTP/[0-9.]+\ 2[0-9][0-9] ]] || exit 1
while IFS= read -r -t "$t" line <&3; do line="${line%$'\r'}"; [[ -z "$line" ]] && break; done
while IFS= read -r -t "$t" line <&3 || [[ -n "$line" ]]; do printf '%s\n' "$line"; line=""; done
) 2>/dev/null
}
port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
free_port() { # first free port from $1 upwards
@ -108,13 +125,64 @@ instance_running() {
}
instance_log() { # name out|err -> path of the instance log (Apptainer keeps them per host and user)
# this host first: a home directory on a network drive holds the logs of every machine; the user
# directory may be spelt "name" or "name@domain", so match any
local f
for f in "$HOME/.apptainer/instances/logs/"*"/$ME/$1.$2"; do
for f in "$HOME/.apptainer/instances/logs/$(hostname)/"*"/$1.$2" "$HOME/.apptainer/instances/logs/"*/*"/$1.$2"; do
[[ -f "$f" ]] && { say "$f"; return 0; }
done
return 1
}
# Apptainer's working and cache directories. Some installations export APPTAINER_TMPDIR or
# APPTAINER_CACHEDIR for a convention such as /scratch/$USER that this machine does not have; every
# pull then dies with "failed to create build parent dir". Create the directory, or fall back.
APPTAINER_DIRS_READY=0
usable_dir() { mkdir -p "$1" 2>/dev/null && [[ -d "$1" && -w "$1" ]]; }
prepare_apptainer_dirs() {
[[ "$APPTAINER_DIRS_READY" == 1 ]] && return 0
local name var legacy dir fallback tmp_free_gb
for name in TMPDIR CACHEDIR; do
var="APPTAINER_$name"; legacy="SINGULARITY_$name"
dir="${!var:-${!legacy:-}}"
[[ -n "$dir" ]] || continue # not set: Apptainer's defaults (/tmp, ~/.apptainer/cache) apply
usable_dir "$dir" && continue
if [[ "$name" == TMPDIR ]]; then
# unpacking the cuOpt image needs ~12 GB: local /tmp if it has the room, else our own directory
tmp_free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 ))
fallback="$QC_HOME/apptainer-tmp"
(( tmp_free_gb >= 15 )) && fallback="/tmp/$ME-apptainer-tmp"
else
fallback="$QC_HOME/apptainer-cache"
fi
usable_dir "$fallback" || die "$var=$dir does not exist, and neither it nor $fallback can be created"
export "$var=$fallback"
unset "$legacy"
if [[ -f "$CONFIG" ]] && ! grep -q "^$var=" "$CONFIG"; then
printf '\n# set by qc.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \
"$var=$fallback" >> "$CONFIG"
warn "$var=$dir (set in your environment) does not exist and cannot be created.
Using $fallback instead; saved in $CONFIG."
else
warn "$var=$dir does not exist and cannot be created; using $fallback for this run."
fi
done
APPTAINER_DIRS_READY=1
}
pull_image() { # destination source -- download an image to a .sif file, with a useful error
prepare_apptainer_dirs
rm -f "$1.part"
if ! "$APPTAINER" pull "$1.part" "$2"; then
rm -f "$1.part"
die "could not download $2. Check that this machine reaches the registry (a proxy needs
https_proxy set), and that ${APPTAINER_TMPDIR:-/tmp} has ~12 GB free (APPTAINER_TMPDIR chooses another
directory). Then run the command again."
fi
mv -f "$1.part" "$1"
}
show_log_tail() { # name
local f
for kind in err out; do
@ -195,6 +263,7 @@ cmd_install() {
say " wrote $CONFIG"
fi
prepare_apptainer_dirs
step "GPU"
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
say " $(nvidia-smi -L | head -n 1)"
@ -235,9 +304,7 @@ cmd_pull_cuopt() {
fi
step "Pulling cuOpt: $QC_CUOPT_IMAGE -> $dst (~3.3 GB, several minutes)"
mkdir -p "$QC_IMAGES"
rm -f "$dst.part"
"$APPTAINER" pull "$dst.part" "$QC_CUOPT_IMAGE"
mv -f "$dst.part" "$dst"
pull_image "$dst" "$QC_CUOPT_IMAGE"
}
qc_selftest() {
@ -271,7 +338,7 @@ get_quantcore_image() { # force(0|1)
docker://*|oras://*|library://*|http://*|https://*)
if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst"; else
say " pulling $QC_IMAGE -> $dst"
rm -f "$dst.part"; "$APPTAINER" pull "$dst.part" "$QC_IMAGE"; mv -f "$dst.part" "$dst"
pull_image "$dst" "$QC_IMAGE"
fi ;;
*)
[[ -f "$QC_IMAGE" ]] || die "QC_IMAGE=$QC_IMAGE is neither 'build', a URL, nor an existing .sif file"
@ -283,6 +350,7 @@ build_sif() { # build quantcore.sif from the definition; non-zero when this App
local dst="$QC_IMAGES/quantcore.sif"
mkdir -p "$QC_IMAGES"
say " building $dst from $REPO_DIR (a few minutes)"
prepare_apptainer_dirs
rm -f "$dst.part"
# %files paths in the definition are relative to the repository root. Without root Apptainer
# builds through --fakeroot (user namespaces); a Spark may not allow that.
@ -298,10 +366,11 @@ build_venv() { # the no-build fallback: pull a plain Python image, install quan
mkdir -p "$QC_IMAGES" "$QC_HOME"
if [[ ! -f "$PY_SIF" ]]; then
say " pulling $QC_PYTHON_IMAGE -> $PY_SIF"
rm -f "$PY_SIF.part"; "$APPTAINER" pull "$PY_SIF.part" "$QC_PYTHON_IMAGE"; mv -f "$PY_SIF.part" "$PY_SIF"
pull_image "$PY_SIF" "$QC_PYTHON_IMAGE"
fi
say " installing quantcore into $QC_VENV (pinned versions from deploy/apptainer/constraints.txt)"
local work; work="$(mktemp -d "${TMPDIR:-/tmp}/quantcore-src.XXXXXX")"
# a copy of the sources under QC_HOME (not $TMPDIR, which may point nowhere on a managed machine)
local work; work="$(mktemp -d "$QC_HOME/.quantcore-src.XXXXXX")"
# install from a copy: pip writes build/ and *.egg-info next to the sources
cp -r "$REPO_DIR/pyproject.toml" "$REPO_DIR/README.md" "$REPO_DIR/quantcore" "$work/"
"$APPTAINER" exec --cleanenv --bind "$QC_HOME" --bind "$work" "$PY_SIF" python -m venv --clear "$QC_VENV"
@ -511,6 +580,11 @@ cmd_data_export() {
say "wrote $dst ($(du -h "$dst" | cut -f1))"
}
cmd_upgrade() { # newest code from the repository, rebuild, restart -- install.sh does all of it
[[ -f "$REPO_DIR/install.sh" ]] || die "$REPO_DIR/install.sh missing; download the code again"
exec bash "$REPO_DIR/install.sh" --dir "$REPO_DIR" --no-data "$@"
}
cmd_env() {
cat <<EOF
QC_HOME=$QC_HOME
@ -537,6 +611,7 @@ Setup
data-export [FILE] pack the data directory (default quantcore_data.tar.gz)
setup [--offline] build the data set from Yahoo Finance (~15 min, needs internet)
update [--full] fetch the newest prices / FX
upgrade newest code from the repository, rebuild, restart (runs install.sh)
Run
start | stop | restart cuOpt + API instances
status instances, health, data, images
@ -570,6 +645,8 @@ main() {
setup|update) data_job "$cmd" "$@" ;;
data-import) cmd_data_import "$@" ;;
data-export) cmd_data_export "$@" ;;
has-data) ! data_is_empty ;;
upgrade) cmd_upgrade "$@" ;;
env) cmd_env ;;
clean-cache) need_apptainer; "$APPTAINER" cache clean --force ;;
help|-h|--help) usage ;;

197
install.sh Executable file
View File

@ -0,0 +1,197 @@
#!/usr/bin/env bash
# install.sh -- install or update quantcore on a DGX Spark in one step. Apptainer, no root, no git needed.
#
# On the Spark, in a terminal, ONE of these lines (whichever download tool the Spark has):
#
# wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash
# curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash
# python3 -c "import urllib.request as u;print(u.urlopen('https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh').read().decode())" | bash
#
# Options go after `bash -s --`, e.g. ... | bash -s -- --data ~/Downloads/quantcore_data.tar.gz
# --data FILE|URL import this prepared data set instead of building one from Yahoo Finance
# --no-data skip the data step (later: qc setup, or qc data-import FILE)
# --no-start install only, do not start the services
# --dir DIR where the code goes (default ~/Portfolio_Risk_Management)
#
# Steps, each skipped when it is already done -- running the script again updates everything:
# 1. checks Apptainer and the GPU, and repairs APPTAINER_TMPDIR / APPTAINER_CACHEDIR when they
# point to a directory that does not exist (seen on managed machines: /scratch/$USER/...)
# 2. downloads the code with git -- or, without git on the machine, with git in a small container
# 3. deploy/apptainer/qc.sh install: the cuOpt image and the quantcore image (or its no-build fallback)
# 4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none
# 5. starts cuOpt and the API, runs the self-test, and installs the command `qc`
#
# Everything is wrapped in main(), which runs only after bash has read the whole script: with
# `wget ... | bash` no command inside can swallow the rest of the script from standard input.
main() {
set -euo pipefail
local repo_url="${QC_REPO_URL:-https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git}"
local branch="${QC_BRANCH:-main}"
local dir="${QC_DIR:-$HOME/Portfolio_Risk_Management}"
local data="" do_data=1 do_start=1
QC_HOME="${QC_HOME:-$HOME/quantcore}"
APPTAINER="${APPTAINER:-apptainer}"
GIT_IMAGE="${QC_GIT_IMAGE:-docker://alpine/git:2.54.0}"
FIXED_VARS=()
while [[ $# -gt 0 ]]; do
case "$1" in
--data) data="${2:?--data needs a file or URL}"; shift ;;
--data=*) data="${1#--data=}" ;;
--no-data) do_data=0 ;;
--no-start) do_start=0 ;;
--dir) dir="${2:?--dir needs a directory}"; shift ;;
--dir=*) dir="${1#--dir=}" ;;
-h|--help) sed -n '2,26p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;;
*) die "unknown option: $1 (see the top of install.sh)" ;;
esac
shift
done
dir="${dir%/}"
local qc="$dir/deploy/apptainer/qc.sh"
step "1/5 Checking this machine"
command -v "$APPTAINER" >/dev/null 2>&1 \
|| die "Apptainer is not installed on this machine ('$APPTAINER' not found). Ask the administrator."
say " $("$APPTAINER" --version), user $(id -un), home $HOME"
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
say " $(nvidia-smi -L | head -n 1)"
else
say " no GPU visible: cuOpt will not run, optimisations will use HiGHS on the CPU"
fi
mkdir -p "$QC_HOME"
fix_apptainer_dirs
step "2/5 Code: $repo_url ($branch) -> $dir"
local before="" after=""
if [[ -d "$dir/.git" ]]; then
before="$(run_git -C "$dir" rev-parse HEAD 2>/dev/null || true)"
run_git -C "$dir" pull --ff-only origin "$branch" \
|| die "could not update $dir (did you change files there? 'git -C $dir status' shows which)"
elif [[ -e "$dir" && -n "$(ls -A "$dir" 2>/dev/null)" ]]; then
die "$dir exists but is not a git checkout of the project. Move it away, or use --dir DIR."
else
mkdir -p "$(dirname "$dir")"
run_git clone --branch "$branch" "$repo_url" "$dir"
fi
after="$(run_git -C "$dir" rev-parse HEAD)"
say " at $(run_git -C "$dir" log -1 --format='%h %s (%cd)' --date=short)"
chmod +x "$qc" "$dir/install.sh" 2>/dev/null || true
step "3/5 Container images"
"$qc" install
persist_fixes
if [[ -n "$before" && "$before" != "$after" ]]; then
say " the code changed since the last installation: rebuilding quantcore"
"$qc" build
fi
step "4/5 Market data"
if [[ -n "$data" ]]; then
local file="$data"
if [[ "$data" == http://* || "$data" == https://* ]]; then
file="$QC_HOME/quantcore_data.tar.gz"
say " downloading $data"
fetch "$data" "$file"
fi
if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi
elif "$qc" has-data; then
say " present -- newest prices later with: qc update"
elif [[ "$do_data" == 1 ]]; then
say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)"
"$qc" setup
else
say " skipped (--no-data). Later: qc setup or qc data-import FILE"
fi
step "5/5 The command 'qc'"
mkdir -p "$HOME/.local/bin"
ln -sf "$qc" "$HOME/.local/bin/qc"
say " $HOME/.local/bin/qc -> $qc"
case ":$PATH:" in
*":$HOME/.local/bin:"*) ;;
*) say " open a new terminal to use it (or run: export PATH=\"\$HOME/.local/bin:\$PATH\")" ;;
esac
if [[ "$do_start" == 1 ]]; then
if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi
if "$qc" has-data; then
step "Self-test"
"$qc" api selftest || say " some checks failed -- see the table above and: qc logs"
fi
fi
step "Done"
say " Test bench: http://127.0.0.1:$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')/ui/ (on this Spark)"
say " Commands: qc status | qc stop | qc start | qc logs | qc update | qc upgrade | qc help"
say " Guide: $dir/docs/SPARK_APPTAINER.md"
}
say() { printf '%s\n' "$*"; }
step() { printf '\n== %s\n' "$*"; }
die() { printf '\nerror: %s\n' "$*" >&2; exit 1; }
# APPTAINER_TMPDIR / APPTAINER_CACHEDIR (or the older SINGULARITY_* names) that point to a missing
# directory make every download fail ("failed to create build parent dir"). Create the directory,
# or use one of ours; persist_fixes() then records the choice in the settings file.
fix_apptainer_dirs() {
local name var legacy dirpath fallback free_gb
for name in TMPDIR CACHEDIR; do
var="APPTAINER_$name"; legacy="SINGULARITY_$name"
dirpath="${!var:-${!legacy:-}}"
[[ -n "$dirpath" ]] || continue
if mkdir -p "$dirpath" 2>/dev/null && [[ -w "$dirpath" ]]; then continue; fi
if [[ "$name" == TMPDIR ]]; then
free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 ))
fallback="$QC_HOME/apptainer-tmp"
(( free_gb >= 15 )) && fallback="/tmp/$(id -un)-apptainer-tmp"
else
fallback="$QC_HOME/apptainer-cache"
fi
mkdir -p "$fallback" || die "$var=$dirpath does not exist and $fallback cannot be created"
say " $var=$dirpath does not exist on this machine -> using $fallback"
export "$var=$fallback"
unset "$legacy"
FIXED_VARS+=("$var=$fallback")
done
}
persist_fixes() { # record the repaired directories in the settings file, so `qc` uses them too
local config="$QC_HOME/quantcore.env" kv
[[ -f "$config" ]] || return 0
for kv in "${FIXED_VARS[@]+"${FIXED_VARS[@]}"}"; do
grep -q "^${kv%%=*}=" "$config" && continue
printf '\n# set by install.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \
"$kv" >> "$config"
done
}
# git on the machine, else git in a small container (alpine/git) -- Apptainer is all a Spark needs
run_git() {
if command -v git >/dev/null 2>&1; then git "$@" </dev/null; return; fi
local sif="$QC_HOME/images/git.sif" v
if [[ ! -f "$sif" ]]; then
say " git is not installed here: using git from a container ($GIT_IMAGE, ~40 MB, once)" >&2
mkdir -p "$QC_HOME/images"
rm -f "$sif.part"
"$APPTAINER" pull "$sif.part" "$GIT_IMAGE" </dev/null >&2 \
|| die "could not download $GIT_IMAGE (network or proxy? a proxy needs https_proxy set)"
mv -f "$sif.part" "$sif"
fi
for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY; do
[[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}"
done
# the target directory's parent is bound in case it lies outside the home directory
"$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" </dev/null
}
fetch() { # url file -- with whatever this machine has
if command -v curl >/dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1"
elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1"
elif command -v python3 >/dev/null 2>&1; then
python3 -c 'import sys, urllib.request as u; u.urlretrieve(sys.argv[1], sys.argv[2])' "$1" "$2"
else die "no download tool (curl, wget or python3) -- copy the file to this machine and pass its path"; fi
}
main "$@"