diff --git a/deploy/apptainer/qc.sh b/deploy/apptainer/qc.sh index 195ad48..80582d0 100755 --- a/deploy/apptainer/qc.sh +++ b/deploy/apptainer/qc.sh @@ -19,7 +19,8 @@ # Guide: docs/SPARK_APPTAINER.md. set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# resolve the link ~/.local/bin/qc (made by install.sh) to the script in the repository +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" QC_HOME="${QC_HOME:-$HOME/quantcore}" CONFIG="$QC_HOME/quantcore.env" @@ -82,7 +83,8 @@ need_image() { # path what [[ -f "$1" ]] || die "$2 image missing ($1) -- run: $0 install" } -# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3 (both on DGX OS). +# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3, else plain bash: +# a barebone Spark may have neither of the first two. http_get() { local url="$1" timeout="${2:-3}" if command -v curl >/dev/null 2>&1; then @@ -91,10 +93,25 @@ http_get() { python3 -c 'import sys, urllib.request as u; sys.stdout.write(u.urlopen(sys.argv[1], timeout=float(sys.argv[2])).read().decode())' \ "$url" "$timeout" 2>/dev/null else - return 2 + http_get_bash "$url" "$timeout" fi } +# HTTP/1.0 GET over bash's /dev/tcp, for the http://host:port/path addresses qc.sh checks. +http_get_bash() { + local rest="${1#http://}" t="${2:-3}" hostport path host port + hostport="${rest%%/*}"; path="${rest#"$hostport"}"; [[ -n "$path" ]] || path="/" + host="${hostport%%:*}"; port=80; [[ "$hostport" == *:* ]] && port="${hostport##*:}" + ( + exec 3<>"/dev/tcp/$host/$port" || exit 1 + printf 'GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' "$path" "$host" >&3 + IFS= read -r -t "$t" status <&3 || exit 1 + [[ "$status" =~ ^HTTP/[0-9.]+\ 2[0-9][0-9] ]] || exit 1 + while IFS= read -r -t "$t" line <&3; do line="${line%$'\r'}"; [[ -z "$line" ]] && break; done + while IFS= read -r -t "$t" line <&3 || [[ -n "$line" ]]; do printf '%s\n' "$line"; line=""; done + ) 2>/dev/null +} + port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } free_port() { # first free port from $1 upwards @@ -108,13 +125,64 @@ instance_running() { } instance_log() { # name out|err -> path of the instance log (Apptainer keeps them per host and user) + # this host first: a home directory on a network drive holds the logs of every machine; the user + # directory may be spelt "name" or "name@domain", so match any local f - for f in "$HOME/.apptainer/instances/logs/"*"/$ME/$1.$2"; do + for f in "$HOME/.apptainer/instances/logs/$(hostname)/"*"/$1.$2" "$HOME/.apptainer/instances/logs/"*/*"/$1.$2"; do [[ -f "$f" ]] && { say "$f"; return 0; } done return 1 } +# Apptainer's working and cache directories. Some installations export APPTAINER_TMPDIR or +# APPTAINER_CACHEDIR for a convention such as /scratch/$USER that this machine does not have; every +# pull then dies with "failed to create build parent dir". Create the directory, or fall back. +APPTAINER_DIRS_READY=0 +usable_dir() { mkdir -p "$1" 2>/dev/null && [[ -d "$1" && -w "$1" ]]; } + +prepare_apptainer_dirs() { + [[ "$APPTAINER_DIRS_READY" == 1 ]] && return 0 + local name var legacy dir fallback tmp_free_gb + for name in TMPDIR CACHEDIR; do + var="APPTAINER_$name"; legacy="SINGULARITY_$name" + dir="${!var:-${!legacy:-}}" + [[ -n "$dir" ]] || continue # not set: Apptainer's defaults (/tmp, ~/.apptainer/cache) apply + usable_dir "$dir" && continue + if [[ "$name" == TMPDIR ]]; then + # unpacking the cuOpt image needs ~12 GB: local /tmp if it has the room, else our own directory + tmp_free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 )) + fallback="$QC_HOME/apptainer-tmp" + (( tmp_free_gb >= 15 )) && fallback="/tmp/$ME-apptainer-tmp" + else + fallback="$QC_HOME/apptainer-cache" + fi + usable_dir "$fallback" || die "$var=$dir does not exist, and neither it nor $fallback can be created" + export "$var=$fallback" + unset "$legacy" + if [[ -f "$CONFIG" ]] && ! grep -q "^$var=" "$CONFIG"; then + printf '\n# set by qc.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \ + "$var=$fallback" >> "$CONFIG" + warn "$var=$dir (set in your environment) does not exist and cannot be created. + Using $fallback instead; saved in $CONFIG." + else + warn "$var=$dir does not exist and cannot be created; using $fallback for this run." + fi + done + APPTAINER_DIRS_READY=1 +} + +pull_image() { # destination source -- download an image to a .sif file, with a useful error + prepare_apptainer_dirs + rm -f "$1.part" + if ! "$APPTAINER" pull "$1.part" "$2"; then + rm -f "$1.part" + die "could not download $2. Check that this machine reaches the registry (a proxy needs + https_proxy set), and that ${APPTAINER_TMPDIR:-/tmp} has ~12 GB free (APPTAINER_TMPDIR chooses another + directory). Then run the command again." + fi + mv -f "$1.part" "$1" +} + show_log_tail() { # name local f for kind in err out; do @@ -195,6 +263,7 @@ cmd_install() { say " wrote $CONFIG" fi + prepare_apptainer_dirs step "GPU" if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then say " $(nvidia-smi -L | head -n 1)" @@ -235,9 +304,7 @@ cmd_pull_cuopt() { fi step "Pulling cuOpt: $QC_CUOPT_IMAGE -> $dst (~3.3 GB, several minutes)" mkdir -p "$QC_IMAGES" - rm -f "$dst.part" - "$APPTAINER" pull "$dst.part" "$QC_CUOPT_IMAGE" - mv -f "$dst.part" "$dst" + pull_image "$dst" "$QC_CUOPT_IMAGE" } qc_selftest() { @@ -271,7 +338,7 @@ get_quantcore_image() { # force(0|1) docker://*|oras://*|library://*|http://*|https://*) if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst"; else say " pulling $QC_IMAGE -> $dst" - rm -f "$dst.part"; "$APPTAINER" pull "$dst.part" "$QC_IMAGE"; mv -f "$dst.part" "$dst" + pull_image "$dst" "$QC_IMAGE" fi ;; *) [[ -f "$QC_IMAGE" ]] || die "QC_IMAGE=$QC_IMAGE is neither 'build', a URL, nor an existing .sif file" @@ -283,6 +350,7 @@ build_sif() { # build quantcore.sif from the definition; non-zero when this App local dst="$QC_IMAGES/quantcore.sif" mkdir -p "$QC_IMAGES" say " building $dst from $REPO_DIR (a few minutes)" + prepare_apptainer_dirs rm -f "$dst.part" # %files paths in the definition are relative to the repository root. Without root Apptainer # builds through --fakeroot (user namespaces); a Spark may not allow that. @@ -298,10 +366,11 @@ build_venv() { # the no-build fallback: pull a plain Python image, install quan mkdir -p "$QC_IMAGES" "$QC_HOME" if [[ ! -f "$PY_SIF" ]]; then say " pulling $QC_PYTHON_IMAGE -> $PY_SIF" - rm -f "$PY_SIF.part"; "$APPTAINER" pull "$PY_SIF.part" "$QC_PYTHON_IMAGE"; mv -f "$PY_SIF.part" "$PY_SIF" + pull_image "$PY_SIF" "$QC_PYTHON_IMAGE" fi say " installing quantcore into $QC_VENV (pinned versions from deploy/apptainer/constraints.txt)" - local work; work="$(mktemp -d "${TMPDIR:-/tmp}/quantcore-src.XXXXXX")" + # a copy of the sources under QC_HOME (not $TMPDIR, which may point nowhere on a managed machine) + local work; work="$(mktemp -d "$QC_HOME/.quantcore-src.XXXXXX")" # install from a copy: pip writes build/ and *.egg-info next to the sources cp -r "$REPO_DIR/pyproject.toml" "$REPO_DIR/README.md" "$REPO_DIR/quantcore" "$work/" "$APPTAINER" exec --cleanenv --bind "$QC_HOME" --bind "$work" "$PY_SIF" python -m venv --clear "$QC_VENV" @@ -511,6 +580,11 @@ cmd_data_export() { say "wrote $dst ($(du -h "$dst" | cut -f1))" } +cmd_upgrade() { # newest code from the repository, rebuild, restart -- install.sh does all of it + [[ -f "$REPO_DIR/install.sh" ]] || die "$REPO_DIR/install.sh missing; download the code again" + exec bash "$REPO_DIR/install.sh" --dir "$REPO_DIR" --no-data "$@" +} + cmd_env() { cat </dev/null || true; return 0 ;; + *) die "unknown option: $1 (see the top of install.sh)" ;; + esac + shift + done + dir="${dir%/}" + local qc="$dir/deploy/apptainer/qc.sh" + + step "1/5 Checking this machine" + command -v "$APPTAINER" >/dev/null 2>&1 \ + || die "Apptainer is not installed on this machine ('$APPTAINER' not found). Ask the administrator." + say " $("$APPTAINER" --version), user $(id -un), home $HOME" + if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then + say " $(nvidia-smi -L | head -n 1)" + else + say " no GPU visible: cuOpt will not run, optimisations will use HiGHS on the CPU" + fi + mkdir -p "$QC_HOME" + fix_apptainer_dirs + + step "2/5 Code: $repo_url ($branch) -> $dir" + local before="" after="" + if [[ -d "$dir/.git" ]]; then + before="$(run_git -C "$dir" rev-parse HEAD 2>/dev/null || true)" + run_git -C "$dir" pull --ff-only origin "$branch" \ + || die "could not update $dir (did you change files there? 'git -C $dir status' shows which)" + elif [[ -e "$dir" && -n "$(ls -A "$dir" 2>/dev/null)" ]]; then + die "$dir exists but is not a git checkout of the project. Move it away, or use --dir DIR." + else + mkdir -p "$(dirname "$dir")" + run_git clone --branch "$branch" "$repo_url" "$dir" + fi + after="$(run_git -C "$dir" rev-parse HEAD)" + say " at $(run_git -C "$dir" log -1 --format='%h %s (%cd)' --date=short)" + chmod +x "$qc" "$dir/install.sh" 2>/dev/null || true + + step "3/5 Container images" + "$qc" install + persist_fixes + if [[ -n "$before" && "$before" != "$after" ]]; then + say " the code changed since the last installation: rebuilding quantcore" + "$qc" build + fi + + step "4/5 Market data" + if [[ -n "$data" ]]; then + local file="$data" + if [[ "$data" == http://* || "$data" == https://* ]]; then + file="$QC_HOME/quantcore_data.tar.gz" + say " downloading $data" + fetch "$data" "$file" + fi + if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi + elif "$qc" has-data; then + say " present -- newest prices later with: qc update" + elif [[ "$do_data" == 1 ]]; then + say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)" + "$qc" setup + else + say " skipped (--no-data). Later: qc setup or qc data-import FILE" + fi + + step "5/5 The command 'qc'" + mkdir -p "$HOME/.local/bin" + ln -sf "$qc" "$HOME/.local/bin/qc" + say " $HOME/.local/bin/qc -> $qc" + case ":$PATH:" in + *":$HOME/.local/bin:"*) ;; + *) say " open a new terminal to use it (or run: export PATH=\"\$HOME/.local/bin:\$PATH\")" ;; + esac + + if [[ "$do_start" == 1 ]]; then + if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi + if "$qc" has-data; then + step "Self-test" + "$qc" api selftest || say " some checks failed -- see the table above and: qc logs" + fi + fi + + step "Done" + say " Test bench: http://127.0.0.1:$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')/ui/ (on this Spark)" + say " Commands: qc status | qc stop | qc start | qc logs | qc update | qc upgrade | qc help" + say " Guide: $dir/docs/SPARK_APPTAINER.md" +} + +say() { printf '%s\n' "$*"; } +step() { printf '\n== %s\n' "$*"; } +die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } + +# APPTAINER_TMPDIR / APPTAINER_CACHEDIR (or the older SINGULARITY_* names) that point to a missing +# directory make every download fail ("failed to create build parent dir"). Create the directory, +# or use one of ours; persist_fixes() then records the choice in the settings file. +fix_apptainer_dirs() { + local name var legacy dirpath fallback free_gb + for name in TMPDIR CACHEDIR; do + var="APPTAINER_$name"; legacy="SINGULARITY_$name" + dirpath="${!var:-${!legacy:-}}" + [[ -n "$dirpath" ]] || continue + if mkdir -p "$dirpath" 2>/dev/null && [[ -w "$dirpath" ]]; then continue; fi + if [[ "$name" == TMPDIR ]]; then + free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 )) + fallback="$QC_HOME/apptainer-tmp" + (( free_gb >= 15 )) && fallback="/tmp/$(id -un)-apptainer-tmp" + else + fallback="$QC_HOME/apptainer-cache" + fi + mkdir -p "$fallback" || die "$var=$dirpath does not exist and $fallback cannot be created" + say " $var=$dirpath does not exist on this machine -> using $fallback" + export "$var=$fallback" + unset "$legacy" + FIXED_VARS+=("$var=$fallback") + done +} + +persist_fixes() { # record the repaired directories in the settings file, so `qc` uses them too + local config="$QC_HOME/quantcore.env" kv + [[ -f "$config" ]] || return 0 + for kv in "${FIXED_VARS[@]+"${FIXED_VARS[@]}"}"; do + grep -q "^${kv%%=*}=" "$config" && continue + printf '\n# set by install.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \ + "$kv" >> "$config" + done +} + +# git on the machine, else git in a small container (alpine/git) -- Apptainer is all a Spark needs +run_git() { + if command -v git >/dev/null 2>&1; then git "$@" &2 + mkdir -p "$QC_HOME/images" + rm -f "$sif.part" + "$APPTAINER" pull "$sif.part" "$GIT_IMAGE" &2 \ + || die "could not download $GIT_IMAGE (network or proxy? a proxy needs https_proxy set)" + mv -f "$sif.part" "$sif" + fi + for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY; do + [[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}" + done + # the target directory's parent is bound in case it lies outside the home directory + "$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" /dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1" + elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1" + elif command -v python3 >/dev/null 2>&1; then + python3 -c 'import sys, urllib.request as u; u.urlretrieve(sys.argv[1], sys.argv[2])' "$1" "$2" + else die "no download tool (curl, wget or python3) -- copy the file to this machine and pass its path"; fi +} + +main "$@"