772aa2de84
On a student Spark the installer's data step printed hundreds of red yfinance "Quote not found for symbol" lines and built a universe of 652 instead of ~745 stocks. Cause: the iShares holdings download (German site) answers 404 since October 2026, and setup fell back to Wikipedia's STOXX 600 table, whose Reuters-style codes give Yahoo symbols that do not exist (BNPP.PA for BNP.PA, NOV-N.SW for NOVN.SW, VESTAS.CO for VWS.CO). - sources: the iShares file is read from the Swiss site (same German CSV, same parser), the German address stays as a spare; when iShares fails, setup uses the snapshot shipped in quantcore/data/static and not the Wikipedia guesses (now a last resort when no snapshot ships); only a primary source refreshes the snapshot - yahoo: yfinance's per-symbol ERROR lines are silenced during the metadata step and summarised in one line - install.sh: builds the data with `qc setup --offline`, so every student gets the universe of the shipped lists (the course data set's); new --rebuild-data moves the existing data aside and builds it anew - qc.sh: the image build attempt logs to ~/quantcore/build.log; a Spark that allows no build shows one plain line, then the no-build setup - tests for the source order, the address fallback and the summary Tested on a stripped-down machine (no git/curl/wget/python3, broken APPTAINER_TMPDIR): install, then install.sh --rebuild-data built 746 stocks (382 US, 364 Europe) from the 1117 shipped candidates in under 10 minutes with no yfinance error line; self-test passes; qc upgrade rebuilt and restarted. 142 tests pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
212 lines
9.7 KiB
Bash
Executable File
212 lines
9.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# install.sh -- install or update quantcore on a DGX Spark in one step. Apptainer, no root, no git needed.
|
|
#
|
|
# On the Spark, in a terminal, ONE of these lines (whichever download tool the Spark has):
|
|
#
|
|
# wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash
|
|
# curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash
|
|
# python3 -c "import urllib.request as u;print(u.urlopen('https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh').read().decode())" | bash
|
|
#
|
|
# Options go after `bash -s --`, e.g. ... | bash -s -- --data ~/Downloads/quantcore_data.tar.gz
|
|
# --data FILE|URL import this prepared data set instead of building one from Yahoo Finance
|
|
# --no-data skip the data step (later: qc setup --offline, or qc data-import FILE)
|
|
# --rebuild-data move the existing data set aside (to ~/quantcore/data.<date>) and build a new one
|
|
# --no-start install only, do not start the services
|
|
# --dir DIR where the code goes (default ~/Portfolio_Risk_Management)
|
|
#
|
|
# Steps, each skipped when it is already done -- running the script again updates everything:
|
|
# 1. checks Apptainer and the GPU, and repairs APPTAINER_TMPDIR / APPTAINER_CACHEDIR when they
|
|
# point to a directory that does not exist (seen on managed machines: /scratch/$USER/...)
|
|
# 2. downloads the code with git -- or, without git on the machine, with git in a small container
|
|
# 3. deploy/apptainer/qc.sh install: the cuOpt image and the quantcore image (or its no-build fallback)
|
|
# 4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none.
|
|
# The candidate stocks come from the list shipped in the repository (the iShares STOXX Europe 600
|
|
# and S&P 500 constituents the course data was built from), so every installation gets the same
|
|
# universe; prices and company data come from Yahoo.
|
|
# 5. starts cuOpt and the API, runs the self-test, and installs the command `qc`
|
|
#
|
|
# Everything is wrapped in main(), which runs only after bash has read the whole script: with
|
|
# `wget ... | bash` no command inside can swallow the rest of the script from standard input.
|
|
|
|
main() {
|
|
set -euo pipefail
|
|
local repo_url="${QC_REPO_URL:-https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git}"
|
|
local branch="${QC_BRANCH:-main}"
|
|
local dir="${QC_DIR:-$HOME/Portfolio_Risk_Management}"
|
|
local data="" do_data=1 do_start=1 rebuild_data=0
|
|
QC_HOME="${QC_HOME:-$HOME/quantcore}"
|
|
APPTAINER="${APPTAINER:-apptainer}"
|
|
GIT_IMAGE="${QC_GIT_IMAGE:-docker://alpine/git:2.54.0}"
|
|
FIXED_VARS=()
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--data) data="${2:?--data needs a file or URL}"; shift ;;
|
|
--data=*) data="${1#--data=}" ;;
|
|
--no-data) do_data=0 ;;
|
|
--no-start) do_start=0 ;;
|
|
--dir) dir="${2:?--dir needs a directory}"; shift ;;
|
|
--dir=*) dir="${1#--dir=}" ;;
|
|
--rebuild-data) rebuild_data=1 ;;
|
|
-h|--help) sed -n '2,30p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;;
|
|
*) die "unknown option: $1 (see the top of install.sh)" ;;
|
|
esac
|
|
shift
|
|
done
|
|
dir="${dir%/}"
|
|
local qc="$dir/deploy/apptainer/qc.sh"
|
|
|
|
step "1/5 Checking this machine"
|
|
command -v "$APPTAINER" >/dev/null 2>&1 \
|
|
|| die "Apptainer is not installed on this machine ('$APPTAINER' not found). Ask the administrator."
|
|
say " $("$APPTAINER" --version), user $(id -un), home $HOME"
|
|
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
|
|
say " $(nvidia-smi -L | head -n 1)"
|
|
else
|
|
say " no GPU visible: cuOpt will not run, optimisations will use HiGHS on the CPU"
|
|
fi
|
|
mkdir -p "$QC_HOME"
|
|
fix_apptainer_dirs
|
|
|
|
step "2/5 Code: $repo_url ($branch) -> $dir"
|
|
local before="" after=""
|
|
if [[ -d "$dir/.git" ]]; then
|
|
before="$(run_git -C "$dir" rev-parse HEAD 2>/dev/null || true)"
|
|
run_git -C "$dir" pull --ff-only origin "$branch" \
|
|
|| die "could not update $dir (did you change files there? 'git -C $dir status' shows which)"
|
|
elif [[ -e "$dir" && -n "$(ls -A "$dir" 2>/dev/null)" ]]; then
|
|
die "$dir exists but is not a git checkout of the project. Move it away, or use --dir DIR."
|
|
else
|
|
mkdir -p "$(dirname "$dir")"
|
|
run_git clone --branch "$branch" "$repo_url" "$dir"
|
|
fi
|
|
after="$(run_git -C "$dir" rev-parse HEAD)"
|
|
say " at $(run_git -C "$dir" log -1 --format='%h %s (%cd)' --date=short)"
|
|
chmod +x "$qc" "$dir/install.sh" 2>/dev/null || true
|
|
|
|
step "3/5 Container images"
|
|
QC_FROM_INSTALLER=1 "$qc" install
|
|
persist_fixes
|
|
if [[ -n "$before" && "$before" != "$after" ]]; then
|
|
say " the code changed since the last installation: rebuilding quantcore"
|
|
"$qc" build
|
|
fi
|
|
|
|
step "4/5 Market data"
|
|
if [[ "$rebuild_data" == 1 ]] && "$qc" has-data; then
|
|
local data_dir aside
|
|
data_dir="$("$qc" env | sed -n 's|^QC_DATA=||p')"
|
|
aside="$data_dir.$(date +%Y%m%d-%H%M%S)"
|
|
"$qc" stop >/dev/null 2>&1 || true
|
|
mv "$data_dir" "$aside"
|
|
say " moved the existing data set to $aside"
|
|
fi
|
|
if [[ -n "$data" ]]; then
|
|
local file="$data"
|
|
if [[ "$data" == http://* || "$data" == https://* ]]; then
|
|
file="$QC_HOME/quantcore_data.tar.gz"
|
|
say " downloading $data"
|
|
fetch "$data" "$file"
|
|
fi
|
|
if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi
|
|
elif "$qc" has-data; then
|
|
say " present -- newest prices later with: qc update"
|
|
elif [[ "$do_data" == 1 ]]; then
|
|
say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)"
|
|
say " candidates: the list shipped in the repository; companies Yahoo no longer knows are left out"
|
|
"$qc" setup --offline
|
|
else
|
|
say " skipped (--no-data). Later: qc setup or qc data-import FILE"
|
|
fi
|
|
|
|
step "5/5 The command 'qc'"
|
|
mkdir -p "$HOME/.local/bin"
|
|
ln -sf "$qc" "$HOME/.local/bin/qc"
|
|
say " $HOME/.local/bin/qc -> $qc"
|
|
case ":$PATH:" in
|
|
*":$HOME/.local/bin:"*) ;;
|
|
*) say " open a new terminal to use it (or run: export PATH=\"\$HOME/.local/bin:\$PATH\")" ;;
|
|
esac
|
|
|
|
if [[ "$do_start" == 1 ]]; then
|
|
if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi
|
|
if "$qc" has-data; then
|
|
step "Self-test"
|
|
"$qc" api selftest || say " some checks failed -- see the table above and: qc logs"
|
|
fi
|
|
fi
|
|
|
|
step "Done"
|
|
say " Test bench: http://127.0.0.1:$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')/ui/ (on this Spark)"
|
|
say " Commands: qc status | qc stop | qc start | qc logs | qc update | qc upgrade | qc help"
|
|
say " Guide: $dir/docs/SPARK_APPTAINER.md"
|
|
}
|
|
|
|
say() { printf '%s\n' "$*"; }
|
|
step() { printf '\n== %s\n' "$*"; }
|
|
die() { printf '\nerror: %s\n' "$*" >&2; exit 1; }
|
|
|
|
# APPTAINER_TMPDIR / APPTAINER_CACHEDIR (or the older SINGULARITY_* names) that point to a missing
|
|
# directory make every download fail ("failed to create build parent dir"). Create the directory,
|
|
# or use one of ours; persist_fixes() then records the choice in the settings file.
|
|
fix_apptainer_dirs() {
|
|
local name var legacy dirpath fallback free_gb
|
|
for name in TMPDIR CACHEDIR; do
|
|
var="APPTAINER_$name"; legacy="SINGULARITY_$name"
|
|
dirpath="${!var:-${!legacy:-}}"
|
|
[[ -n "$dirpath" ]] || continue
|
|
if mkdir -p "$dirpath" 2>/dev/null && [[ -w "$dirpath" ]]; then continue; fi
|
|
if [[ "$name" == TMPDIR ]]; then
|
|
free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 ))
|
|
fallback="$QC_HOME/apptainer-tmp"
|
|
(( free_gb >= 15 )) && fallback="/tmp/$(id -un)-apptainer-tmp"
|
|
else
|
|
fallback="$QC_HOME/apptainer-cache"
|
|
fi
|
|
mkdir -p "$fallback" || die "$var=$dirpath does not exist and $fallback cannot be created"
|
|
say " $var=$dirpath does not exist on this machine -> using $fallback"
|
|
export "$var=$fallback"
|
|
unset "$legacy"
|
|
FIXED_VARS+=("$var=$fallback")
|
|
done
|
|
}
|
|
|
|
persist_fixes() { # record the repaired directories in the settings file, so `qc` uses them too
|
|
local config="$QC_HOME/quantcore.env" kv
|
|
[[ -f "$config" ]] || return 0
|
|
for kv in "${FIXED_VARS[@]+"${FIXED_VARS[@]}"}"; do
|
|
grep -q "^${kv%%=*}=" "$config" && continue
|
|
printf '\n# set by install.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \
|
|
"$kv" >> "$config"
|
|
done
|
|
}
|
|
|
|
# git on the machine, else git in a small container (alpine/git) -- Apptainer is all a Spark needs
|
|
run_git() {
|
|
if command -v git >/dev/null 2>&1; then git "$@" </dev/null; return; fi
|
|
local sif="$QC_HOME/images/git.sif" v
|
|
if [[ ! -f "$sif" ]]; then
|
|
say " git is not installed here: using git from a container ($GIT_IMAGE, ~40 MB, once)" >&2
|
|
mkdir -p "$QC_HOME/images"
|
|
rm -f "$sif.part"
|
|
"$APPTAINER" pull "$sif.part" "$GIT_IMAGE" </dev/null >&2 \
|
|
|| die "could not download $GIT_IMAGE (network or proxy? a proxy needs https_proxy set)"
|
|
mv -f "$sif.part" "$sif"
|
|
fi
|
|
for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY; do
|
|
[[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}"
|
|
done
|
|
# the target directory's parent is bound in case it lies outside the home directory
|
|
"$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" </dev/null
|
|
}
|
|
|
|
fetch() { # url file -- with whatever this machine has
|
|
if command -v curl >/dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1"
|
|
elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
python3 -c 'import sys, urllib.request as u; u.urlretrieve(sys.argv[1], sys.argv[2])' "$1" "$2"
|
|
else die "no download tool (curl, wget or python3) -- copy the file to this machine and pass its path"; fi
|
|
}
|
|
|
|
main "$@"
|