#!/usr/bin/env bash # install.sh -- install or update quantcore on a DGX Spark in one step. Apptainer, no root, no git needed. # # On the Spark, in a terminal, ONE of these lines (whichever download tool the Spark has): # # wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash # curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash # python3 -c "import urllib.request as u;print(u.urlopen('https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh').read().decode())" | bash # # Options go after `bash -s --`, e.g. ... | bash -s -- --data ~/Downloads/quantcore_data.tar.gz # --data FILE|URL import this prepared data set instead of building one from Yahoo Finance # --no-data skip the data step (later: qc setup --offline, or qc data-import FILE) # --rebuild-data move the existing data set aside (to ~/quantcore/data.) and build a new one # --no-start install only, do not start the services # --dir DIR where the code goes (default ~/Portfolio_Risk_Management) # # Steps, each skipped when it is already done -- running the script again updates everything: # 1. checks Apptainer and the GPU, and repairs APPTAINER_TMPDIR / APPTAINER_CACHEDIR when they # point to a directory that does not exist (seen on managed machines: /scratch/$USER/...) # 2. downloads the code with git -- or, without git on the machine, with git in a small container # 3. deploy/apptainer/qc.sh install: the cuOpt image and the quantcore image (or its no-build fallback) # 4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none. # The candidate stocks come from the list shipped in the repository (the iShares STOXX Europe 600 # and S&P 500 constituents the course data was built from), so every installation gets the same # universe; prices and company data come from Yahoo. # 5. starts cuOpt and the API, runs the self-test, and installs the command `qc` # # Everything is wrapped in main(), which runs only after bash has read the whole script: with # `wget ... | bash` no command inside can swallow the rest of the script from standard input. main() { set -euo pipefail local repo_url="${QC_REPO_URL:-https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git}" local branch="${QC_BRANCH:-main}" local dir="${QC_DIR:-$HOME/Portfolio_Risk_Management}" local data="" do_data=1 do_start=1 rebuild_data=0 QC_HOME="${QC_HOME:-$HOME/quantcore}" APPTAINER="${APPTAINER:-apptainer}" GIT_IMAGE="${QC_GIT_IMAGE:-docker://alpine/git:2.54.0}" FIXED_VARS=() while [[ $# -gt 0 ]]; do case "$1" in --data) data="${2:?--data needs a file or URL}"; shift ;; --data=*) data="${1#--data=}" ;; --no-data) do_data=0 ;; --no-start) do_start=0 ;; --dir) dir="${2:?--dir needs a directory}"; shift ;; --dir=*) dir="${1#--dir=}" ;; --rebuild-data) rebuild_data=1 ;; -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;; *) die "unknown option: $1 (see the top of install.sh)" ;; esac shift done dir="${dir%/}" local qc="$dir/deploy/apptainer/qc.sh" step "1/5 Checking this machine" command -v "$APPTAINER" >/dev/null 2>&1 \ || die "Apptainer is not installed on this machine ('$APPTAINER' not found). Ask the administrator." say " $("$APPTAINER" --version), user $(id -un), home $HOME" if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then say " $(nvidia-smi -L | head -n 1)" else say " no GPU visible: cuOpt will not run, optimisations will use HiGHS on the CPU" fi mkdir -p "$QC_HOME" fix_apptainer_dirs step "2/5 Code: $repo_url ($branch) -> $dir" local before="" after="" if [[ -d "$dir/.git" ]]; then before="$(run_git -C "$dir" rev-parse HEAD 2>/dev/null || true)" run_git -C "$dir" pull --ff-only origin "$branch" \ || die "could not update $dir (did you change files there? 'git -C $dir status' shows which)" elif [[ -e "$dir" && -n "$(ls -A "$dir" 2>/dev/null)" ]]; then die "$dir exists but is not a git checkout of the project. Move it away, or use --dir DIR." else mkdir -p "$(dirname "$dir")" run_git clone --branch "$branch" "$repo_url" "$dir" fi after="$(run_git -C "$dir" rev-parse HEAD)" say " at $(run_git -C "$dir" log -1 --format='%h %s (%cd)' --date=short)" chmod +x "$qc" "$dir/install.sh" 2>/dev/null || true step "3/5 Container images" QC_FROM_INSTALLER=1 "$qc" install persist_fixes if [[ -n "$before" && "$before" != "$after" ]]; then say " the code changed since the last installation: rebuilding quantcore" "$qc" build fi step "4/5 Market data" if [[ "$rebuild_data" == 1 ]] && "$qc" has-data; then local data_dir aside data_dir="$("$qc" env | sed -n 's|^QC_DATA=||p')" aside="$data_dir.$(date +%Y%m%d-%H%M%S)" "$qc" stop ! "$qc" is-running || die "the service is still running; stop it (qc stop) and run this again" # move the contents, never the folder itself: a container that still had the folder mounted would # keep reading and writing the moved copy, while every new command looks at the empty new folder mkdir -p "$aside" find "$data_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$aside/" \; say " moved the existing data set to $aside" fi if [[ -n "$data" ]]; then local file="$data" if [[ "$data" == http://* || "$data" == https://* ]]; then file="$QC_HOME/quantcore_data.tar.gz" say " downloading $data" fetch "$data" "$file" fi if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi elif "$qc" has-data; then say " present -- newest prices later with: qc update" local universe_csv universe_csv="$("$qc" env | sed -n 's|^QC_DATA=||p')/universe/universe.csv" if grep -q ',wikipedia_stoxx600,' "$universe_csv" 2>/dev/null; then say "" say " NOTE: this data set was built by an earlier version from Wikipedia's STOXX 600 list, whose" say " guessed Yahoo symbols left out many European companies. Build it anew (~10 min):" say " $dir/install.sh --rebuild-data" fi elif [[ "$do_data" == 1 ]]; then say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)" say " candidates: the list shipped in the repository; companies Yahoo no longer knows are left out" "$qc" setup --offline else say " no data yet. Build it (~10 min): qc setup --offline or import a prepared one: qc data-import FILE" fi step "5/5 Commands: qc, quantcore, quantcore-api" local bin="$HOME/.local/bin" path_ok=0 mkdir -p "$bin" ln -sf "$qc" "$bin/qc" # `quantcore` and `quantcore-api` as on a laptop -- here they run inside the container, so the # commands in the README work on the Spark as written write_wrapper "$bin/quantcore" "$qc" quantcore write_wrapper "$bin/quantcore-api" "$qc" quantcore-api say " $bin/qc, quantcore, quantcore-api -> the containers ($qc)" case ":$PATH:" in *":$bin:"*) path_ok=1 ;; esac ensure_path_in_bashrc "$bin" if [[ "$do_start" == 1 ]]; then if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi if "$qc" has-data; then step "Self-test" "$qc" api selftest || say " some checks failed -- see the table above and: qc logs" fi fi local port port="$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')" step "Done" say " Test bench: http://127.0.0.1:$port/ui/ API docs: http://127.0.0.1:$port/docs (in the Spark's browser)" if [[ "$path_ok" == 0 ]]; then say " This terminal does not know the new commands yet. Open a NEW terminal, or run here:" say " export PATH=\"\$HOME/.local/bin:\$PATH\"" fi say " Then try:" say " qc status # what runs, cuOpt, data" say " quantcore risk NESN.SW AAPL --horizon week # the CLI, inside the container" say " quantcore-api optimize --cash 1000000 --cvar-limit 40000 --horizon week" say " The service keeps running; after a reboot: qc start. All commands: qc help" say " Guide: $dir/docs/SPARK_APPTAINER.md" } say() { printf '%s\n' "$*"; } step() { printf '\n== %s\n' "$*"; } die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } # APPTAINER_TMPDIR / APPTAINER_CACHEDIR (or the older SINGULARITY_* names) that point to a missing # directory make every download fail ("failed to create build parent dir"). Create the directory, # or use one of ours; persist_fixes() then records the choice in the settings file. fix_apptainer_dirs() { local name var legacy dirpath fallback free_gb for name in TMPDIR CACHEDIR; do var="APPTAINER_$name"; legacy="SINGULARITY_$name" dirpath="${!var:-${!legacy:-}}" [[ -n "$dirpath" ]] || continue if mkdir -p "$dirpath" 2>/dev/null && [[ -w "$dirpath" ]]; then continue; fi if [[ "$name" == TMPDIR ]]; then free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 )) fallback="$QC_HOME/apptainer-tmp" (( free_gb >= 15 )) && fallback="/tmp/$(id -un)-apptainer-tmp" else fallback="$QC_HOME/apptainer-cache" fi mkdir -p "$fallback" || die "$var=$dirpath does not exist and $fallback cannot be created" say " $var=$dirpath does not exist on this machine -> using $fallback" export "$var=$fallback" unset "$legacy" FIXED_VARS+=("$var=$fallback") done } persist_fixes() { # record the repaired directories in the settings file, so `qc` uses them too local config="$QC_HOME/quantcore.env" kv [[ -f "$config" ]] || return 0 for kv in "${FIXED_VARS[@]+"${FIXED_VARS[@]}"}"; do grep -q "^${kv%%=*}=" "$config" && continue printf '\n# set by install.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \ "$kv" >> "$config" done } # git on the machine, else git in a small container (alpine/git) -- Apptainer is all a Spark needs run_git() { if command -v git >/dev/null 2>&1; then git "$@" &2 mkdir -p "$QC_HOME/images" rm -f "$sif.part" "$APPTAINER" pull "$sif.part" "$GIT_IMAGE" &2 \ || die "could not download $GIT_IMAGE (network or proxy? a proxy needs https_proxy set)" mv -f "$sif.part" "$sif" fi for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY; do [[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}" done # the target directory's parent is bound in case it lies outside the home directory "$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" /dev/null; then say " kept your own $path (not replaced)" return 0 fi rm -f "$path" { printf '#!/usr/bin/env bash\n%s, written by install.sh\n' "$marker" printf '# %s on this DGX Spark: runs inside the quantcore container via deploy/apptainer/qc.sh\n' "$name" printf 'QC=%q\n' "$qcpath" if [[ "$name" == quantcore ]]; then cat <<'EOF' case "${1:-}" in serve) echo "On the Spark the API runs as a background service: qc start (stop: qc stop)" >&2 exec "$QC" start ;; setup|update) exec "$QC" "$@" ;; # through the running API, which reloads the data afterwards *) exec "$QC" cli "$@" ;; esac EOF else printf 'exec "$QC" api "$@"\n' fi } > "$path" chmod +x "$path" } # New terminals must find ~/.local/bin. Ubuntu's ~/.profile adds it only when it existed at login, # so a terminal opened right after the installation would not -- hence one guarded line in ~/.bashrc. ensure_path_in_bashrc() { # bin local rc="$HOME/.bashrc" marker="# added by Portfolio_Risk_Management install.sh" grep -q "$marker" "$rc" 2>/dev/null && return 0 printf '\n%s: the commands qc, quantcore, quantcore-api\ncase ":$PATH:" in *":%s:"*) ;; *) export PATH="%s:$PATH" ;; esac\n' \ "$marker" "$1" "$1" >> "$rc" say " added $1 to PATH in $rc" } fetch() { # url file -- with whatever this machine has if command -v curl >/dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1" elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1" elif command -v python3 >/dev/null 2>&1; then python3 -c 'import sys, urllib.request as u; u.urlretrieve(sys.argv[1], sys.argv[2])' "$1" "$2" else die "no download tool (curl, wget or python3) -- copy the file to this machine and pass its path"; fi } main "$@"