diff --git a/README.md b/README.md index 8c1fd49..5a7faab 100644 --- a/README.md +++ b/README.md @@ -39,18 +39,17 @@ Apptainer, no root), [docs/SPARK_DEPLOYMENT.md](docs/SPARK_DEPLOYMENT.md) (the D ## Quick start on your DGX Spark (students) The service and cuOpt run as two Apptainer containers on your Spark and talk over `127.0.0.1`. -No root rights, no Docker, nothing exposed to the network: +No root rights, no Docker, no git, nothing exposed to the network. One line in a terminal on the +Spark installs everything, builds the market data from Yahoo Finance and starts the service: ```bash -git clone https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git -cd Portfolio_Risk_Management -deploy/apptainer/qc.sh install # once: cuOpt image, quantcore image, self-test -deploy/apptainer/qc.sh data-import quantcore_data.tar.gz # prepared data set (or: qc.sh setup, ~15 min) -deploy/apptainer/qc.sh start # -> http://127.0.0.1:5001/ui/ and /docs -deploy/apptainer/qc.sh api selftest # every endpoint, pass/fail +wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash ``` -Your platform then calls the API at `http://127.0.0.1:5001`. The full guide, with settings, +Without `wget`, use `curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash`. With a prepared data set, add +`bash -s -- --data /path/to/quantcore_data.tar.gz` in place of the final `bash`. Afterwards the command `qc` +runs the service (`qc status`, `qc stop`, `qc start`, `qc upgrade`), the test bench is at +`http://127.0.0.1:5001/ui/`, and your platform calls the API at `http://127.0.0.1:5001`. The full guide, with settings, several users on one Spark and troubleshooting, is [docs/SPARK_APPTAINER.md](docs/SPARK_APPTAINER.md). ## Quick start (local) @@ -243,6 +242,7 @@ quantcore/ client.py QuantcoreClient -- the API as a Python object (needs only a URL) cli.py `quantcore` command (in-process services, needs the data directory) apicli.py `quantcore-api` command (HTTP client of a running service, incl. `selftest`) +install.sh one-step installer/updater for a DGX Spark (wget ... | bash): code, images, data, start deploy/apptainer/ qc.sh (install/start/stop/cli on a Spark without root), quantcore.def, pinned constraints.txt deploy/ Dockerfile, compose service blocks (Spark + a bench-free one), deploy_to_spark.sh docs/ USAGE.md, ARCHITECTURE.md, METHODOLOGY.md, SPARK_APPTAINER.md, SPARK_DEPLOYMENT.md diff --git a/deploy/apptainer/qc.sh b/deploy/apptainer/qc.sh index 195ad48..588b118 100755 --- a/deploy/apptainer/qc.sh +++ b/deploy/apptainer/qc.sh @@ -19,7 +19,8 @@ # Guide: docs/SPARK_APPTAINER.md. set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# resolve the link ~/.local/bin/qc (made by install.sh) to the script in the repository +SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" QC_HOME="${QC_HOME:-$HOME/quantcore}" CONFIG="$QC_HOME/quantcore.env" @@ -82,7 +83,8 @@ need_image() { # path what [[ -f "$1" ]] || die "$2 image missing ($1) -- run: $0 install" } -# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3 (both on DGX OS). +# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3, else plain bash: +# a barebone Spark may have neither of the first two. http_get() { local url="$1" timeout="${2:-3}" if command -v curl >/dev/null 2>&1; then @@ -91,10 +93,25 @@ http_get() { python3 -c 'import sys, urllib.request as u; sys.stdout.write(u.urlopen(sys.argv[1], timeout=float(sys.argv[2])).read().decode())' \ "$url" "$timeout" 2>/dev/null else - return 2 + http_get_bash "$url" "$timeout" fi } +# HTTP/1.0 GET over bash's /dev/tcp, for the http://host:port/path addresses qc.sh checks. +http_get_bash() { + local rest="${1#http://}" t="${2:-3}" hostport path host port + hostport="${rest%%/*}"; path="${rest#"$hostport"}"; [[ -n "$path" ]] || path="/" + host="${hostport%%:*}"; port=80; [[ "$hostport" == *:* ]] && port="${hostport##*:}" + ( + exec 3<>"/dev/tcp/$host/$port" || exit 1 + printf 'GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' "$path" "$host" >&3 + IFS= read -r -t "$t" status <&3 || exit 1 + [[ "$status" =~ ^HTTP/[0-9.]+\ 2[0-9][0-9] ]] || exit 1 + while IFS= read -r -t "$t" line <&3; do line="${line%$'\r'}"; [[ -z "$line" ]] && break; done + while IFS= read -r -t "$t" line <&3 || [[ -n "$line" ]]; do printf '%s\n' "$line"; line=""; done + ) 2>/dev/null +} + port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } free_port() { # first free port from $1 upwards @@ -108,13 +125,64 @@ instance_running() { } instance_log() { # name out|err -> path of the instance log (Apptainer keeps them per host and user) + # this host first: a home directory on a network drive holds the logs of every machine; the user + # directory may be spelt "name" or "name@domain", so match any local f - for f in "$HOME/.apptainer/instances/logs/"*"/$ME/$1.$2"; do + for f in "$HOME/.apptainer/instances/logs/$(hostname)/"*"/$1.$2" "$HOME/.apptainer/instances/logs/"*/*"/$1.$2"; do [[ -f "$f" ]] && { say "$f"; return 0; } done return 1 } +# Apptainer's working and cache directories. Some installations export APPTAINER_TMPDIR or +# APPTAINER_CACHEDIR for a convention such as /scratch/$USER that this machine does not have; every +# pull then dies with "failed to create build parent dir". Create the directory, or fall back. +APPTAINER_DIRS_READY=0 +usable_dir() { mkdir -p "$1" 2>/dev/null && [[ -d "$1" && -w "$1" ]]; } + +prepare_apptainer_dirs() { + [[ "$APPTAINER_DIRS_READY" == 1 ]] && return 0 + local name var legacy dir fallback tmp_free_gb + for name in TMPDIR CACHEDIR; do + var="APPTAINER_$name"; legacy="SINGULARITY_$name" + dir="${!var:-${!legacy:-}}" + [[ -n "$dir" ]] || continue # not set: Apptainer's defaults (/tmp, ~/.apptainer/cache) apply + usable_dir "$dir" && continue + if [[ "$name" == TMPDIR ]]; then + # unpacking the cuOpt image needs ~12 GB: local /tmp if it has the room, else our own directory + tmp_free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 )) + fallback="$QC_HOME/apptainer-tmp" + (( tmp_free_gb >= 15 )) && fallback="/tmp/$ME-apptainer-tmp" + else + fallback="$QC_HOME/apptainer-cache" + fi + usable_dir "$fallback" || die "$var=$dir does not exist, and neither it nor $fallback can be created" + export "$var=$fallback" + unset "$legacy" + if [[ -f "$CONFIG" ]] && ! grep -q "^$var=" "$CONFIG"; then + printf '\n# set by qc.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \ + "$var=$fallback" >> "$CONFIG" + warn "$var=$dir (set in your environment) does not exist and cannot be created. + Using $fallback instead; saved in $CONFIG." + else + warn "$var=$dir does not exist and cannot be created; using $fallback for this run." + fi + done + APPTAINER_DIRS_READY=1 +} + +pull_image() { # destination source -- download an image to a .sif file, with a useful error + prepare_apptainer_dirs + rm -f "$1.part" + if ! "$APPTAINER" pull "$1.part" "$2"; then + rm -f "$1.part" + die "could not download $2. Check that this machine reaches the registry (a proxy needs + https_proxy set), and that ${APPTAINER_TMPDIR:-/tmp} has ~12 GB free (APPTAINER_TMPDIR chooses another + directory). Then run the command again." + fi + mv -f "$1.part" "$1" +} + show_log_tail() { # name local f for kind in err out; do @@ -195,6 +263,7 @@ cmd_install() { say " wrote $CONFIG" fi + prepare_apptainer_dirs step "GPU" if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then say " $(nvidia-smi -L | head -n 1)" @@ -212,6 +281,7 @@ cmd_install() { step "Self-test of the quantcore container ($(qc_mode) mode)" qc_selftest + [[ "${QC_FROM_INSTALLER:-0}" == 1 ]] && return 0 # install.sh does the data and the start itself step "Data ($QC_DATA)" if data_is_empty; then say " empty. Either copy a prepared data set: $0 data-import quantcore_data.tar.gz" @@ -235,9 +305,7 @@ cmd_pull_cuopt() { fi step "Pulling cuOpt: $QC_CUOPT_IMAGE -> $dst (~3.3 GB, several minutes)" mkdir -p "$QC_IMAGES" - rm -f "$dst.part" - "$APPTAINER" pull "$dst.part" "$QC_CUOPT_IMAGE" - mv -f "$dst.part" "$dst" + pull_image "$dst" "$QC_CUOPT_IMAGE" } qc_selftest() { @@ -271,7 +339,7 @@ get_quantcore_image() { # force(0|1) docker://*|oras://*|library://*|http://*|https://*) if [[ -f "$dst" && "$1" != 1 ]]; then say " present: $dst"; else say " pulling $QC_IMAGE -> $dst" - rm -f "$dst.part"; "$APPTAINER" pull "$dst.part" "$QC_IMAGE"; mv -f "$dst.part" "$dst" + pull_image "$dst" "$QC_IMAGE" fi ;; *) [[ -f "$QC_IMAGE" ]] || die "QC_IMAGE=$QC_IMAGE is neither 'build', a URL, nor an existing .sif file" @@ -283,6 +351,7 @@ build_sif() { # build quantcore.sif from the definition; non-zero when this App local dst="$QC_IMAGES/quantcore.sif" mkdir -p "$QC_IMAGES" say " building $dst from $REPO_DIR (a few minutes)" + prepare_apptainer_dirs rm -f "$dst.part" # %files paths in the definition are relative to the repository root. Without root Apptainer # builds through --fakeroot (user namespaces); a Spark may not allow that. @@ -298,10 +367,11 @@ build_venv() { # the no-build fallback: pull a plain Python image, install quan mkdir -p "$QC_IMAGES" "$QC_HOME" if [[ ! -f "$PY_SIF" ]]; then say " pulling $QC_PYTHON_IMAGE -> $PY_SIF" - rm -f "$PY_SIF.part"; "$APPTAINER" pull "$PY_SIF.part" "$QC_PYTHON_IMAGE"; mv -f "$PY_SIF.part" "$PY_SIF" + pull_image "$PY_SIF" "$QC_PYTHON_IMAGE" fi say " installing quantcore into $QC_VENV (pinned versions from deploy/apptainer/constraints.txt)" - local work; work="$(mktemp -d "${TMPDIR:-/tmp}/quantcore-src.XXXXXX")" + # a copy of the sources under QC_HOME (not $TMPDIR, which may point nowhere on a managed machine) + local work; work="$(mktemp -d "$QC_HOME/.quantcore-src.XXXXXX")" # install from a copy: pip writes build/ and *.egg-info next to the sources cp -r "$REPO_DIR/pyproject.toml" "$REPO_DIR/README.md" "$REPO_DIR/quantcore" "$work/" "$APPTAINER" exec --cleanenv --bind "$QC_HOME" --bind "$work" "$PY_SIF" python -m venv --clear "$QC_VENV" @@ -511,6 +581,11 @@ cmd_data_export() { say "wrote $dst ($(du -h "$dst" | cut -f1))" } +cmd_upgrade() { # newest code from the repository, rebuild, restart -- install.sh does all of it + [[ -f "$REPO_DIR/install.sh" ]] || die "$REPO_DIR/install.sh missing; download the code again" + exec bash "$REPO_DIR/install.sh" --dir "$REPO_DIR" --no-data "$@" +} + cmd_env() { cat <@ ``` -A shorter command saves typing. Put this line in your `~/.bashrc`: - -```bash -alias qc="$HOME/Portfolio_Risk_Management/deploy/apptainer/qc.sh" -``` - -The rest of this guide writes `qc` for the script. +The installer puts the command `qc` into `~/.local/bin`. Open a new terminal once so that the +shell finds it. `qc` is the same as `deploy/apptainer/qc.sh` in the code folder. ## Daily use @@ -60,6 +77,7 @@ qc start # cuOpt + API; after a reboot, run it again qc status # what runs, whether cuOpt is reachable, how much data there is qc logs # API log; `qc logs cuopt` for the solver, add -f to follow qc update # the newest prices and FX rates from Yahoo Finance +qc upgrade # the newest version of the code: download, rebuild, restart qc stop # stop both containers ``` @@ -140,11 +158,8 @@ with `QC_IMAGE=venv`. `QC_IMAGE` also accepts the path of a prebuilt `quantcore.sif`, or a `docker://` or `oras://` address to pull one from. -**After `git pull`**, rebuild and restart: - -```bash -git pull && qc build && qc restart -``` +**New versions of the code** come with `qc upgrade`. It downloads the changes, rebuilds quantcore +and restarts the service. ## How it works @@ -198,6 +213,9 @@ git pull && qc build && qc restart | The cuOpt log says `Read-only file system: '/opt/cuopt/.cupy'` | cuOpt was started without `qc.sh`. Start it with `qc start`, or give it a writable home directory as described in "Running cuOpt by hand". | | `/optimize` answers 503, "no market data yet" | Run `qc data-import ` or `qc setup`. | | Yahoo errors during `qc setup` or `qc update` | Yahoo limits how fast it answers. Run the command again later; stored data stays intact. | +| `FATAL: ... failed to create build parent dir: stat /scratch/...` | Your account sets `APPTAINER_TMPDIR` to a folder that does not exist. The installer and `qc` now detect this, use a folder that works, and save the choice in `~/quantcore/quantcore.env`. Run the installer again. | +| `git: command not found` | Use the one-line installer. It needs no git. | +| `qc: command not found` right after the installation | Open a new terminal, or run `export PATH="$HOME/.local/bin:$PATH"`. | | `qc install` stops with "no 'instance run'" | The Spark's Apptainer is older than 1.1. Ask the administrator to update it. | | The cuOpt pull fails with "no space left on device" | Apptainer unpacks images in `/tmp`. Point it at a larger directory: `APPTAINER_TMPDIR=~/quantcore/tmp qc pull-cuopt`. | | Model status `FeasibleFound` instead of `Optimal` | The time limit was reached. Raise `time_limit` in the request or `QUANTCORE_CUOPT_TIME_LIMIT`, or reduce `n_scenarios` or `max_assets`. | @@ -252,3 +270,14 @@ was then tested with a prebuilt `quantcore.sif`. In both modes: - `qc update` fetched the newest Yahoo prices through the running API. - A taken API port stopped `qc start` with a suggested free port, and that port worked. - The test suite passed inside the container, including the round trip to the live cuOpt server. + +The one-line installer was then tested on a stripped-down machine. It had no git, curl, wget or +python3, a home folder named like a domain account (`/home/name@edu.local`), and `APPTAINER_TMPDIR` +pointing to a `/scratch` folder that does not exist. The installer repaired the folders, fetched the +code with git from a container, installed, started and passed the self-test. `qc upgrade` picked up +a new commit, rebuilt quantcore and restarted it in under a minute. `qc setup` built a data set from +scratch inside the container. + +`qc setup` takes the STOXX 600 candidates from the iShares holdings file. In October 2026 that link +answered 404, and setup fell back to Wikipedia's list of 465 names. A data set built by a student can +therefore have fewer European stocks than the prepared one. diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..49c8bc0 --- /dev/null +++ b/install.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# install.sh -- install or update quantcore on a DGX Spark in one step. Apptainer, no root, no git needed. +# +# On the Spark, in a terminal, ONE of these lines (whichever download tool the Spark has): +# +# wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash +# curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash +# python3 -c "import urllib.request as u;print(u.urlopen('https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh').read().decode())" | bash +# +# Options go after `bash -s --`, e.g. ... | bash -s -- --data ~/Downloads/quantcore_data.tar.gz +# --data FILE|URL import this prepared data set instead of building one from Yahoo Finance +# --no-data skip the data step (later: qc setup, or qc data-import FILE) +# --no-start install only, do not start the services +# --dir DIR where the code goes (default ~/Portfolio_Risk_Management) +# +# Steps, each skipped when it is already done -- running the script again updates everything: +# 1. checks Apptainer and the GPU, and repairs APPTAINER_TMPDIR / APPTAINER_CACHEDIR when they +# point to a directory that does not exist (seen on managed machines: /scratch/$USER/...) +# 2. downloads the code with git -- or, without git on the machine, with git in a small container +# 3. deploy/apptainer/qc.sh install: the cuOpt image and the quantcore image (or its no-build fallback) +# 4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none +# 5. starts cuOpt and the API, runs the self-test, and installs the command `qc` +# +# Everything is wrapped in main(), which runs only after bash has read the whole script: with +# `wget ... | bash` no command inside can swallow the rest of the script from standard input. + +main() { + set -euo pipefail + local repo_url="${QC_REPO_URL:-https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git}" + local branch="${QC_BRANCH:-main}" + local dir="${QC_DIR:-$HOME/Portfolio_Risk_Management}" + local data="" do_data=1 do_start=1 + QC_HOME="${QC_HOME:-$HOME/quantcore}" + APPTAINER="${APPTAINER:-apptainer}" + GIT_IMAGE="${QC_GIT_IMAGE:-docker://alpine/git:2.54.0}" + FIXED_VARS=() + + while [[ $# -gt 0 ]]; do + case "$1" in + --data) data="${2:?--data needs a file or URL}"; shift ;; + --data=*) data="${1#--data=}" ;; + --no-data) do_data=0 ;; + --no-start) do_start=0 ;; + --dir) dir="${2:?--dir needs a directory}"; shift ;; + --dir=*) dir="${1#--dir=}" ;; + -h|--help) sed -n '2,26p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;; + *) die "unknown option: $1 (see the top of install.sh)" ;; + esac + shift + done + dir="${dir%/}" + local qc="$dir/deploy/apptainer/qc.sh" + + step "1/5 Checking this machine" + command -v "$APPTAINER" >/dev/null 2>&1 \ + || die "Apptainer is not installed on this machine ('$APPTAINER' not found). Ask the administrator." + say " $("$APPTAINER" --version), user $(id -un), home $HOME" + if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then + say " $(nvidia-smi -L | head -n 1)" + else + say " no GPU visible: cuOpt will not run, optimisations will use HiGHS on the CPU" + fi + mkdir -p "$QC_HOME" + fix_apptainer_dirs + + step "2/5 Code: $repo_url ($branch) -> $dir" + local before="" after="" + if [[ -d "$dir/.git" ]]; then + before="$(run_git -C "$dir" rev-parse HEAD 2>/dev/null || true)" + run_git -C "$dir" pull --ff-only origin "$branch" \ + || die "could not update $dir (did you change files there? 'git -C $dir status' shows which)" + elif [[ -e "$dir" && -n "$(ls -A "$dir" 2>/dev/null)" ]]; then + die "$dir exists but is not a git checkout of the project. Move it away, or use --dir DIR." + else + mkdir -p "$(dirname "$dir")" + run_git clone --branch "$branch" "$repo_url" "$dir" + fi + after="$(run_git -C "$dir" rev-parse HEAD)" + say " at $(run_git -C "$dir" log -1 --format='%h %s (%cd)' --date=short)" + chmod +x "$qc" "$dir/install.sh" 2>/dev/null || true + + step "3/5 Container images" + QC_FROM_INSTALLER=1 "$qc" install + persist_fixes + if [[ -n "$before" && "$before" != "$after" ]]; then + say " the code changed since the last installation: rebuilding quantcore" + "$qc" build + fi + + step "4/5 Market data" + if [[ -n "$data" ]]; then + local file="$data" + if [[ "$data" == http://* || "$data" == https://* ]]; then + file="$QC_HOME/quantcore_data.tar.gz" + say " downloading $data" + fetch "$data" "$file" + fi + if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi + elif "$qc" has-data; then + say " present -- newest prices later with: qc update" + elif [[ "$do_data" == 1 ]]; then + say " no data yet: building the data set from Yahoo Finance (~15 min, needs internet)" + "$qc" setup + else + say " skipped (--no-data). Later: qc setup or qc data-import FILE" + fi + + step "5/5 The command 'qc'" + mkdir -p "$HOME/.local/bin" + ln -sf "$qc" "$HOME/.local/bin/qc" + say " $HOME/.local/bin/qc -> $qc" + case ":$PATH:" in + *":$HOME/.local/bin:"*) ;; + *) say " open a new terminal to use it (or run: export PATH=\"\$HOME/.local/bin:\$PATH\")" ;; + esac + + if [[ "$do_start" == 1 ]]; then + if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi + if "$qc" has-data; then + step "Self-test" + "$qc" api selftest || say " some checks failed -- see the table above and: qc logs" + fi + fi + + step "Done" + say " Test bench: http://127.0.0.1:$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')/ui/ (on this Spark)" + say " Commands: qc status | qc stop | qc start | qc logs | qc update | qc upgrade | qc help" + say " Guide: $dir/docs/SPARK_APPTAINER.md" +} + +say() { printf '%s\n' "$*"; } +step() { printf '\n== %s\n' "$*"; } +die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } + +# APPTAINER_TMPDIR / APPTAINER_CACHEDIR (or the older SINGULARITY_* names) that point to a missing +# directory make every download fail ("failed to create build parent dir"). Create the directory, +# or use one of ours; persist_fixes() then records the choice in the settings file. +fix_apptainer_dirs() { + local name var legacy dirpath fallback free_gb + for name in TMPDIR CACHEDIR; do + var="APPTAINER_$name"; legacy="SINGULARITY_$name" + dirpath="${!var:-${!legacy:-}}" + [[ -n "$dirpath" ]] || continue + if mkdir -p "$dirpath" 2>/dev/null && [[ -w "$dirpath" ]]; then continue; fi + if [[ "$name" == TMPDIR ]]; then + free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 )) + fallback="$QC_HOME/apptainer-tmp" + (( free_gb >= 15 )) && fallback="/tmp/$(id -un)-apptainer-tmp" + else + fallback="$QC_HOME/apptainer-cache" + fi + mkdir -p "$fallback" || die "$var=$dirpath does not exist and $fallback cannot be created" + say " $var=$dirpath does not exist on this machine -> using $fallback" + export "$var=$fallback" + unset "$legacy" + FIXED_VARS+=("$var=$fallback") + done +} + +persist_fixes() { # record the repaired directories in the settings file, so `qc` uses them too + local config="$QC_HOME/quantcore.env" kv + [[ -f "$config" ]] || return 0 + for kv in "${FIXED_VARS[@]+"${FIXED_VARS[@]}"}"; do + grep -q "^${kv%%=*}=" "$config" && continue + printf '\n# set by install.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \ + "$kv" >> "$config" + done +} + +# git on the machine, else git in a small container (alpine/git) -- Apptainer is all a Spark needs +run_git() { + if command -v git >/dev/null 2>&1; then git "$@" &2 + mkdir -p "$QC_HOME/images" + rm -f "$sif.part" + "$APPTAINER" pull "$sif.part" "$GIT_IMAGE" &2 \ + || die "could not download $GIT_IMAGE (network or proxy? a proxy needs https_proxy set)" + mv -f "$sif.part" "$sif" + fi + for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY; do + [[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}" + done + # the target directory's parent is bound in case it lies outside the home directory + "$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" /dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1" + elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1" + elif command -v python3 >/dev/null 2>&1; then + python3 -c 'import sys, urllib.request as u; u.urlretrieve(sys.argv[1], sys.argv[2])' "$1" "$2" + else die "no download tool (curl, wget or python3) -- copy the file to this machine and pass its path"; fi +} + +main "$@"