Portfolio_Risk_Management/install.sh

#!/usr/bin/env bash
# install.sh -- install or update quantcore on a DGX Spark in one step. Apptainer, no root, no git needed.
#
# On the Spark, in a terminal, ONE of these lines (whichever download tool the Spark has):
#
#   wget -qO- https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash
#   curl -fsSL https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh | bash
#   python3 -c "import urllib.request as u;print(u.urlopen('https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management/raw/branch/main/install.sh').read().decode())" | bash
#
# Options go after `bash -s --`, e.g.  ... | bash -s -- --data ~/Downloads/quantcore_data.tar.gz
#   --data FILE|URL   import this prepared data set instead of building one from Yahoo Finance
#   --no-data         skip the data step (later: qc setup --offline, or qc data-import FILE)
#   --rebuild-data    move the existing data set aside (to ~/quantcore/data.<date>) and build a new one
#   --no-start        install only, do not start the services
#   --dir DIR         where the code goes (default ~/Portfolio_Risk_Management)
#
# Steps, each skipped when it is already done -- running the script again updates everything:
#   1. checks Apptainer and the GPU, and repairs APPTAINER_TMPDIR / APPTAINER_CACHEDIR when they
#      point to a directory that does not exist (seen on managed machines: /scratch/$USER/...)
#   2. downloads the code with git -- or, without git on the machine, with git in a small container
#   3. deploy/apptainer/qc.sh install: the cuOpt image and the quantcore image (or its no-build fallback)
#   4. market data: imports --data, or builds the data set from Yahoo Finance (~15 min) when there is none.
#      The candidate stocks come from the list shipped in the repository (the iShares STOXX Europe 600
#      and S&P 500 constituents the course data was built from), so every installation gets the same
#      universe; prices and company data come from Yahoo.
#   5. starts cuOpt and the API, runs the self-test, and installs the command `qc`
#
# Everything is wrapped in main(), which runs only after bash has read the whole script: with
# `wget ... | bash` no command inside can swallow the rest of the script from standard input.

main() {
  set -euo pipefail
  local repo_url="${QC_REPO_URL:-https://gitea.fhgr.ch/herzogfloria/Portfolio_Risk_Management.git}"
  local branch="${QC_BRANCH:-main}"
  local dir="${QC_DIR:-$HOME/Portfolio_Risk_Management}"
  local data="" do_data=1 do_start=1 rebuild_data=0
  QC_HOME="${QC_HOME:-$HOME/quantcore}"
  APPTAINER="${APPTAINER:-apptainer}"
  GIT_IMAGE="${QC_GIT_IMAGE:-docker://alpine/git:2.54.0}"
  FIXED_VARS=()

  while [[ $# -gt 0 ]]; do
    case "$1" in
      --data)     data="${2:?--data needs a file or URL}"; shift ;;
      --data=*)   data="${1#--data=}" ;;
      --no-data)  do_data=0 ;;
      --no-start) do_start=0 ;;
      --dir)      dir="${2:?--dir needs a directory}"; shift ;;
      --dir=*)    dir="${1#--dir=}" ;;
      --rebuild-data) rebuild_data=1 ;;
      -h|--help)  sed -n '2,30p' "${BASH_SOURCE[0]:-/dev/null}" 2>/dev/null || true; return 0 ;;
      *)          die "unknown option: $1 (see the top of install.sh)" ;;
    esac
    shift
  done
  dir="${dir%/}"
  local qc="$dir/deploy/apptainer/qc.sh"

  step "1/5  Checking this machine"
  command -v "$APPTAINER" >/dev/null 2>&1 \
    || die "Apptainer is not installed on this machine ('$APPTAINER' not found). Ask the administrator."
  say "   $("$APPTAINER" --version), user $(id -un), home $HOME"
  if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
    say "   $(nvidia-smi -L | head -n 1)"
  else
    say "   no GPU visible: cuOpt will not run, optimisations will use HiGHS on the CPU"
  fi
  mkdir -p "$QC_HOME"
  fix_apptainer_dirs

  step "2/5  Code: $repo_url ($branch) -> $dir"
  local before="" after=""
  if [[ -d "$dir/.git" ]]; then
    before="$(run_git -C "$dir" rev-parse HEAD 2>/dev/null || true)"
    run_git -C "$dir" pull --ff-only origin "$branch" \
      || die "could not update $dir (did you change files there? 'git -C $dir status' shows which)"
  elif [[ -e "$dir" && -n "$(ls -A "$dir" 2>/dev/null)" ]]; then
    die "$dir exists but is not a git checkout of the project. Move it away, or use --dir DIR."
  else
    mkdir -p "$(dirname "$dir")"
    run_git clone --branch "$branch" "$repo_url" "$dir"
  fi
  after="$(run_git -C "$dir" rev-parse HEAD)"
  say "   at $(run_git -C "$dir" log -1 --format='%h %s (%cd)' --date=short)"
  chmod +x "$qc" "$dir/install.sh" 2>/dev/null || true

  step "3/5  Container images"
  QC_FROM_INSTALLER=1 "$qc" install
  persist_fixes
  if [[ -n "$before" && "$before" != "$after" ]]; then
    say "   the code changed since the last installation: rebuilding quantcore"
    "$qc" build
  fi

  step "4/5  Market data"
  if [[ "$rebuild_data" == 1 ]] && "$qc" has-data; then
    local data_dir aside
    data_dir="$("$qc" env | sed -n 's|^QC_DATA=||p')"
    aside="$data_dir.$(date +%Y%m%d-%H%M%S)"
    "$qc" stop
    ! "$qc" is-running || die "the service is still running; stop it (qc stop) and run this again"
    # move the contents, never the folder itself: a container that still had the folder mounted would
    # keep reading and writing the moved copy, while every new command looks at the empty new folder
    mkdir -p "$aside"
    find "$data_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$aside/" \;
    say "   moved the existing data set to $aside"
  fi
  if [[ -n "$data" ]]; then
    local file="$data"
    if [[ "$data" == http://* || "$data" == https://* ]]; then
      file="$QC_HOME/quantcore_data.tar.gz"
      say "   downloading $data"
      fetch "$data" "$file"
    fi
    if "$qc" has-data; then "$qc" data-import "$file" --force; else "$qc" data-import "$file"; fi
  elif "$qc" has-data; then
    say "   present -- newest prices later with: qc update"
    local universe_csv
    universe_csv="$("$qc" env | sed -n 's|^QC_DATA=||p')/universe/universe.csv"
    if grep -q ',wikipedia_stoxx600,' "$universe_csv" 2>/dev/null; then
      say ""
      say "   NOTE: this data set was built by an earlier version from Wikipedia's STOXX 600 list, whose"
      say "   guessed Yahoo symbols left out many European companies. Build it anew (~10 min):"
      say "       $dir/install.sh --rebuild-data"
    fi
  elif [[ "$do_data" == 1 ]]; then
    say "   no data yet: building the data set from Yahoo Finance (~15 min, needs internet)"
    say "   candidates: the list shipped in the repository; companies Yahoo no longer knows are left out"
    "$qc" setup --offline
  else
    say "   no data yet. Build it (~10 min): qc setup --offline   or import a prepared one: qc data-import FILE"
  fi

  step "5/5  Commands: qc, quantcore, quantcore-api"
  local bin="$HOME/.local/bin" path_ok=0
  mkdir -p "$bin"
  ln -sf "$qc" "$bin/qc"
  # `quantcore` and `quantcore-api` as on a laptop -- here they run inside the container, so the
  # commands in the README work on the Spark as written
  write_wrapper "$bin/quantcore" "$qc" quantcore
  write_wrapper "$bin/quantcore-api" "$qc" quantcore-api
  say "   $bin/qc, quantcore, quantcore-api -> the containers ($qc)"
  case ":$PATH:" in *":$bin:"*) path_ok=1 ;; esac
  ensure_path_in_bashrc "$bin"

  if [[ "$do_start" == 1 ]]; then
    if [[ -n "$before" && "$before" != "$after" ]]; then "$qc" restart; else "$qc" start; fi
    if "$qc" has-data; then
      step "Self-test"
      "$qc" api selftest || say "   some checks failed -- see the table above and: qc logs"
    fi
  fi

  local port
  port="$("$qc" env | sed -n 's|^API=http://127.0.0.1:||p')"
  step "Done"
  say "   Test bench:  http://127.0.0.1:$port/ui/     API docs: http://127.0.0.1:$port/docs   (in the Spark's browser)"
  if [[ "$path_ok" == 0 ]]; then
    say "   This terminal does not know the new commands yet. Open a NEW terminal, or run here:"
    say "       export PATH=\"\$HOME/.local/bin:\$PATH\""
  fi
  say "   Then try:"
  say "       qc status                                     # what runs, cuOpt, data"
  say "       quantcore risk NESN.SW AAPL --horizon week     # the CLI, inside the container"
  say "       quantcore-api optimize --cash 1000000 --cvar-limit 40000 --horizon week"
  say "   The service keeps running; after a reboot: qc start.  All commands: qc help"
  say "   Guide: $dir/docs/SPARK_APPTAINER.md"
}

say()  { printf '%s\n' "$*"; }
step() { printf '\n== %s\n' "$*"; }
die()  { printf '\nerror: %s\n' "$*" >&2; exit 1; }

# APPTAINER_TMPDIR / APPTAINER_CACHEDIR (or the older SINGULARITY_* names) that point to a missing
# directory make every download fail ("failed to create build parent dir"). Create the directory,
# or use one of ours; persist_fixes() then records the choice in the settings file.
fix_apptainer_dirs() {
  local name var legacy dirpath fallback free_gb
  for name in TMPDIR CACHEDIR; do
    var="APPTAINER_$name"; legacy="SINGULARITY_$name"
    dirpath="${!var:-${!legacy:-}}"
    [[ -n "$dirpath" ]] || continue
    if mkdir -p "$dirpath" 2>/dev/null && [[ -w "$dirpath" ]]; then continue; fi
    if [[ "$name" == TMPDIR ]]; then
      free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 ))
      fallback="$QC_HOME/apptainer-tmp"
      (( free_gb >= 15 )) && fallback="/tmp/$(id -un)-apptainer-tmp"
    else
      fallback="$QC_HOME/apptainer-cache"
    fi
    mkdir -p "$fallback" || die "$var=$dirpath does not exist and $fallback cannot be created"
    say "   $var=$dirpath does not exist on this machine -> using $fallback"
    export "$var=$fallback"
    unset "$legacy"
    FIXED_VARS+=("$var=$fallback")
  done
}

persist_fixes() {  # record the repaired directories in the settings file, so `qc` uses them too
  local config="$QC_HOME/quantcore.env" kv
  [[ -f "$config" ]] || return 0
  for kv in "${FIXED_VARS[@]+"${FIXED_VARS[@]}"}"; do
    grep -q "^${kv%%=*}=" "$config" && continue
    printf '\n# set by install.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \
      "$kv" >> "$config"
  done
}

# git on the machine, else git in a small container (alpine/git) -- Apptainer is all a Spark needs
run_git() {
  if command -v git >/dev/null 2>&1; then git "$@" </dev/null; return; fi
  local sif="$QC_HOME/images/git.sif" v
  if [[ ! -f "$sif" ]]; then
    say "   git is not installed here: using git from a container ($GIT_IMAGE, ~40 MB, once)" >&2
    mkdir -p "$QC_HOME/images"
    rm -f "$sif.part"
    "$APPTAINER" pull "$sif.part" "$GIT_IMAGE" </dev/null >&2 \
      || die "could not download $GIT_IMAGE (network or proxy? a proxy needs https_proxy set)"
    mv -f "$sif.part" "$sif"
  fi
  for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY; do
    [[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}"
  done
  # the target directory's parent is bound in case it lies outside the home directory
  "$APPTAINER" exec --cleanenv --bind "$(dirname "$dir")" "$sif" git "$@" </dev/null
}

# A command in ~/.local/bin that forwards to qc.sh. An existing file that install.sh did not write
# (say, a pip --user install) is left alone.
write_wrapper() {  # path qc.sh name
  local path="$1" qcpath="$2" name="$3" marker="# portfolio-risk-management wrapper"
  if [[ -e "$path" && ! -L "$path" ]] && ! grep -q "$marker" "$path" 2>/dev/null; then
    say "   kept your own $path (not replaced)"
    return 0
  fi
  rm -f "$path"
  {
    printf '#!/usr/bin/env bash\n%s, written by install.sh\n' "$marker"
    printf '# %s on this DGX Spark: runs inside the quantcore container via deploy/apptainer/qc.sh\n' "$name"
    printf 'QC=%q\n' "$qcpath"
    if [[ "$name" == quantcore ]]; then
      cat <<'EOF'
case "${1:-}" in
  serve)        echo "On the Spark the API runs as a background service: qc start (stop: qc stop)" >&2
                exec "$QC" start ;;
  setup|update) exec "$QC" "$@" ;;   # through the running API, which reloads the data afterwards
  *)            exec "$QC" cli "$@" ;;
esac
EOF
    else
      printf 'exec "$QC" api "$@"\n'
    fi
  } > "$path"
  chmod +x "$path"
}

# New terminals must find ~/.local/bin. Ubuntu's ~/.profile adds it only when it existed at login,
# so a terminal opened right after the installation would not -- hence one guarded line in ~/.bashrc.
ensure_path_in_bashrc() {  # bin
  local rc="$HOME/.bashrc" marker="# added by Portfolio_Risk_Management install.sh"
  grep -q "$marker" "$rc" 2>/dev/null && return 0
  printf '\n%s: the commands qc, quantcore, quantcore-api\ncase ":$PATH:" in *":%s:"*) ;; *) export PATH="%s:$PATH" ;; esac\n' \
    "$marker" "$1" "$1" >> "$rc"
  say "   added $1 to PATH in $rc"
}

fetch() {  # url file -- with whatever this machine has
  if command -v curl >/dev/null 2>&1; then curl -fL --retry 3 -o "$2" "$1"
  elif command -v wget >/dev/null 2>&1; then wget -O "$2" "$1"
  elif command -v python3 >/dev/null 2>&1; then
    python3 -c 'import sys, urllib.request as u; u.urlretrieve(sys.argv[1], sys.argv[2])' "$1" "$2"
  else die "no download tool (curl, wget or python3) -- copy the file to this machine and pass its path"; fi
}

main "$@"