#!/usr/bin/env bash
# qc.sh -- quantcore standalone on a DGX Spark with Apptainer: no root, no Docker, no Tailscale.
#
# cuOpt and quantcore each run in their own Apptainer container on this machine and talk over
# 127.0.0.1. The API and the browser test bench listen on 127.0.0.1 too, so they are used from this
# Spark (its desktop browser, a terminal, a notebook) and are not reachable from the network.
#
#   qc.sh install              once: config, cuOpt image, quantcore image, data directory
#   qc.sh data-import FILE     copy a prepared data set (.tar.gz or directory) ...
#   qc.sh setup                ... or build it from Yahoo Finance (~15 min)
#   qc.sh start                cuOpt + API  ->  http://127.0.0.1:5001/ui/  and  /docs
#   qc.sh status | logs [api|cuopt] [-f] | stop | restart
#   qc.sh cli <command> ...    the quantcore CLI in the container, no API needed
#                              (e.g. qc.sh cli optimize --cash 1e6 --cvar-limit 40000)
#   qc.sh api <command> ...    quantcore-api against the local API (e.g. qc.sh api selftest)
#   qc.sh help                 all commands
#
# Settings: $QC_HOME/quantcore.env (QC_HOME defaults to ~/quantcore); see quantcore.env.example.
# Guide: docs/SPARK_APPTAINER.md.
set -euo pipefail

# resolve the link ~/.local/bin/qc (made by install.sh) to the script in the repository
SCRIPT_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
QC_HOME="${QC_HOME:-$HOME/quantcore}"
CONFIG="$QC_HOME/quantcore.env"
APPTAINER="${APPTAINER:-apptainer}"
API_INSTANCE="quantcore-api"
CUOPT_INSTANCE="quantcore-cuopt"
ME="$(id -un)"

# --- settings: defaults < config file < environment ------------------------------------------
_env_overrides="$(declare -p $(compgen -v | grep -E '^(QC_|QUANTCORE_)' || true) 2>/dev/null || true)"
QC_DATA="$QC_HOME/data"
QC_IMAGES="$QC_HOME/images"
QC_API_PORT=5001
QC_CUOPT_PORT=5000
QC_CUOPT_START=auto
QC_CUOPT_IMAGE="docker://nvidia/cuopt:26.8.0-cuda13.3-py3.14"
QC_IMAGE=build
QC_PYTHON_IMAGE="docker://python:3.12-slim"   # base of the no-build fallback (same as quantcore.def)
if [[ -f "$CONFIG" ]]; then
  set -a
  # shellcheck disable=SC1090
  source "$CONFIG"
  set +a
fi
[[ -n "$_env_overrides" ]] && eval "$_env_overrides"
unset _env_overrides
QC_API_URL="http://127.0.0.1:$QC_API_PORT"
QC_VENV="$QC_HOME/venv"
PY_SIF="$QC_IMAGES/python.sif"
CUOPT_URL="${QUANTCORE_CUOPT_URL:-http://127.0.0.1:$QC_CUOPT_PORT}"
CUOPT_EXTERNAL=0   # 1: QUANTCORE_CUOPT_URL names a cuOpt somewhere else -- never start our own
[[ -n "${QUANTCORE_CUOPT_URL:-}" ]] && CUOPT_EXTERNAL=1

# --- helpers ----------------------------------------------------------------------------------
say()  { printf '%s\n' "$*"; }
step() { printf '\n>> %s\n' "$*"; }
warn() { printf 'warning: %s\n' "$*" >&2; }
die()  { printf 'error: %s\n' "$*" >&2; exit 1; }

quantcore_sif() { if [[ "$QC_IMAGE" == *.sif && -f "$QC_IMAGE" ]]; then say "$QC_IMAGE"; else say "$QC_IMAGES/quantcore.sif"; fi; }

# How quantcore runs: "sif" = the quantcore image (built or prebuilt); "venv" = the no-build fallback,
# a plain Python image plus a virtualenv in $QC_HOME/venv (for a Spark that allows no image builds).
qc_mode() {
  if [[ "$QC_IMAGE" == "venv" ]]; then say venv; return; fi
  if [[ -f "$(quantcore_sif)" ]]; then say sif; return; fi
  if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" ]]; then say venv; return; fi
  say none
}
qc_image() { if [[ "$(qc_mode)" == venv ]]; then say "$PY_SIF"; else quantcore_sif; fi; }
qc_bin()   { if [[ "$(qc_mode)" == venv ]]; then say "$QC_VENV/bin/$1"; else say "$1"; fi; }
cuopt_sif()     { if [[ -f "$QC_CUOPT_IMAGE" ]]; then say "$QC_CUOPT_IMAGE"; else say "$QC_IMAGES/cuopt.sif"; fi; }

need_apptainer() {
  command -v "$APPTAINER" >/dev/null 2>&1 \
    || die "'$APPTAINER' not found. This Spark needs Apptainer (ask the administrator), or set APPTAINER=singularity."
}

need_image() {  # path what
  [[ -f "$1" ]] || die "$2 image missing ($1) -- run: $0 install"
}

# HTTP GET to stdout; exit status 0 only for a 2xx answer. curl, else python3, else plain bash:
# a barebone Spark may have neither of the first two.
http_get() {
  local url="$1" timeout="${2:-3}"
  if command -v curl >/dev/null 2>&1; then
    curl -sf --max-time "$timeout" "$url"
  elif command -v python3 >/dev/null 2>&1; then
    python3 -c 'import sys, urllib.request as u; sys.stdout.write(u.urlopen(sys.argv[1], timeout=float(sys.argv[2])).read().decode())' \
      "$url" "$timeout" 2>/dev/null
  else
    http_get_bash "$url" "$timeout"
  fi
}

# HTTP/1.0 GET over bash's /dev/tcp, for the http://host:port/path addresses qc.sh checks.
http_get_bash() {
  local rest="${1#http://}" t="${2:-3}" hostport path host port
  hostport="${rest%%/*}"; path="${rest#"$hostport"}"; [[ -n "$path" ]] || path="/"
  host="${hostport%%:*}"; port=80; [[ "$hostport" == *:* ]] && port="${hostport##*:}"
  (
    exec 3<>"/dev/tcp/$host/$port" || exit 1
    printf 'GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' "$path" "$host" >&3
    IFS= read -r -t "$t" status <&3 || exit 1
    [[ "$status" =~ ^HTTP/[0-9.]+\ 2[0-9][0-9] ]] || exit 1
    while IFS= read -r -t "$t" line <&3; do line="${line%$'\r'}"; [[ -z "$line" ]] && break; done
    while IFS= read -r -t "$t" line <&3 || [[ -n "$line" ]]; do printf '%s\n' "$line"; line=""; done
  ) 2>/dev/null
}

port_busy() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }

free_port() {  # first free port from $1 upwards
  local p="$1"
  while port_busy "$p"; do p=$((p + 1)); done
  say "$p"
}

instance_running() {
  "$APPTAINER" instance list 2>/dev/null | awk 'NR > 1 {print $1}' | grep -qx "$1"
}

instance_log() {  # name out|err -> path of the instance log (Apptainer keeps them per host and user)
  # this host first: a home directory on a network drive holds the logs of every machine; the user
  # directory may be spelt "name" or "name@domain", so match any
  local f
  for f in "$HOME/.apptainer/instances/logs/$(hostname)/"*"/$1.$2" "$HOME/.apptainer/instances/logs/"*/*"/$1.$2"; do
    [[ -f "$f" ]] && { say "$f"; return 0; }
  done
  return 1
}

# Apptainer's working and cache directories. Some installations export APPTAINER_TMPDIR or
# APPTAINER_CACHEDIR for a convention such as /scratch/$USER that this machine does not have; every
# pull then dies with "failed to create build parent dir". Create the directory, or fall back.
APPTAINER_DIRS_READY=0
usable_dir() { mkdir -p "$1" 2>/dev/null && [[ -d "$1" && -w "$1" ]]; }

prepare_apptainer_dirs() {
  [[ "$APPTAINER_DIRS_READY" == 1 ]] && return 0
  local name var legacy dir fallback tmp_free_gb
  for name in TMPDIR CACHEDIR; do
    var="APPTAINER_$name"; legacy="SINGULARITY_$name"
    dir="${!var:-${!legacy:-}}"
    [[ -n "$dir" ]] || continue   # not set: Apptainer's defaults (/tmp, ~/.apptainer/cache) apply
    usable_dir "$dir" && continue
    if [[ "$name" == TMPDIR ]]; then
      # unpacking the cuOpt image needs ~12 GB: local /tmp if it has the room, else our own directory
      tmp_free_gb=$(( $(df -Pk /tmp 2>/dev/null | awk 'NR == 2 {print $4}' || echo 0) / 1024 / 1024 ))
      fallback="$QC_HOME/apptainer-tmp"
      (( tmp_free_gb >= 15 )) && fallback="/tmp/$ME-apptainer-tmp"
    else
      fallback="$QC_HOME/apptainer-cache"
    fi
    usable_dir "$fallback" || die "$var=$dir does not exist, and neither it nor $fallback can be created"
    export "$var=$fallback"
    unset "$legacy"
    if [[ -f "$CONFIG" ]] && ! grep -q "^$var=" "$CONFIG"; then
      printf '\n# set by qc.sh: the value in the environment pointed to a directory that does not exist\n%s\n' \
        "$var=$fallback" >> "$CONFIG"
      warn "$var=$dir (set in your environment) does not exist and cannot be created.
         Using $fallback instead; saved in $CONFIG."
    else
      warn "$var=$dir does not exist and cannot be created; using $fallback for this run."
    fi
  done
  APPTAINER_DIRS_READY=1
}

pull_image() {  # destination source -- download an image to a .sif file, with a useful error
  prepare_apptainer_dirs
  rm -f "$1.part"
  if ! "$APPTAINER" pull "$1.part" "$2"; then
    rm -f "$1.part"
    die "could not download $2. Check that this machine reaches the registry (a proxy needs
       https_proxy set), and that ${APPTAINER_TMPDIR:-/tmp} has ~12 GB free (APPTAINER_TMPDIR chooses another
       directory). Then run the command again."
  fi
  mv -f "$1.part" "$1"
}

show_log_tail() {  # name
  local f
  for kind in err out; do
    if f="$(instance_log "$1" "$kind")"; then
      say "--- last lines of $f"
      tail -n 25 "$f" >&2 || true
    fi
  done
}

wait_for() {  # url seconds label
  local url="$1" limit="$2" label="$3" waited=0
  printf '   waiting for %s ' "$label"
  until http_get "$url" 2 >/dev/null; do
    if (( waited >= limit )); then printf ' timed out after %ss\n' "$limit"; return 1; fi
    printf '.'; sleep 2; waited=$((waited + 2))
  done
  printf ' up (%ss)\n' "$waited"
}

# Hand the settings to a container through APPTAINERENV_* (unlike --env these survive commas,
# e.g. in no_proxy). The containers start with --cleanenv, so nothing else from the host leaks in:
# no PYTHONPATH, no activated virtualenv or conda, no LD_LIBRARY_PATH.
export_quantcore_env() {
  local v
  for v in $(compgen -v | grep '^QUANTCORE_' || true); do
    export "APPTAINERENV_$v=${!v}"
  done
  export APPTAINERENV_QUANTCORE_DATA_DIR=/data
  export APPTAINERENV_QUANTCORE_CUOPT_URL="$CUOPT_URL"
  export APPTAINERENV_QUANTCORE_API_HOST=127.0.0.1
  export APPTAINERENV_QUANTCORE_API_PORT="$QC_API_PORT"
  export APPTAINERENV_QUANTCORE_API_URL="$QC_API_URL"
  # what quantcore.def's %environment sets, for the venv fallback (an explicit value wins)
  export APPTAINERENV_PYTHONNOUSERSITE=1 APPTAINERENV_PYTHONUNBUFFERED=1 APPTAINERENV_PYTHONDONTWRITEBYTECODE=1
  export APPTAINERENV_OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-4}" APPTAINERENV_OMP_NUM_THREADS="${OMP_NUM_THREADS:-4}"
  export APPTAINERENV_MKL_NUM_THREADS="${MKL_NUM_THREADS:-4}"
  for v in http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY TZ; do
    [[ -n "${!v:-}" ]] && export "APPTAINERENV_$v=${!v}"
  done
  return 0
}

# Run a command in the quantcore container (the CLI needs no running API).
qc_binds() {  # bind options for the quantcore container
  local b=(--bind "$QC_DATA:/data")
  [[ "$(qc_mode)" == venv ]] && b+=(--bind "$QC_VENV")
  printf '%s\n' "${b[@]}"
}

run_qc() {  # command [args]: quantcore | quantcore-api | python ...
  need_apptainer
  [[ "$(qc_mode)" != none ]] || die "no quantcore image yet -- run: $0 install"
  mkdir -p "$QC_DATA"
  export_quantcore_env
  local cmd="$1"; shift
  local binds; mapfile -t binds < <(qc_binds)
  exec "$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin "$cmd")" "$@"
}

restore_hint() {  # list earlier data sets next to $QC_DATA (from --rebuild-data) that can be restored
  local d n=0 when stocks
  for d in $(ls -1dt "$QC_DATA".* 2>/dev/null); do
    [[ -f "$d/universe/universe.csv" ]] || continue
    if [[ $n == 0 ]]; then say "   earlier data sets found (newest first):"; fi
    when="$(grep -o '"setup_at": *"[^"]*"' "$d/manifest.json" 2>/dev/null | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/' || true)"
    stocks=$(( $(wc -l < "$d/universe/universe.csv") - 1 ))
    say "     $d   ($stocks stocks, built ${when:-?})"
    n=$((n + 1))
    if [[ $n -ge 4 ]]; then break; fi
  done
  if [[ $n -gt 0 ]]; then say "   restore one with: $0 data-import <folder>"; fi
}

data_is_empty() { [[ -z "$(ls -A "$QC_DATA" 2>/dev/null | grep -v -x -e jobs -e '.*\.log')" ]]; }

# --- install ----------------------------------------------------------------------------------
cmd_install() {
  need_apptainer
  local force=0; [[ "${1:-}" == "--force" ]] && force=1
  step "Apptainer"
  say "   $("$APPTAINER" --version)"
  "$APPTAINER" instance run --help >/dev/null 2>&1 \
    || die "this Apptainer has no 'instance run' (needs Apptainer >= 1.1); ask the administrator to update it"

  step "Directories and settings ($QC_HOME)"
  mkdir -p "$QC_HOME" "$QC_DATA" "$QC_IMAGES"
  if [[ -f "$CONFIG" ]]; then
    say "   keeping $CONFIG"
  else
    cp "$SCRIPT_DIR/quantcore.env.example" "$CONFIG"
    say "   wrote $CONFIG"
  fi

  prepare_apptainer_dirs
  step "GPU"
  if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
    say "   $(nvidia-smi -L | head -n 1)"
  else
    warn "no GPU visible (nvidia-smi). cuOpt will not start; optimisations will use HiGHS on the CPU."
  fi
  local free_gb; free_gb=$(( $(df -Pk "$QC_IMAGES" | awk 'NR == 2 {print $4}') / 1024 / 1024 ))
  say "   free disk under $QC_IMAGES: ${free_gb} GB (the cuOpt pull needs ~12 GB while it runs, 3.3 GB after)"

  if [[ "$QC_CUOPT_START" != "no" && "$CUOPT_EXTERNAL" == 0 ]]; then
    cmd_pull_cuopt $([[ $force == 1 ]] && echo --force)
  fi
  step "quantcore image"
  get_quantcore_image "$force"
  step "Self-test of the quantcore container ($(qc_mode) mode)"
  qc_selftest

  [[ "${QC_FROM_INSTALLER:-0}" == 1 ]] && return 0   # install.sh does the data and the start itself
  step "Data ($QC_DATA)"
  if data_is_empty; then
    say "   empty. Either copy a prepared data set:   $0 data-import quantcore_data.tar.gz"
    say "   or build it from Yahoo Finance (~15 min):  $0 setup"
  else
    say "   present ($(du -sh "$QC_DATA" | cut -f1))"
  fi
  step "Done. Next: $0 start"
}

cmd_pull_cuopt() {
  need_apptainer
  local dst="$QC_IMAGES/cuopt.sif"
  if [[ -f "$QC_CUOPT_IMAGE" ]]; then
    step "cuOpt image: $QC_CUOPT_IMAGE (provided on this machine, not pulled)"
    return 0
  fi
  if [[ -f "$dst" && "${1:-}" != "--force" ]]; then
    step "cuOpt image present: $dst"
    return 0
  fi
  step "Pulling cuOpt: $QC_CUOPT_IMAGE -> $dst (~3.3 GB, several minutes)"
  mkdir -p "$QC_IMAGES"
  pull_image "$dst" "$QC_CUOPT_IMAGE"
}

qc_selftest() {
  # from /: `python -c` puts the working directory first on sys.path, and from the repository root
  # it would import the sources there instead of what was installed into the container
  ( cd / && export_quantcore_env
    local binds; mapfile -t binds < <(qc_binds)
    "$APPTAINER" exec --cleanenv "${binds[@]}" "$(qc_image)" "$(qc_bin python)" -c '
import importlib.resources as r, scipy, quantcore, quantcore.api.app as app
from scipy.optimize import milp
assert (r.files("quantcore.api") / "ui" / "index.html").is_file(), "test bench missing"
print("   quantcore", quantcore.__version__, "| scipy", scipy.__version__, "| UI mounted:", app.UI_ENABLED)' )
}

get_quantcore_image() {  # force(0|1)
  local dst="$QC_IMAGES/quantcore.sif"
  case "$QC_IMAGE" in
    build)
      if [[ -f "$dst" && "$1" != 1 ]]; then say "   present: $dst (rebuild: $0 build)"; return 0; fi
      if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" && "$1" != 1 ]]; then
        say "   present: virtualenv $QC_VENV (no-build fallback; reinstall: $0 build)"; return 0
      fi
      build_sif || {
        say "   -> using the no-build setup instead: a plain Python image plus a virtualenv in $QC_VENV"
        say "      (same libraries, same results)"
        build_venv
      } ;;
    venv)
      if [[ -x "$QC_VENV/bin/quantcore" && -f "$PY_SIF" && "$1" != 1 ]]; then say "   present: $QC_VENV"; else build_venv; fi ;;
    docker://*|oras://*|library://*|http://*|https://*)
      if [[ -f "$dst" && "$1" != 1 ]]; then say "   present: $dst"; else
        say "   pulling $QC_IMAGE -> $dst"
        pull_image "$dst" "$QC_IMAGE"
      fi ;;
    *)
      [[ -f "$QC_IMAGE" ]] || die "QC_IMAGE=$QC_IMAGE is neither 'build', a URL, nor an existing .sif file"
      say "   using $QC_IMAGE" ;;
  esac
}

build_sif() {  # build quantcore.sif from the definition; non-zero when this Apptainer cannot build
  local dst="$QC_IMAGES/quantcore.sif" log="$QC_HOME/build.log" reason
  mkdir -p "$QC_IMAGES"
  say "   building $dst from $REPO_DIR (a few minutes; log: $log)"
  prepare_apptainer_dirs
  rm -f "$dst.part"
  # %files paths in the definition are relative to the repository root. Without root Apptainer
  # builds through --fakeroot (user namespaces); many Sparks do not allow that. The output goes to a
  # log: on such a Spark Apptainer's red ERROR line is expected, and the caller falls back.
  if ! (cd "$REPO_DIR" && "$APPTAINER" build "$dst.part" deploy/apptainer/quantcore.def) >"$log" 2>&1; then
    rm -f "$dst.part"
    reason="$(grep -E 'FATAL|ERROR' "$log" | tail -n 1 | sed -e 's/\x1b\[[0-9;]*m//g' -e 's/^[A-Z]* *: *//')"
    say "   no image build on this machine${reason:+ ($reason)}"
    return 1
  fi
  mv -f "$dst.part" "$dst"
  say "   built $dst ($(du -h "$dst" | cut -f1))"
}

build_venv() {  # the no-build fallback: pull a plain Python image, install quantcore into a virtualenv
  mkdir -p "$QC_IMAGES" "$QC_HOME"
  if [[ ! -f "$PY_SIF" ]]; then
    say "   pulling $QC_PYTHON_IMAGE -> $PY_SIF"
    pull_image "$PY_SIF" "$QC_PYTHON_IMAGE"
  fi
  say "   installing quantcore into $QC_VENV (pinned versions from deploy/apptainer/constraints.txt)"
  # a copy of the sources under QC_HOME (not $TMPDIR, which may point nowhere on a managed machine)
  local work; work="$(mktemp -d "$QC_HOME/.quantcore-src.XXXXXX")"
  # install from a copy: pip writes build/ and *.egg-info next to the sources
  cp -r "$REPO_DIR/pyproject.toml" "$REPO_DIR/README.md" "$REPO_DIR/quantcore" "$work/"
  "$APPTAINER" exec --cleanenv --bind "$QC_HOME" --bind "$work" "$PY_SIF" python -m venv --clear "$QC_VENV"
  "$APPTAINER" exec --cleanenv --env PIP_DISABLE_PIP_VERSION_CHECK=1 --bind "$QC_HOME" --bind "$work" \
    --bind "$REPO_DIR/deploy/apptainer/constraints.txt:/tmp/qc-constraints.txt" "$PY_SIF" \
    "$QC_VENV/bin/pip" install --quiet --no-cache-dir -c /tmp/qc-constraints.txt "$work"
  rm -rf "$work"
  ( cd / && "$APPTAINER" exec --cleanenv --bind "$QC_VENV" "$PY_SIF" "$QC_VENV/bin/quantcore" --help >/dev/null ) \
    || die "quantcore was installed into $QC_VENV but does not start (see the error above)"
  say "   installed"
}

cmd_build() {  # rebuild after a code change, in the mode in use
  need_apptainer
  local mode; mode="$(qc_mode)"
  step "Rebuilding quantcore ($([[ $mode == venv ]] && echo "virtualenv $QC_VENV" || echo "image $QC_IMAGES/quantcore.sif"))"
  if [[ "$mode" == venv ]]; then
    build_venv
  elif [[ "$QC_IMAGE" == build ]]; then
    build_sif || die "the image build failed (log: $QC_HOME/build.log). Without root it needs Apptainer's
       --fakeroot. Use the no-build setup instead: set QC_IMAGE=venv in $CONFIG and run: $0 build"
  else
    die "QC_IMAGE=$QC_IMAGE is a prebuilt image; nothing to build"
  fi
  if instance_running "$API_INSTANCE"; then say "   the running API still uses the old code: $0 restart"; fi
}

# --- run --------------------------------------------------------------------------------------
start_cuopt() {
  if [[ "$CUOPT_EXTERNAL" == 1 ]]; then
    if http_get "$CUOPT_URL/cuopt/health" >/dev/null; then say "cuOpt: using $CUOPT_URL (QUANTCORE_CUOPT_URL)"
    else warn "cuOpt at $CUOPT_URL (QUANTCORE_CUOPT_URL) does not answer -- optimisations will use HiGHS"; fi
    return 0
  fi
  if [[ "$QC_CUOPT_START" == "no" ]]; then
    say "cuOpt: not started (QC_CUOPT_START=no) -- optimisations use HiGHS on the CPU"
    return 0
  fi
  if http_get "$CUOPT_URL/cuopt/health" >/dev/null; then
    if instance_running "$CUOPT_INSTANCE"; then
      say "cuOpt: running ($CUOPT_INSTANCE, $CUOPT_URL)"
    elif [[ "$QC_CUOPT_START" == "yes" ]]; then
      die "QC_CUOPT_START=yes, but another cuOpt already serves port $QC_CUOPT_PORT. Use it (QC_CUOPT_START=auto)
       or pick another port: QC_CUOPT_PORT=$(free_port $((QC_CUOPT_PORT + 100))) in $CONFIG"
    else
      say "cuOpt: using the server that already answers at $CUOPT_URL"
    fi
    return 0
  fi
  if instance_running "$CUOPT_INSTANCE"; then   # still starting, or hung
    wait_for "$CUOPT_URL/cuopt/health" 180 "cuOpt ($CUOPT_INSTANCE)" && return 0
    show_log_tail "$CUOPT_INSTANCE"
    die "cuOpt instance runs but does not answer; try: $0 restart"
  fi
  if port_busy "$QC_CUOPT_PORT"; then
    die "port $QC_CUOPT_PORT is taken by something that is not cuOpt.
       Set QC_CUOPT_PORT=$(free_port $((QC_CUOPT_PORT + 100))) in $CONFIG and run: $0 start"
  fi
  local sif; sif="$(cuopt_sif)"
  need_image "$sif" cuOpt
  say "cuOpt: starting instance $CUOPT_INSTANCE on 127.0.0.1:$QC_CUOPT_PORT (GPU)"
  # NVIDIA's entrypoint sets HOME=/opt/cuopt. That is writable under Docker but read-only in a SIF,
  # and the solver process dies on its first job when CuPy cannot write its kernel cache there.
  # So the server runs with HOME on a writable directory of ours (CuPy, Numba and CUDA caches).
  local cuopt_home="$QC_HOME/cuopt-home"
  mkdir -p "$cuopt_home"
  ( [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]] && export APPTAINERENV_CUDA_VISIBLE_DEVICES="$CUDA_VISIBLE_DEVICES"
    cd /
    # the image's entrypoint runs its arguments: the cuOpt REST server, on localhost only
    "$APPTAINER" instance run --nv --cleanenv --bind "$cuopt_home" "$sif" "$CUOPT_INSTANCE" \
      env HOME="$cuopt_home" python -m cuopt_server.cuopt_service \
        -i 127.0.0.1 -p "$QC_CUOPT_PORT" -l "${QC_CUOPT_LOG_LEVEL:-info}" >/dev/null )
  if ! wait_for "$CUOPT_URL/cuopt/health" 180 "cuOpt"; then
    show_log_tail "$CUOPT_INSTANCE"
    warn "cuOpt did not come up -- the API starts anyway and optimises with HiGHS"
  fi
}

start_api() {
  if instance_running "$API_INSTANCE"; then
    if http_get "$QC_API_URL/health" >/dev/null; then say "API: running ($API_INSTANCE, $QC_API_URL)"; return 0; fi
    wait_for "$QC_API_URL/health" 60 "API ($API_INSTANCE)" && return 0
    show_log_tail "$API_INSTANCE"
    die "the API instance runs but does not answer; try: $0 restart"
  fi
  if port_busy "$QC_API_PORT"; then
    die "port $QC_API_PORT is taken (another user's quantcore?).
       Set QC_API_PORT=$(free_port $((QC_API_PORT + 10))) in $CONFIG and run: $0 start"
  fi
  local mode; mode="$(qc_mode)"
  [[ "$mode" != none ]] || die "no quantcore image yet -- run: $0 install"
  mkdir -p "$QC_DATA"
  say "API: starting instance $API_INSTANCE on $QC_API_URL ($mode mode)"
  ( export_quantcore_env
    local binds; mapfile -t binds < <(qc_binds)
    # start from /: pydantic-settings reads a .env from the working directory, and the container's
    # / has none (instances take the caller's directory; they have no --cwd option)
    cd /
    if [[ "$mode" == venv ]]; then   # the Python image's runscript executes its arguments
      "$APPTAINER" instance run --cleanenv "${binds[@]}" "$PY_SIF" "$API_INSTANCE" "$QC_VENV/bin/quantcore" serve >/dev/null
    else                              # quantcore.def's startscript: quantcore serve
      "$APPTAINER" instance start --cleanenv "${binds[@]}" "$(quantcore_sif)" "$API_INSTANCE" >/dev/null
    fi )
  if ! wait_for "$QC_API_URL/health" 90 "API"; then
    show_log_tail "$API_INSTANCE"
    die "the API did not come up"
  fi
}

cmd_start() {
  need_apptainer
  step "Starting quantcore (user $ME, settings $([[ -f "$CONFIG" ]] && echo "$CONFIG" || echo "defaults"))"
  start_cuopt
  start_api
  if data_is_empty; then
    warn "no market data in $QC_DATA -- the API answers 503 until there is some."
    restore_hint
    say "   build it (~10 min): $0 setup --offline     or import a prepared one: $0 data-import FILE"
  fi
  local ui=""; [[ "${QUANTCORE_ENABLE_UI:-true}" != "false" ]] && ui="$QC_API_URL/ui/   (test bench)"
  step "Ready -- on this Spark:"
  [[ -n "$ui" ]] && say "   $ui"
  say "   $QC_API_URL/docs   (API, Swagger)"
  say "   Python: QuantcoreClient(\"$QC_API_URL\")    CLI: $0 api health"
}

cmd_stop() {
  need_apptainer
  local name stopped=0
  for name in "$API_INSTANCE" "$CUOPT_INSTANCE"; do
    if instance_running "$name"; then
      stop_instance "$name"; say "stopped $name"; stopped=1
    fi
  done
  [[ $stopped == 1 ]] || say "nothing running"
}

cmd_status() {
  need_apptainer
  step "Instances of $ME"
  "$APPTAINER" instance list
  step "cuOpt $CUOPT_URL"
  local h
  if h="$(http_get "$CUOPT_URL/cuopt/health")"; then say "   up: $h"; else say "   not answering"; fi
  step "API $QC_API_URL"
  if h="$(http_get "$QC_API_URL/health")"; then
    local api_setup host_setup
    # `|| true`: under pipefail a grep without a match would end the script silently
    api_setup="$(printf '%s' "$h" | grep -o '"setup_at":"[^"]*"' | head -n 1 || true)"
    host_setup="$(grep -o '"setup_at": *"[^"]*"' "$QC_DATA/manifest.json" 2>/dev/null | head -n 1 | tr -d ' ' || true)"
    if [[ "$api_setup" != "$host_setup" ]]; then
      warn "the running API serves a different data set than $QC_DATA (API ${api_setup:-none}, folder ${host_setup:-none}).
         Run: $0 restart"
    fi
    if command -v python3 >/dev/null 2>&1; then
      printf '%s' "$h" | python3 -c '
import json, sys
h = json.load(sys.stdin); s = h.get("store", {}); c = h.get("cuopt", {})
print("   up: version", h.get("version"), "| base currency", h.get("base_currency"))
print("   data:", {k: s[k] for k in list(s)[:6]})
print("   cuOpt seen by the API:", "reachable" if c.get("reachable") else "NOT reachable -> HiGHS")'
    else
      say "   up: $h"
    fi
  else
    say "   not answering"
  fi
  step "Files"
  say "   settings: $CONFIG $([[ -f "$CONFIG" ]] || echo '(missing: defaults in use)')"
  say "   data:     $QC_DATA ($(du -sh "$QC_DATA" 2>/dev/null | cut -f1 || echo -))"
  data_is_empty && restore_hint
  if grep -q ',wikipedia_stoxx600,' "$QC_DATA/universe/universe.csv" 2>/dev/null; then
    say "   NOTE: this data set comes from an earlier version's guessed European symbols and misses many"
    say "         companies. Build it anew (~10 min): $REPO_DIR/install.sh --rebuild-data"
  fi
  say "   quantcore: $(qc_mode) mode, $(qc_image)$([[ $(qc_mode) == venv ]] && echo " + $QC_VENV")"
  say "   cuOpt:     $(cuopt_sif)"
}

cmd_logs() {
  local which="${1:-api}" follow=""
  [[ "${2:-}" == "-f" || "${1:-}" == "-f" ]] && follow=1
  [[ "$which" == "-f" ]] && which=api
  local name="$API_INSTANCE"; [[ "$which" == "cuopt" ]] && name="$CUOPT_INSTANCE"
  local out err
  out="$(instance_log "$name" out || true)"; err="$(instance_log "$name" err || true)"
  [[ -n "$out$err" ]] || die "no logs for $name yet"
  if [[ -n "$follow" ]]; then exec tail -n 50 -F ${out:+"$out"} ${err:+"$err"}; fi
  tail -n 50 ${out:+"$out"} ${err:+"$err"}
}

# --- data -------------------------------------------------------------------------------------
data_job() {  # setup|update [args]
  # Always run in a fresh container on $QC_DATA, never inside the running API: the API holds the
  # directory it was started with, and if that is no longer $QC_DATA its writes would end up somewhere
  # the CLI never looks. A running API is restarted afterwards so that it serves the new data; for a
  # full setup it is stopped first (setup rewrites every file it reads).
  local job="$1"; shift
  local api_was_running=0 rc=0
  instance_running "$API_INSTANCE" && api_was_running=1
  if [[ "$job" == setup && "$api_was_running" == 1 ]]; then
    say "stopping the API while the data set is rebuilt"
    stop_instance "$API_INSTANCE"
  fi
  ( run_qc quantcore "$job" "$@" ) || rc=$?
  if [[ "$api_was_running" == 1 ]]; then
    say "restarting the API so that it serves the new data"
    restart_api
  fi
  return "$rc"
}

stop_instance() {  # name -- stop and wait until it is gone
  local i
  instance_running "$1" || return 0
  "$APPTAINER" instance stop "$1" >/dev/null 2>&1 || "$APPTAINER" instance stop --force "$1" >/dev/null 2>&1 || true
  for i in $(seq 1 30); do instance_running "$1" || return 0; sleep 1; done
  die "could not stop the instance $1 (apptainer instance stop --force $1)"
}

restart_api() {
  stop_instance "$API_INSTANCE"
  local i
  for i in $(seq 1 15); do port_busy "$QC_API_PORT" || break; sleep 1; done
  start_api
}

cmd_data_import() {
  local src="${1:-}" force="${2:-}"
  [[ -n "$src" ]] || die "usage: $0 data-import <quantcore_data.tar.gz | directory> [--force]"
  [[ -e "$src" ]] || die "$src not found"
  mkdir -p "$QC_DATA"
  if ! data_is_empty && [[ "$force" != "--force" ]]; then
    die "$QC_DATA already holds data; add --force to overwrite it"
  fi
  if [[ -d "$src" ]]; then
    cp -a "$src/." "$QC_DATA/"
  else
    tar -xzf "$src" -C "$QC_DATA"
  fi
  say "imported into $QC_DATA ($(du -sh "$QC_DATA" | cut -f1))"
  if instance_running "$API_INSTANCE"; then say "restarting the API so that it serves the imported data"; restart_api; fi
}

cmd_data_export() {
  local dst="${1:-quantcore_data.tar.gz}"
  [[ "$dst" = /* ]] || dst="$PWD/$dst"
  tar -czf "$dst" -C "$QC_DATA" --exclude './jobs' --exclude '*.log' .
  say "wrote $dst ($(du -h "$dst" | cut -f1))"
}

cmd_upgrade() {  # newest code from the repository, rebuild, restart -- install.sh does all of it
  [[ -f "$REPO_DIR/install.sh" ]] || die "$REPO_DIR/install.sh missing; download the code again"
  exec bash "$REPO_DIR/install.sh" --dir "$REPO_DIR" --no-data "$@"
}

cmd_env() {
  cat <<EOF
QC_HOME=$QC_HOME
config=$CONFIG $([[ -f "$CONFIG" ]] || echo '(missing)')
QC_DATA=$QC_DATA
QC_IMAGES=$QC_IMAGES
quantcore=$(qc_mode) mode: $(qc_image)   (QC_IMAGE=$QC_IMAGE)
cuOpt image=$(cuopt_sif)   (QC_CUOPT_IMAGE=$QC_CUOPT_IMAGE)
API=$QC_API_URL
cuOpt=$CUOPT_URL   (QC_CUOPT_START=$QC_CUOPT_START)
EOF
  compgen -v | grep '^QUANTCORE_' | while read -r v; do say "$v=${!v}"; done
}

usage() {
  cat <<EOF
quantcore on this DGX Spark (Apptainer, localhost only).  Settings: $CONFIG

Setup
  install [--force]          config, cuOpt image (pull), quantcore image (build or prebuilt), checks
  build                      rebuild quantcore from this repository (after a code change / git pull)
  pull-cuopt [--force]       download the cuOpt image again
  data-import SRC [--force]  copy a prepared data set (.tar.gz or directory) into $QC_DATA
  data-export [FILE]         pack the data directory (default quantcore_data.tar.gz)
  setup [--offline]          build the data set from Yahoo Finance (~15 min, needs internet)
  update [--full]            fetch the newest prices / FX
  upgrade                    newest code from the repository, rebuild, restart (runs install.sh)
Run
  start | stop | restart     cuOpt + API instances
  status                     instances, health, data, images
  logs [api|cuopt] [-f]      instance logs
Use
  cli COMMAND ...            quantcore CLI in the container: status, risk, performance, optimize, snapshot ...
  api COMMAND ...            quantcore-api against $QC_API_URL: health, selftest, optimize ...
  python [ARGS]              Python with quantcore installed (QUANTCORE_* set as for the API)
  shell                      a shell in the quantcore container
  env                        the effective settings
  clean-cache                remove Apptainer's download cache (layers of pulled images)
EOF
}

main() {
  local cmd="${1:-help}"; [[ $# -gt 0 ]] && shift
  case "$cmd" in
    install)      cmd_install "$@" ;;
    build)        cmd_build ;;
    pull-cuopt)   cmd_pull_cuopt "$@" ;;
    start)        cmd_start ;;
    stop)         cmd_stop ;;
    restart)      cmd_stop; cmd_start ;;
    status)       cmd_status ;;
    logs)         cmd_logs "$@" ;;
    cli)          run_qc quantcore "$@" ;;
    api)          run_qc quantcore-api --url "$QC_API_URL" "$@" ;;
    python)       run_qc python "$@" ;;
    shell)        need_apptainer; export_quantcore_env; mapfile -t binds < <(qc_binds)
                  exec "$APPTAINER" shell --cleanenv "${binds[@]}" "$(qc_image)" ;;
    setup|update) data_job "$cmd" "$@" ;;
    data-import)  cmd_data_import "$@" ;;
    data-export)  cmd_data_export "$@" ;;
    has-data)     ! data_is_empty ;;
    is-running)   need_apptainer; instance_running "$API_INSTANCE" || instance_running "$CUOPT_INSTANCE" ;;
    upgrade)      cmd_upgrade "$@" ;;
    env)          cmd_env ;;
    clean-cache)  need_apptainer; "$APPTAINER" cache clean --force ;;
    help|-h|--help) usage ;;
    *)            usage; die "unknown command: $cmd" ;;
  esac
}

main "$@"