Default 122B server to 128k context, add health-check script
- 11_start_server_122b.sh: raise MAX_MODEL_LEN default to 131072 (128k), make max-num-seqs overridable via MAX_NUM_SEQS (default 4 for low concurrency / large KV cache). Echo concurrency in startup banner. - 13_check_server.sh: new health check that polls /v1/models until ready, then sends a properly-sized test prompt and reports OK/failure. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
herzogflorian
parent
030d9f7935
commit
b9eaf2df18
@ -13,10 +13,13 @@
|
||||
# Environment variables (override defaults):
|
||||
# MODEL_DIR - Path to model weights (default: ~/models/Qwen3.5-122B-A10B-FP8)
|
||||
# PORT - Server port (default: 7080)
|
||||
# MAX_MODEL_LEN - Maximum context length (default: 32768)
|
||||
# MAX_MODEL_LEN - Maximum context length (default: 131072 = 128k)
|
||||
# GPU_MEM_UTIL - GPU memory utilization fraction (default: 0.92)
|
||||
# API_KEY - API key for authentication (default: none)
|
||||
# TENSOR_PARALLEL - Number of GPUs for tensor parallelism (default: 4)
|
||||
# MAX_NUM_SEQS - Max concurrent sequences (default: 4). Raise this
|
||||
# when more users need it AND the KV cache fits;
|
||||
# lower it if 128k context runs out of memory.
|
||||
# ------------------------------------------------------------------
|
||||
set -euo pipefail
|
||||
|
||||
@ -25,10 +28,11 @@ SIF_FILE="${SCRIPT_DIR}/vllm_qwen.sif"
|
||||
|
||||
MODEL_DIR="${MODEL_DIR:-$HOME/models/Qwen3.5-122B-A10B-FP8}"
|
||||
PORT="${PORT:-7080}"
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||||
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.92}"
|
||||
API_KEY="${API_KEY:-}"
|
||||
TENSOR_PARALLEL="${TENSOR_PARALLEL:-4}"
|
||||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
|
||||
|
||||
if [ ! -f "$SIF_FILE" ]; then
|
||||
echo "ERROR: Container image not found at ${SIF_FILE}"
|
||||
@ -56,7 +60,7 @@ VLLM_ARGS=(
|
||||
--trust-remote-code
|
||||
--reasoning-parser qwen3
|
||||
--served-model-name "qwen3.5-122b-a10b-fp8"
|
||||
--max-num-seqs 16
|
||||
--max-num-seqs "$MAX_NUM_SEQS"
|
||||
--enable-prefix-caching
|
||||
)
|
||||
|
||||
@ -73,6 +77,7 @@ echo " Port: ${PORT}"
|
||||
echo " Context len: ${MAX_MODEL_LEN}"
|
||||
echo " GPU util: ${GPU_MEM_UTIL}"
|
||||
echo " TP size: ${TENSOR_PARALLEL}"
|
||||
echo " Max seqs: ${MAX_NUM_SEQS}"
|
||||
echo " dtype: auto (FP8)"
|
||||
echo " API key: ${API_KEY:-<none>}"
|
||||
echo "=============================================="
|
||||
|
||||
128
13_check_server.sh
Executable file
128
13_check_server.sh
Executable file
@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env bash
|
||||
# ------------------------------------------------------------------
|
||||
# 13_check_server.sh
|
||||
# Health-checks the vLLM server: polls /v1/models until the model is
|
||||
# ready, then sends a properly-sized test prompt and prints the reply.
|
||||
#
|
||||
# Works whether you run it ON the server (localhost) or from another
|
||||
# machine (use HOST=silicon.fhgr.ch).
|
||||
#
|
||||
# Usage:
|
||||
# bash 13_check_server.sh
|
||||
# HOST=silicon.fhgr.ch bash 13_check_server.sh
|
||||
#
|
||||
# Environment variables (override defaults):
|
||||
# HOST - Server hostname (default: localhost)
|
||||
# PORT - Server port (default: 7080)
|
||||
# API_KEY - API key, if the server needs one (default: none)
|
||||
# TIMEOUT - Seconds to wait for model (default: 600)
|
||||
# MAX_TOKENS - Tokens for the test prompt (default: 512)
|
||||
#
|
||||
# NOTE: The model is a reasoning model — it spends tokens "thinking"
|
||||
# before the final answer. MAX_TOKENS must be generous (>=256),
|
||||
# or the reply ends up empty with finish_reason "length".
|
||||
# ------------------------------------------------------------------
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${HOST:-localhost}"
|
||||
PORT="${PORT:-7080}"
|
||||
API_KEY="${API_KEY:-}"
|
||||
TIMEOUT="${TIMEOUT:-600}"
|
||||
MAX_TOKENS="${MAX_TOKENS:-512}"
|
||||
|
||||
BASE_URL="http://${HOST}:${PORT}/v1"
|
||||
|
||||
AUTH_ARGS=()
|
||||
if [ -n "$API_KEY" ]; then
|
||||
AUTH_ARGS=(-H "Authorization: Bearer ${API_KEY}")
|
||||
fi
|
||||
|
||||
echo "=============================================="
|
||||
echo " vLLM Server Health Check"
|
||||
echo "=============================================="
|
||||
echo " Endpoint: ${BASE_URL}"
|
||||
echo " Timeout: ${TIMEOUT}s"
|
||||
echo " Max tokens: ${MAX_TOKENS}"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
|
||||
# --- Step 1: poll /v1/models until ready -------------------------
|
||||
echo "Waiting for server to respond at ${BASE_URL}/models ..."
|
||||
DEADLINE=$(( $(date +%s) + TIMEOUT ))
|
||||
MODELS_JSON=""
|
||||
while true; do
|
||||
if MODELS_JSON=$(curl -s -m 5 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} "${BASE_URL}/models" 2>/dev/null) \
|
||||
&& echo "$MODELS_JSON" | grep -q '"data"'; then
|
||||
break
|
||||
fi
|
||||
if [ "$(date +%s)" -ge "$DEADLINE" ]; then
|
||||
echo "ERROR: Server did not become ready within ${TIMEOUT}s."
|
||||
echo " Check the log: tail -f logs/vllm_server_122b_*.log"
|
||||
exit 1
|
||||
fi
|
||||
printf '.'
|
||||
sleep 5
|
||||
done
|
||||
echo ""
|
||||
|
||||
# Extract the served model id (first entry under data[]).
|
||||
MODEL_ID=$(echo "$MODELS_JSON" | python3 -c \
|
||||
"import sys, json; print(json.load(sys.stdin)['data'][0]['id'])" 2>/dev/null || true)
|
||||
|
||||
if [ -z "$MODEL_ID" ]; then
|
||||
echo "ERROR: Could not parse model id from /v1/models response:"
|
||||
echo "$MODELS_JSON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Server is up. Serving model: ${MODEL_ID}"
|
||||
echo ""
|
||||
|
||||
# --- Step 2: send a real generation request ----------------------
|
||||
echo "Sending test prompt..."
|
||||
REQUEST=$(python3 -c "
|
||||
import json, sys
|
||||
print(json.dumps({
|
||||
'model': sys.argv[1],
|
||||
'messages': [{'role': 'user', 'content': 'Reply with exactly one word: hello'}],
|
||||
'max_tokens': int(sys.argv[2]),
|
||||
}))
|
||||
" "$MODEL_ID" "$MAX_TOKENS")
|
||||
|
||||
RESPONSE=$(curl -s -m 120 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$REQUEST" \
|
||||
"${BASE_URL}/chat/completions")
|
||||
|
||||
echo ""
|
||||
echo "$RESPONSE" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
except Exception:
|
||||
print('ERROR: Non-JSON response from server:')
|
||||
print(sys.stdin.read() if not sys.stdin.closed else '<empty>')
|
||||
sys.exit(1)
|
||||
|
||||
choice = d['choices'][0]
|
||||
msg = choice.get('message', {})
|
||||
content = msg.get('content')
|
||||
finish = choice.get('finish_reason')
|
||||
usage = d.get('usage', {})
|
||||
|
||||
print('Finish reason: %s' % finish)
|
||||
print('Usage: %s' % usage)
|
||||
print('Content: %r' % content)
|
||||
print()
|
||||
|
||||
if content and content.strip():
|
||||
print('RESULT: OK — model generated a reply. Server is working.')
|
||||
elif finish == 'length':
|
||||
print('RESULT: Model hit the token limit while reasoning and never')
|
||||
print(' produced an answer. Re-run with a larger budget, e.g.:')
|
||||
print(' MAX_TOKENS=1024 bash 13_check_server.sh')
|
||||
sys.exit(2)
|
||||
else:
|
||||
print('RESULT: Empty content with finish_reason=%r. Investigate.' % finish)
|
||||
sys.exit(2)
|
||||
"
|
||||
Loading…
x
Reference in New Issue
Block a user