b9eaf2df18
- 11_start_server_122b.sh: raise MAX_MODEL_LEN default to 131072 (128k), make max-num-seqs overridable via MAX_NUM_SEQS (default 4 for low concurrency / large KV cache). Echo concurrency in startup banner. - 13_check_server.sh: new health check that polls /v1/models until ready, then sends a properly-sized test prompt and reports OK/failure. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
129 lines
4.1 KiB
Bash
Executable File
129 lines
4.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ------------------------------------------------------------------
|
|
# 13_check_server.sh
|
|
# Health-checks the vLLM server: polls /v1/models until the model is
|
|
# ready, then sends a properly-sized test prompt and prints the reply.
|
|
#
|
|
# Works whether you run it ON the server (localhost) or from another
|
|
# machine (use HOST=silicon.fhgr.ch).
|
|
#
|
|
# Usage:
|
|
# bash 13_check_server.sh
|
|
# HOST=silicon.fhgr.ch bash 13_check_server.sh
|
|
#
|
|
# Environment variables (override defaults):
|
|
# HOST - Server hostname (default: localhost)
|
|
# PORT - Server port (default: 7080)
|
|
# API_KEY - API key, if the server needs one (default: none)
|
|
# TIMEOUT - Seconds to wait for model (default: 600)
|
|
# MAX_TOKENS - Tokens for the test prompt (default: 512)
|
|
#
|
|
# NOTE: The model is a reasoning model — it spends tokens "thinking"
|
|
# before the final answer. MAX_TOKENS must be generous (>=256),
|
|
# or the reply ends up empty with finish_reason "length".
|
|
# ------------------------------------------------------------------
|
|
set -euo pipefail
|
|
|
|
HOST="${HOST:-localhost}"
|
|
PORT="${PORT:-7080}"
|
|
API_KEY="${API_KEY:-}"
|
|
TIMEOUT="${TIMEOUT:-600}"
|
|
MAX_TOKENS="${MAX_TOKENS:-512}"
|
|
|
|
BASE_URL="http://${HOST}:${PORT}/v1"
|
|
|
|
AUTH_ARGS=()
|
|
if [ -n "$API_KEY" ]; then
|
|
AUTH_ARGS=(-H "Authorization: Bearer ${API_KEY}")
|
|
fi
|
|
|
|
echo "=============================================="
|
|
echo " vLLM Server Health Check"
|
|
echo "=============================================="
|
|
echo " Endpoint: ${BASE_URL}"
|
|
echo " Timeout: ${TIMEOUT}s"
|
|
echo " Max tokens: ${MAX_TOKENS}"
|
|
echo "=============================================="
|
|
echo ""
|
|
|
|
# --- Step 1: poll /v1/models until ready -------------------------
|
|
echo "Waiting for server to respond at ${BASE_URL}/models ..."
|
|
DEADLINE=$(( $(date +%s) + TIMEOUT ))
|
|
MODELS_JSON=""
|
|
while true; do
|
|
if MODELS_JSON=$(curl -s -m 5 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} "${BASE_URL}/models" 2>/dev/null) \
|
|
&& echo "$MODELS_JSON" | grep -q '"data"'; then
|
|
break
|
|
fi
|
|
if [ "$(date +%s)" -ge "$DEADLINE" ]; then
|
|
echo "ERROR: Server did not become ready within ${TIMEOUT}s."
|
|
echo " Check the log: tail -f logs/vllm_server_122b_*.log"
|
|
exit 1
|
|
fi
|
|
printf '.'
|
|
sleep 5
|
|
done
|
|
echo ""
|
|
|
|
# Extract the served model id (first entry under data[]).
|
|
MODEL_ID=$(echo "$MODELS_JSON" | python3 -c \
|
|
"import sys, json; print(json.load(sys.stdin)['data'][0]['id'])" 2>/dev/null || true)
|
|
|
|
if [ -z "$MODEL_ID" ]; then
|
|
echo "ERROR: Could not parse model id from /v1/models response:"
|
|
echo "$MODELS_JSON"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Server is up. Serving model: ${MODEL_ID}"
|
|
echo ""
|
|
|
|
# --- Step 2: send a real generation request ----------------------
|
|
echo "Sending test prompt..."
|
|
REQUEST=$(python3 -c "
|
|
import json, sys
|
|
print(json.dumps({
|
|
'model': sys.argv[1],
|
|
'messages': [{'role': 'user', 'content': 'Reply with exactly one word: hello'}],
|
|
'max_tokens': int(sys.argv[2]),
|
|
}))
|
|
" "$MODEL_ID" "$MAX_TOKENS")
|
|
|
|
RESPONSE=$(curl -s -m 120 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} \
|
|
-H "Content-Type: application/json" \
|
|
-d "$REQUEST" \
|
|
"${BASE_URL}/chat/completions")
|
|
|
|
echo ""
|
|
echo "$RESPONSE" | python3 -c "
|
|
import sys, json
|
|
try:
|
|
d = json.load(sys.stdin)
|
|
except Exception:
|
|
print('ERROR: Non-JSON response from server:')
|
|
print(sys.stdin.read() if not sys.stdin.closed else '<empty>')
|
|
sys.exit(1)
|
|
|
|
choice = d['choices'][0]
|
|
msg = choice.get('message', {})
|
|
content = msg.get('content')
|
|
finish = choice.get('finish_reason')
|
|
usage = d.get('usage', {})
|
|
|
|
print('Finish reason: %s' % finish)
|
|
print('Usage: %s' % usage)
|
|
print('Content: %r' % content)
|
|
print()
|
|
|
|
if content and content.strip():
|
|
print('RESULT: OK — model generated a reply. Server is working.')
|
|
elif finish == 'length':
|
|
print('RESULT: Model hit the token limit while reasoning and never')
|
|
print(' produced an answer. Re-run with a larger budget, e.g.:')
|
|
print(' MAX_TOKENS=1024 bash 13_check_server.sh')
|
|
sys.exit(2)
|
|
else:
|
|
print('RESULT: Empty content with finish_reason=%r. Investigate.' % finish)
|
|
sys.exit(2)
|
|
"
|