Fase 5: servicio de diagnostico vllm-eval-nvfp4-nospec (puerto 8003)
Identico a vllm-eval-nvfp4 pero sin --speculative-config, para aislar si la regresion de calidad observada en las puertas 2-3 (vs. Fase 4) viene del speculative decoding (MTP) o de la cuantizacion NVFP4 en si -- decision explicita del usuario antes de invertir tiempo en recalibrar con mas muestras de calibracion.
This commit is contained in:
@@ -80,3 +80,56 @@ services:
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 600s
|
||||
|
||||
# Fase 5 -- diagnostico de aislamiento: identico a vllm-eval-nvfp4 pero SIN
|
||||
# --speculative-config, para determinar si la regresion de calidad observada
|
||||
# en las puertas 2-3 (vs. Fase 4) viene del speculative decoding (MTP) o de
|
||||
# la cuantizacion NVFP4 en si. Puerto 8003 (distinto de 8000 produccion, 8001
|
||||
# Fase 4, 8002 Fase 5 con speculative real). Servicio temporal de diagnostico,
|
||||
# no clona produccion 1:1 a proposito (esa es la variable que se esta aislando).
|
||||
vllm-eval-nvfp4-nospec:
|
||||
image: vllm/vllm-openai:cu130-nightly-aarch64
|
||||
container_name: vllm-eval-nvfp4-nospec
|
||||
restart: "no"
|
||||
runtime: nvidia
|
||||
environment:
|
||||
NVIDIA_VISIBLE_DEVICES: all
|
||||
NVIDIA_DRIVER_CAPABILITIES: compute,utility
|
||||
ports:
|
||||
- "8003:8000"
|
||||
ipc: host
|
||||
ulimits:
|
||||
memlock: -1
|
||||
stack: 67108864
|
||||
volumes:
|
||||
- /home/aleleba/ft-models/Qwen3.6-35B-A3B-mcp-NVFP4:/models/Qwen3.6-35B-A3B-mcp-NVFP4:ro
|
||||
command: >
|
||||
--model /models/Qwen3.6-35B-A3B-mcp-NVFP4
|
||||
--served-model-name qwen3.6-35b-a3b-mcp-nvfp4-nospec
|
||||
--host 0.0.0.0
|
||||
--port 8000
|
||||
--tensor-parallel-size 1
|
||||
--trust-remote-code
|
||||
--quantization compressed-tensors
|
||||
--moe-backend flashinfer_cutlass
|
||||
--kv-cache-dtype fp8_e4m3
|
||||
--gpu-memory-utilization 0.45
|
||||
--max-model-len 524288
|
||||
--max-num-seqs 8
|
||||
--max-num-batched-tokens 32768
|
||||
--enable-chunked-prefill
|
||||
--enable-prefix-caching
|
||||
--reasoning-parser qwen3
|
||||
--tool-call-parser qwen3_coder
|
||||
--enable-auto-tool-choice
|
||||
--default-chat-template-kwargs '{"preserve_thinking":true}'
|
||||
--limit-mm-per-prompt '{"image":4,"video":0,"audio":0}'
|
||||
--generation-config vllm
|
||||
--override-generation-config '{"temperature":0.6,"top_p":0.80,"top_k":20,"presence_penalty":0.0,"repetition_penalty":1.0}'
|
||||
--hf-overrides '{"text_config":{"rope_scaling":{"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":262144}}}'
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 600s
|
||||
|
||||
Reference in New Issue
Block a user