# Kimi K2.6 DFlash Production Configuration
# 507 tok/s on 8x AMD Instinct MI300X (gfx942)
#
# Prerequisites:
#   - NUMA balancing disabled: echo 0 > /proc/sys/kernel/numa_balancing
#   - Docker with ROCm support
#   - vllm/vllm-openai-rocm:nightly image
#   - Model: moonshotai/Kimi-K2.6 on local NVMe
#   - Draft: z-lab/Kimi-K2.5-DFlash on local NVMe

# Target model
MODEL_DIR=/mnt/nvme5n1p1/hydra/models/Kimi-K2.6
DRAFT_MODEL_DIR=/mnt/nvme5n1p1/hydra/models/Kimi-K2.5-DFlash
IMAGE=vllm/vllm-openai-rocm:nightly
PORT=8262

# DFlash speculative decoding
SPEC_METHOD=dflash
NUM_SPECULATIVE_TOKENS=2
BLOCK_SIZE=16

# KV cache — fp8 halves KV memory, ~2x token capacity (~2.6M vs ~1.3M)
# This does NOT touch MoE kernels. The 384-expert AITER crash only triggers
# at max_num_batched_tokens > 32768, which we avoid.
KV_CACHE_DTYPE=auto

# Scheduler
MAX_NUM_SEQS=32
MAX_NUM_BATCHED_TOKENS=32768
MAX_MODEL_LEN=262144
GPU_MEMORY_UTILIZATION=0.90

# Runtime
TENSOR_PARALLEL_SIZE=8
ENFORCE_EAGER=true
MOE_BACKEND=aiter
OPTIMIZATION_LEVEL=2
PERFORMANCE_MODE=throughput
SAFETENSORS_LOAD_STRATEGY=lazy
ENABLE_PREFIX_CACHING=false
ENABLE_CHUNKED_PREFILL=true

# ROCm environment
PYTORCH_ROCM_ARCH=gfx942
AITER_ROCM_ARCH=gfx942
GPU_ARCHS=gfx942
VLLM_ROCM_USE_AITER=1
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
VLLM_ROCM_USE_AITER_RMSNORM=0
HSA_ENABLE_SDMA=0
HSA_NO_SCRATCH_RECLAIM=1
OMP_NUM_THREADS=1