"""Generalized Modal vLLM server for the multi-model council (v2). One parameterized app so each sponsor model can be deployed as its own engine: set TTW_APP_NAME + TTW_MODEL (+ optional TTW_VLLM pin, TTW_GPU, TTW_MAX_LEN) and deploy or run. Used both for Phase 0 serve smoke tests and for the live council. Smoke test one model (ephemeral, no deploy): TTW_APP_NAME=ttw-smoke TTW_MODEL=openai/gpt-oss-20b TTW_GPU=L4 \ python -m modal run serve_council.py Deploy an engine for the live council: TTW_APP_NAME=ttw-serve-gptoss TTW_MODEL=openai/gpt-oss-20b TTW_GPU=L4 \ python -m modal deploy serve_council.py Notes: - gpt-oss needs a recent vLLM (the v1 pin 0.6.6 predates it). Default TTW_VLLM is unpinned to pull current support; pin per engine once a good version is known. - dtype="auto" lets a model use its native quant (gpt-oss MXFP4, etc.). - gpt-oss MXFP4 kernels JIT-compile at load and need nvcc, which debian_slim lacks. Set TTW_CUDA_DEVEL=1 to build on an nvidia/cuda devel base (ships the toolkit). - Some models (MiniCPM3) load only with trust_remote_code; set TTW_TRUST_REMOTE=1. """ import os import modal APP_NAME = os.environ.get("TTW_APP_NAME", "ttw-serve-council") MODEL = os.environ.get("TTW_MODEL", "openai/gpt-oss-20b") MODEL_REVISION = os.environ.get("TTW_MODEL_REVISION", "main") VLLM = os.environ.get("TTW_VLLM", "vllm==0.22.1") # pinned: verified serving all 4 council models GPU = os.environ.get("TTW_GPU", "L4") MAX_LEN = int(os.environ.get("TTW_MAX_LEN", "2048")) DTYPE = os.environ.get("TTW_DTYPE", "auto") # "auto" keeps native quant (MXFP4 etc.) # nvcc is needed when vLLM JIT-compiles kernels at load (gpt-oss MXFP4 path). A # devel CUDA base ships the toolkit; debian_slim does not. Off by default so the # known-good Qwen path stays on the lean image. CUDA_DEVEL = os.environ.get("TTW_CUDA_DEVEL", "0") == "1" CUDA_TAG = os.environ.get("TTW_CUDA_TAG", "12.4.1-devel-ubuntu22.04") # Some models ship custom modeling code (MiniCPM3); vLLM needs explicit opt-in. TRUST_REMOTE = os.environ.get("TTW_TRUST_REMOTE", "0") == "1" # Chat-template reasoning effort (gpt-oss harmony: low/medium/high; empty = leave # the template default). At the default (medium) gpt-oss-20b spends the whole # per-turn budget in its analysis channel and never emits `assistantfinal`, so # the council's harmony normalizer drops the output and Oona sits every turn out # (measured: 10/10 empty turns). Deploy the gptoss engine with # TTW_REASONING_EFFORT=low to reach the final answer inside the budget. A plain # "Reasoning: low" line in the system message does NOT work: the template routes # system content to the developer block; the knob must be a template kwarg. REASONING_EFFORT = os.environ.get("TTW_REASONING_EFFORT", "") app = modal.App(APP_NAME) def build_image() -> modal.Image: """Build the per-model image. Default path uses the lean debian_slim base (known-good for Qwen). When TTW_CUDA_DEVEL=1, base on an nvidia/cuda devel image so nvcc is present for vLLM's load-time kernel compilation (required by gpt-oss MXFP4). """ if CUDA_DEVEL: base = modal.Image.from_registry( f"nvidia/cuda:{CUDA_TAG}", add_python="3.12" ) else: base = modal.Image.debian_slim(python_version="3.12") return ( base.pip_install(VLLM, "huggingface_hub[hf_transfer]") # Bake config into the image env so the CONTAINER (which re-imports fresh) # sees it; a local deploy-time env var does not propagate on its own. .env( { "HF_HUB_ENABLE_HF_TRANSFER": "1", "VLLM_DO_NOT_TRACK": "1", "TTW_MODEL": MODEL, "TTW_MODEL_REVISION": MODEL_REVISION, "TTW_MAX_LEN": str(MAX_LEN), "TTW_DTYPE": DTYPE, "TTW_TRUST_REMOTE": "1" if TRUST_REMOTE else "0", "TTW_REASONING_EFFORT": REASONING_EFFORT, } ) ) image = build_image() hf_cache = modal.Volume.from_name("ttw-hf-cache", create_if_missing=True) CACHE_DIR = "/root/.cache/huggingface" @app.cls( gpu=GPU, image=image, volumes={CACHE_DIR: hf_cache}, scaledown_window=300, timeout=900, ) class Engine: @modal.enter() def load(self): from vllm import LLM, SamplingParams self.SamplingParams = SamplingParams self.llm = LLM( model=os.environ["TTW_MODEL"], revision=os.environ.get("TTW_MODEL_REVISION", "main"), dtype=os.environ.get("TTW_DTYPE", "auto"), max_model_len=int(os.environ.get("TTW_MAX_LEN", "2048")), gpu_memory_utilization=0.90, enforce_eager=True, trust_remote_code=os.environ.get("TTW_TRUST_REMOTE", "0") == "1", ) @modal.method() def chat_batch( self, conversations: list[list[dict]], max_tokens: int = 256, temperature: float = 0.7, ) -> list[str]: params = self.SamplingParams( temperature=temperature, top_p=0.9, max_tokens=max_tokens ) effort = os.environ.get("TTW_REASONING_EFFORT", "") if effort: try: outputs = self.llm.chat( conversations, params, chat_template_kwargs={"reasoning_effort": effort}, ) except TypeError: # Older vLLM without chat_template_kwargs: serve without the # knob rather than fail the whole engine. outputs = self.llm.chat(conversations, params) else: outputs = self.llm.chat(conversations, params) return [o.outputs[0].text for o in outputs] @app.local_entrypoint() def main(): """Smoke test: one in-character decision, prints the raw completion.""" convos = [ [ {"role": "system", "content": "You are Oona, a cautious owl financier in a woodland economy."}, {"role": "user", "content": "Honey just crashed on a rumor. Buy, sell, or hold? Answer in one short sentence."}, ] ] # gpt-oss emits the harmony format (analysis channel before the final answer), # so it needs a larger budget to reach the final channel; bump via TTW_SMOKE_TOKENS. smoke_tokens = int(os.environ.get("TTW_SMOKE_TOKENS", "80")) print( f"MODEL={os.environ.get('TTW_MODEL')} GPU={GPU} VLLM={VLLM} " f"CUDA_DEVEL={CUDA_DEVEL} TRUST_REMOTE={TRUST_REMOTE} DTYPE={DTYPE}" ) for text in Engine().chat_batch.remote( convos, max_tokens=smoke_tokens, temperature=0.7 ): print("OUT:", text.strip())