Update deepseek-v4-flash.service

deepseek-v4-flash.service

@@ -11,10 +11,17 @@
 # Measured 2026-07-21: 115-133 tok/s sustained generation (~1.5-1.65x over
 # the ~80 tok/s plain-decoding baseline), 96.6-100% draft acceptance.
 #
+# Incident 2026-07-23: a Triton JIT module load OOM'd and killed a worker
+# mid-run; the API server then exited *cleanly* (main PID rc 0), so the old
+# Restart=on-failure never fired and the port stayed dead. Hence Restart=always
+# below -- for an inference server a crash that looks like a clean exit is
+# still a crash; only 'systemctl stop' should keep it down. ExecStartPost runs
+# warmup.sh after every (re)start so Triton specializations compile up front.
+#
 # NOTE: the setup script also generates a copy of this unit inside the
-# project folder with User/paths already baked in. EDIT the three
+# project folder with User/paths already baked in. EDIT the four
 # /home/sinclair/deepseek-v4-serve references and User/Group below if your
-# layout differs.
+# layout differs (three ExecStart* lines + WorkingDirectory).
 #
 # Install:
 #   sudo cp deepseek-v4-flash.service /etc/systemd/system/
@@ -45,7 +52,16 @@ ExecStartPre=/usr/bin/nvidia-smi
 
 ExecStart=/home/sinclair/deepseek-v4-serve/serve.sh
 
-Restart=on-failure
+# Warm the Triton shape specializations after EVERY start (incl. auto-restart).
+# warmup.sh waits for the API itself and is bounded, so it cannot hang the
+# unit; '-' prefix + timeout mean a wedged warmup never fails the start.
+ExecStartPost=-/usr/bin/timeout 900 /home/sinclair/deepseek-v4-serve/warmup.sh
+# Budget must cover ExecStartPre + weight load + bounded warmup.
+TimeoutStartSec=1200
+
+# Restart=always, NOT on-failure: a JIT-OOM crash can present as a clean exit
+# (see incident note above); only an explicit 'systemctl stop' should end it.
+Restart=always
 # Weight load alone is ~80 s (+ ~5 s MTP drafter); don't hammer restarts.
 RestartSec=15
 # Give the engine time to tear down 4 TP workers and NCCL cleanly on stop.