from __future__ import annotations import json from pathlib import Path import numpy as np ROOT = Path(__file__).resolve().parent from telemetry import ( RobustPCADetector, causal_window_features, generate_telemetry_run, ) ARTIFACT_PATH = ROOT / "model.json" METRICS_PATH = ROOT / "metrics.json" ANOMALIES = ("sensor_drift", "valve_stiction", "cooling_degradation", "auxiliary_trip") def feature_matrix(seeds: range) -> np.ndarray: return np.concatenate( [causal_window_features(generate_telemetry_run(seed).values)[0] for seed in seeds] ) def evaluate(detector: RobustPCADetector) -> dict[str, object]: runs = [generate_telemetry_run(seed) for seed in range(300, 340)] for anomaly_index, anomaly in enumerate(ANOMALIES): base = 400 + anomaly_index * 20 runs.extend(generate_telemetry_run(base + index, anomaly) for index in range(5)) tp = fp = fn = 0 detected_events = 0 normal_runs_with_alert = 0 false_alarm_episodes = 0 normal_seconds = 0.0 latencies = [] latencies_by_family: dict[str, list[float]] = {name: [] for name in ANOMALIES} for run in runs: ticks, scores = detector.score_run(run) alerts = detector.persistent_alerts(scores, detector.threshold, samples=3) labels = run.labels[ticks].astype(bool) tp += int(np.sum(alerts & labels)) fp += int(np.sum(alerts & ~labels)) fn += int(np.sum(~alerts & labels)) if run.event_start is None: normal_seconds += len(run.values) * run.cadence_seconds normal_runs_with_alert += int(np.any(alerts)) false_alarm_episodes += int(np.sum(alerts & ~np.r_[False, alerts[:-1]])) else: after_event = alerts & (ticks >= run.event_start) if np.any(after_event): detected_events += 1 first_tick = int(ticks[np.flatnonzero(after_event)[0]]) latency = (first_tick - run.event_start) * run.cadence_seconds latencies.append(latency) latencies_by_family[run.anomaly_type].append(latency) abnormal_events = len(ANOMALIES) * 5 point_precision = tp / max(tp + fp, 1) point_recall = tp / max(tp + fn, 1) event_precision = detected_events / max(detected_events + normal_runs_with_alert, 1) return { "schema_version": "forge.anomaly.metrics.v1", "evaluation_split": { "strategy": "whole simulated runs; no overlapping windows across splits", "normal_runs": 40, "anomalous_runs": abnormal_events, "anomaly_families": list(ANOMALIES), }, "threshold_selection": ( "99.9th percentile of separate normal calibration runs plus " "three-sample persistence" ), "metrics": { "event_recall": round(detected_events / abnormal_events, 6), "event_precision": round(event_precision, 6), "point_precision": round(point_precision, 6), "point_recall": round(point_recall, 6), "false_alarm_episodes_per_hour": round( false_alarm_episodes / max(normal_seconds / 3600.0, 1e-9), 6 ), "median_detection_delay_seconds": round(float(np.median(latencies)), 6), "p95_detection_delay_seconds": round(float(np.quantile(latencies, 0.95)), 6), "median_detection_delay_by_family_seconds": { name: round(float(np.median(values)), 6) for name, values in latencies_by_family.items() }, "normal_evaluation_hours": round(normal_seconds / 3600.0, 6), }, "efficiency": { "feature_dimensions": int(detector.center.size), "pca_components": int(detector.components.shape[0]), "score_path": "robust scale, PCA matrix projection, reconstruction residual", "runtime": "NumPy CPU", }, "authority": "L0 evidence only; deterministic interlocks remain independent", } def main() -> None: train = feature_matrix(range(0, 80)) calibration = feature_matrix(range(100, 140)) detector = RobustPCADetector.fit(train, calibration, calibration_quantile=0.999) ARTIFACT_PATH.write_text(json.dumps(detector.to_dict(), indent=2) + "\n") metrics = evaluate(detector) metrics["artifact_bytes"] = ARTIFACT_PATH.stat().st_size METRICS_PATH.write_text(json.dumps(metrics, indent=2) + "\n") print(json.dumps(metrics, indent=2)) if __name__ == "__main__": main()