Spaces:
Running
Running
chore(sync): mirror backend .py + Dockerfile to Space (hf-sync-backend)
Browse filesAutomated backend sync from szl-holdings/a11oy main via hf-sync-backend.
Updated (differed from the Space): Dockerfile, gdw_auth.py, gdw_drain.py, gdw_proofs.py, gdw_runtime.py, gdw_workspace.py, routers/gdw_frontier.py, serve.py, szl_colang_policy.py
Deleted (gone from the repo + Dockerfile COPY set): (none)
Keeps the Space-built backend (serve.py + the Dockerfile-COPY'd .py
modules) identical to GitHub main so the Space never rebuilds from a
stale backend, new endpoints don't 404 there, and orphaned modules
removed from the repo don't linger in the Space tree.
- Dockerfile +3 -3
- gdw_auth.py +527 -0
- gdw_drain.py +12 -59
- gdw_proofs.py +79 -39
- gdw_runtime.py +601 -0
- gdw_workspace.py +0 -0
- routers/gdw_frontier.py +462 -337
- serve.py +3 -2
- szl_colang_policy.py +285 -156
Dockerfile
CHANGED
|
@@ -521,7 +521,7 @@ COPY static/shared/szl_label_engine.js static/shared/szl_receipt_cosign.js stati
|
|
| 521 |
|
| 522 |
# --- GOVERNANCE / EVAL / CALIBRATION layer (Dev B, 2026-06): ADDITIVE ---
|
| 523 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §67)
|
| 524 |
-
COPY policy/colang/roe_core.co policy/colang/killinchu_threat.co policy/colang/gdw_enforcement_contract.json ./policy/colang/
|
| 525 |
# GOVERNED AUTO-REVIEW (Integration I2) — keystone autonomy layer: governed +
|
| 526 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §68)
|
| 527 |
COPY scripts/check_tau_eval.py ./scripts/check_tau_eval.py
|
|
@@ -612,7 +612,7 @@ COPY szl_spend_cap.py ./szl_spend_cap.py
|
|
| 612 |
# WAVE R Dev 1 — boot-resilience env/secret preflight. Per-file COPY (this
|
| 613 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §91)
|
| 614 |
COPY a11oy_model_intel.py a11oy_experimental_tier.py a11oy_markets.py szl_agent_tts.py szl_gated_delta.py szl_blocksparse.py szl_retrieval_attn.py szl_model_harness.py szl_agent_loop_governed.py szl_crypto_pipeline.py szl_confattest.py szl_agent_operate.py szl_agentloop_brain.py szl_governed_rag.py szl_sovereign_flywheel.py szl_brain_corpus.py szl_verify_transcript.py szl_frontier_index.py szl_whatsnew.py szl_honestywall.py szl_brainmemory.py szl_agentos.py szl_brainground.py szl_brainuncertainty.py szl_brainhealth.py szl_brainwatch.py szl_boot_preflight.py szl_guarded_surface.py szl_status_aggregate.py szl_brainconstitution.py szl_brainagent.py szl_surface_manifests.py szl_source_attestation.py szl_compute_pool_contract.py szl_estateconstitution.py ./
|
| 615 |
-
COPY gdw_attention.py gdw_workspace.py gdw_telemetry.py gdw_proofs.py gdw_drain.py ./
|
| 616 |
COPY static/3d/surfaces/gateddelta.js static/3d/surfaces/blocksparse.js static/3d/surfaces/retrievalattn.js static/3d/surfaces/governedagent.js static/3d/surfaces/cryptopipeline.js static/3d/surfaces/confattest.js static/3d/surfaces/agentops.js static/3d/surfaces/frontierindex.js static/3d/surfaces/whatsnew.js static/3d/surfaces/opsdash.js ./static/3d/surfaces/
|
| 617 |
|
| 618 |
# FORGE-FAMILY WALL (2026-07-14): /api/forge/family — server-side ed25519
|
|
@@ -654,7 +654,7 @@ ENV SZL_GIT_SHA=${SZL_GIT_SHA} \
|
|
| 654 |
# The Second Brain's SQLite index is rebuildable, but a mounted /app/data keeps
|
| 655 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §97)
|
| 656 |
VOLUME ["/app/data"]
|
| 657 |
-
CMD ["python", "
|
| 658 |
|
| 659 |
|
| 660 |
# Build cache-bust 2026-06-05T00:00Z (Orchestrator Squad):
|
|
|
|
| 521 |
|
| 522 |
# --- GOVERNANCE / EVAL / CALIBRATION layer (Dev B, 2026-06): ADDITIVE ---
|
| 523 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §67)
|
| 524 |
+
COPY policy/colang/roe_core.co policy/colang/killinchu_threat.co policy/colang/gdw_enforcement_contract.json policy/colang/enforcement-contract.json ./policy/colang/
|
| 525 |
# GOVERNED AUTO-REVIEW (Integration I2) — keystone autonomy layer: governed +
|
| 526 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §68)
|
| 527 |
COPY scripts/check_tau_eval.py ./scripts/check_tau_eval.py
|
|
|
|
| 612 |
# WAVE R Dev 1 — boot-resilience env/secret preflight. Per-file COPY (this
|
| 613 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §91)
|
| 614 |
COPY a11oy_model_intel.py a11oy_experimental_tier.py a11oy_markets.py szl_agent_tts.py szl_gated_delta.py szl_blocksparse.py szl_retrieval_attn.py szl_model_harness.py szl_agent_loop_governed.py szl_crypto_pipeline.py szl_confattest.py szl_agent_operate.py szl_agentloop_brain.py szl_governed_rag.py szl_sovereign_flywheel.py szl_brain_corpus.py szl_verify_transcript.py szl_frontier_index.py szl_whatsnew.py szl_honestywall.py szl_brainmemory.py szl_agentos.py szl_brainground.py szl_brainuncertainty.py szl_brainhealth.py szl_brainwatch.py szl_boot_preflight.py szl_guarded_surface.py szl_status_aggregate.py szl_brainconstitution.py szl_brainagent.py szl_surface_manifests.py szl_source_attestation.py szl_compute_pool_contract.py szl_estateconstitution.py ./
|
| 615 |
+
COPY gdw_attention.py gdw_auth.py gdw_workspace.py gdw_telemetry.py gdw_proofs.py gdw_runtime.py gdw_drain.py ./
|
| 616 |
COPY static/3d/surfaces/gateddelta.js static/3d/surfaces/blocksparse.js static/3d/surfaces/retrievalattn.js static/3d/surfaces/governedagent.js static/3d/surfaces/cryptopipeline.js static/3d/surfaces/confattest.js static/3d/surfaces/agentops.js static/3d/surfaces/frontierindex.js static/3d/surfaces/whatsnew.js static/3d/surfaces/opsdash.js ./static/3d/surfaces/
|
| 617 |
|
| 618 |
# FORGE-FAMILY WALL (2026-07-14): /api/forge/family — server-side ed25519
|
|
|
|
| 654 |
# The Second Brain's SQLite index is rebuildable, but a mounted /app/data keeps
|
| 655 |
# … (full rationale: docs/DOCKERFILE_NOTES.md §97)
|
| 656 |
VOLUME ["/app/data"]
|
| 657 |
+
CMD ["python", "gdw_runtime.py"]
|
| 658 |
|
| 659 |
|
| 660 |
# Build cache-bust 2026-06-05T00:00Z (Orchestrator Squad):
|
gdw_auth.py
ADDED
|
@@ -0,0 +1,527 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stable principal authentication for the Governed Delta Workspace.
|
| 2 |
+
|
| 3 |
+
This services-layer module accepts a secret-managed JSON registry, reduces raw
|
| 4 |
+
bearer tokens to fixed-length digests during parsing, accepts exact pre-hashed
|
| 5 |
+
token bindings, and returns immutable principal identities. Raw tokens are
|
| 6 |
+
never retained by the resulting registry or included in errors and
|
| 7 |
+
representations.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import hashlib
|
| 11 |
+
import hmac
|
| 12 |
+
import json
|
| 13 |
+
import re
|
| 14 |
+
from dataclasses import dataclass
|
| 15 |
+
from typing import Any, FrozenSet, Iterable, Optional, Tuple, Union
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
_IDENTIFIER_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._:-]{0,127}$")
|
| 19 |
+
_REGISTRY_KEYS = frozenset({"version", "credentials"})
|
| 20 |
+
_CREDENTIAL_KEYS = frozenset(
|
| 21 |
+
{
|
| 22 |
+
"owner_id",
|
| 23 |
+
"namespace",
|
| 24 |
+
"key_id",
|
| 25 |
+
"token",
|
| 26 |
+
"token_sha256",
|
| 27 |
+
"scopes",
|
| 28 |
+
"revoked",
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
_REQUIRED_CREDENTIAL_KEYS = frozenset(
|
| 32 |
+
{"owner_id", "namespace", "key_id", "scopes"}
|
| 33 |
+
)
|
| 34 |
+
_TOKEN_KEYS = frozenset({"token", "token_sha256"})
|
| 35 |
+
_LEGACY_PRINCIPAL_KEYS = frozenset({"token_sha256", "roles"})
|
| 36 |
+
_LEGACY_ROLE_SCOPES = {
|
| 37 |
+
"user": frozenset({"session:read", "step:write"}),
|
| 38 |
+
"admin": frozenset(
|
| 39 |
+
{
|
| 40 |
+
"bench:read",
|
| 41 |
+
"integrity:global",
|
| 42 |
+
"integrity:read",
|
| 43 |
+
"metrics:read",
|
| 44 |
+
"session:read",
|
| 45 |
+
"step:write",
|
| 46 |
+
}
|
| 47 |
+
),
|
| 48 |
+
}
|
| 49 |
+
_SAFE_AUTH_MESSAGES = {
|
| 50 |
+
"missing_authorization": "bearer authorization is required",
|
| 51 |
+
"invalid_authorization": "bearer authorization is invalid",
|
| 52 |
+
"invalid_bearer_token": "bearer credential is invalid",
|
| 53 |
+
"credential_revoked": "bearer credential is revoked",
|
| 54 |
+
"foreign_namespace": "bearer credential is not valid for this namespace",
|
| 55 |
+
"missing_scopes": "bearer credential lacks required scopes",
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class AuthConfigurationError(ValueError):
|
| 60 |
+
"""Raised when authentication configuration is absent or malformed."""
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class AuthenticationError(ValueError):
|
| 64 |
+
"""A token-safe authentication failure with a stable machine code."""
|
| 65 |
+
|
| 66 |
+
def __init__(self, code: str):
|
| 67 |
+
if code not in _SAFE_AUTH_MESSAGES:
|
| 68 |
+
raise ValueError("unknown authentication error code")
|
| 69 |
+
self.code = code
|
| 70 |
+
super().__init__(_SAFE_AUTH_MESSAGES[code])
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@dataclass(frozen=True, slots=True)
|
| 74 |
+
class Principal:
|
| 75 |
+
"""Stable caller identity, independent of the active credential key."""
|
| 76 |
+
|
| 77 |
+
owner_id: str
|
| 78 |
+
namespace: str
|
| 79 |
+
key_id: str
|
| 80 |
+
scopes: Tuple[str, ...]
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
@dataclass(frozen=True, slots=True, repr=False)
|
| 84 |
+
class _Credential:
|
| 85 |
+
owner_id: str
|
| 86 |
+
namespace: str
|
| 87 |
+
key_id: str
|
| 88 |
+
token_digest: bytes
|
| 89 |
+
scopes: FrozenSet[str]
|
| 90 |
+
revoked: bool
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class CredentialRegistry:
|
| 94 |
+
"""Immutable, token-redacted credential registry."""
|
| 95 |
+
|
| 96 |
+
__slots__ = ("_credentials",)
|
| 97 |
+
|
| 98 |
+
def __init__(self, credentials: Iterable[_Credential]):
|
| 99 |
+
values = tuple(credentials)
|
| 100 |
+
if not values:
|
| 101 |
+
raise AuthConfigurationError(
|
| 102 |
+
"credential registry must contain at least one credential"
|
| 103 |
+
)
|
| 104 |
+
self._credentials = values
|
| 105 |
+
|
| 106 |
+
@property
|
| 107 |
+
def credential_count(self) -> int:
|
| 108 |
+
return len(self._credentials)
|
| 109 |
+
|
| 110 |
+
def __repr__(self) -> str:
|
| 111 |
+
return f"CredentialRegistry(credential_count={self.credential_count})"
|
| 112 |
+
|
| 113 |
+
def authenticate(
|
| 114 |
+
self,
|
| 115 |
+
authorization: Optional[str],
|
| 116 |
+
*,
|
| 117 |
+
namespace: str,
|
| 118 |
+
required_scopes: Iterable[str] = (),
|
| 119 |
+
) -> Principal:
|
| 120 |
+
"""Authenticate a bearer header without short-circuiting credential scans."""
|
| 121 |
+
canonical_namespace = _validate_identifier("namespace", namespace)
|
| 122 |
+
required = _normalize_scopes(
|
| 123 |
+
required_scopes,
|
| 124 |
+
field_name="required_scopes",
|
| 125 |
+
allow_empty=True,
|
| 126 |
+
require_list=False,
|
| 127 |
+
)
|
| 128 |
+
token = _parse_bearer_header(authorization)
|
| 129 |
+
try:
|
| 130 |
+
supplied_digest = hashlib.sha256(token.encode("utf-8")).digest()
|
| 131 |
+
finally:
|
| 132 |
+
token = None
|
| 133 |
+
|
| 134 |
+
matched: Optional[_Credential] = None
|
| 135 |
+
for credential in self._credentials:
|
| 136 |
+
is_match = hmac.compare_digest(
|
| 137 |
+
supplied_digest,
|
| 138 |
+
credential.token_digest,
|
| 139 |
+
)
|
| 140 |
+
if is_match:
|
| 141 |
+
matched = credential
|
| 142 |
+
|
| 143 |
+
if matched is None:
|
| 144 |
+
raise AuthenticationError("invalid_bearer_token")
|
| 145 |
+
if matched.revoked:
|
| 146 |
+
raise AuthenticationError("credential_revoked")
|
| 147 |
+
if matched.namespace != canonical_namespace:
|
| 148 |
+
raise AuthenticationError("foreign_namespace")
|
| 149 |
+
if not required.issubset(matched.scopes):
|
| 150 |
+
raise AuthenticationError("missing_scopes")
|
| 151 |
+
return Principal(
|
| 152 |
+
owner_id=matched.owner_id,
|
| 153 |
+
namespace=matched.namespace,
|
| 154 |
+
key_id=matched.key_id,
|
| 155 |
+
scopes=tuple(sorted(matched.scopes)),
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _reject_duplicate_object_keys(pairs):
|
| 160 |
+
result = {}
|
| 161 |
+
for key, value in pairs:
|
| 162 |
+
if key in result:
|
| 163 |
+
raise AuthConfigurationError("credential registry has duplicate object keys")
|
| 164 |
+
result[key] = value
|
| 165 |
+
return result
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def _validate_identifier(field_name: str, value: Any) -> str:
|
| 169 |
+
if type(value) is not str or not _IDENTIFIER_PATTERN.fullmatch(value):
|
| 170 |
+
raise AuthConfigurationError(
|
| 171 |
+
f"{field_name} must be a canonical lowercase identifier"
|
| 172 |
+
)
|
| 173 |
+
return value
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _normalize_scopes(
|
| 177 |
+
value: Any,
|
| 178 |
+
*,
|
| 179 |
+
field_name: str,
|
| 180 |
+
allow_empty: bool,
|
| 181 |
+
require_list: bool,
|
| 182 |
+
) -> FrozenSet[str]:
|
| 183 |
+
if require_list:
|
| 184 |
+
if type(value) is not list:
|
| 185 |
+
raise AuthConfigurationError(f"{field_name} must be a JSON array")
|
| 186 |
+
values = value
|
| 187 |
+
else:
|
| 188 |
+
if isinstance(value, (str, bytes)) or value is None:
|
| 189 |
+
raise AuthConfigurationError(f"{field_name} must be an iterable of scopes")
|
| 190 |
+
try:
|
| 191 |
+
values = list(value)
|
| 192 |
+
except TypeError as exc:
|
| 193 |
+
raise AuthConfigurationError(
|
| 194 |
+
f"{field_name} must be an iterable of scopes"
|
| 195 |
+
) from exc
|
| 196 |
+
if not values and not allow_empty:
|
| 197 |
+
raise AuthConfigurationError(f"{field_name} must not be empty")
|
| 198 |
+
normalized = tuple(
|
| 199 |
+
_validate_identifier(f"{field_name} item", item) for item in values
|
| 200 |
+
)
|
| 201 |
+
if len(set(normalized)) != len(normalized):
|
| 202 |
+
raise AuthConfigurationError(f"{field_name} must not contain duplicates")
|
| 203 |
+
return frozenset(normalized)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def _digest_registry_token(value: Any) -> bytes:
|
| 207 |
+
token = value
|
| 208 |
+
try:
|
| 209 |
+
if type(token) is not str or not token:
|
| 210 |
+
raise AuthConfigurationError("credential token must be a non-empty string")
|
| 211 |
+
if len(token) > 4096 or any(character.isspace() for character in token):
|
| 212 |
+
raise AuthConfigurationError(
|
| 213 |
+
"credential token must be a bounded bearer token"
|
| 214 |
+
)
|
| 215 |
+
return hashlib.sha256(token.encode("utf-8")).digest()
|
| 216 |
+
finally:
|
| 217 |
+
token = None
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _parse_token_sha256(value: Any) -> bytes:
|
| 221 |
+
if (
|
| 222 |
+
type(value) is not str
|
| 223 |
+
or not re.fullmatch(r"[0-9a-f]{64}", value)
|
| 224 |
+
or value == hashlib.sha256(b"").hexdigest()
|
| 225 |
+
):
|
| 226 |
+
raise AuthConfigurationError(
|
| 227 |
+
"credential token_sha256 must be a lowercase SHA-256 digest"
|
| 228 |
+
)
|
| 229 |
+
return bytes.fromhex(value)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _credential_from_mapping(raw: Any, index: int) -> _Credential:
|
| 233 |
+
if type(raw) is not dict:
|
| 234 |
+
raise AuthConfigurationError(
|
| 235 |
+
f"credential registry entry {index} must be an object"
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
keys = frozenset(raw)
|
| 239 |
+
unknown = keys - _CREDENTIAL_KEYS
|
| 240 |
+
missing = _REQUIRED_CREDENTIAL_KEYS - keys
|
| 241 |
+
token_keys = keys & _TOKEN_KEYS
|
| 242 |
+
if unknown or missing or len(token_keys) != 1:
|
| 243 |
+
raise AuthConfigurationError(
|
| 244 |
+
f"credential registry entry {index} has an invalid shape"
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
token_key = next(iter(token_keys))
|
| 248 |
+
token_value = raw.pop(token_key)
|
| 249 |
+
try:
|
| 250 |
+
token_digest = (
|
| 251 |
+
_digest_registry_token(token_value)
|
| 252 |
+
if token_key == "token"
|
| 253 |
+
else _parse_token_sha256(token_value)
|
| 254 |
+
)
|
| 255 |
+
finally:
|
| 256 |
+
token_value = None
|
| 257 |
+
|
| 258 |
+
owner_id = _validate_identifier("owner_id", raw["owner_id"])
|
| 259 |
+
namespace = _validate_identifier("namespace", raw["namespace"])
|
| 260 |
+
key_id = _validate_identifier("key_id", raw["key_id"])
|
| 261 |
+
scopes = _normalize_scopes(
|
| 262 |
+
raw["scopes"],
|
| 263 |
+
field_name="scopes",
|
| 264 |
+
allow_empty=False,
|
| 265 |
+
require_list=True,
|
| 266 |
+
)
|
| 267 |
+
revoked = raw.get("revoked", False)
|
| 268 |
+
if type(revoked) is not bool:
|
| 269 |
+
raise AuthConfigurationError("revoked must be a JSON boolean")
|
| 270 |
+
return _Credential(
|
| 271 |
+
owner_id=owner_id,
|
| 272 |
+
namespace=namespace,
|
| 273 |
+
key_id=key_id,
|
| 274 |
+
token_digest=token_digest,
|
| 275 |
+
scopes=scopes,
|
| 276 |
+
revoked=revoked,
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _registry_from_credentials(
|
| 281 |
+
credentials: Iterable[_Credential],
|
| 282 |
+
) -> CredentialRegistry:
|
| 283 |
+
values = []
|
| 284 |
+
token_digests = set()
|
| 285 |
+
key_ids = set()
|
| 286 |
+
for credential in credentials:
|
| 287 |
+
if credential.token_digest in token_digests:
|
| 288 |
+
raise AuthConfigurationError(
|
| 289 |
+
"credential registry contains a duplicate token"
|
| 290 |
+
)
|
| 291 |
+
key_identity = (credential.namespace, credential.key_id)
|
| 292 |
+
if key_identity in key_ids:
|
| 293 |
+
raise AuthConfigurationError(
|
| 294 |
+
"credential registry contains a duplicate namespace/key_id"
|
| 295 |
+
)
|
| 296 |
+
token_digests.add(credential.token_digest)
|
| 297 |
+
key_ids.add(key_identity)
|
| 298 |
+
values.append(credential)
|
| 299 |
+
return CredentialRegistry(values)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def parse_credential_registry(
|
| 303 |
+
registry_json: Union[str, bytes, bytearray],
|
| 304 |
+
) -> CredentialRegistry:
|
| 305 |
+
"""Parse a strict version-1 JSON registry and immediately redact raw tokens."""
|
| 306 |
+
raw_registry = registry_json
|
| 307 |
+
try:
|
| 308 |
+
if not isinstance(raw_registry, (str, bytes, bytearray)):
|
| 309 |
+
raise AuthConfigurationError("credential registry must be JSON text")
|
| 310 |
+
if not raw_registry:
|
| 311 |
+
raise AuthConfigurationError("credential registry must not be empty")
|
| 312 |
+
try:
|
| 313 |
+
decoded = json.loads(
|
| 314 |
+
raw_registry,
|
| 315 |
+
object_pairs_hook=_reject_duplicate_object_keys,
|
| 316 |
+
)
|
| 317 |
+
except AuthConfigurationError:
|
| 318 |
+
raise
|
| 319 |
+
except (TypeError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
| 320 |
+
raise AuthConfigurationError("credential registry is not valid JSON") from exc
|
| 321 |
+
finally:
|
| 322 |
+
raw_registry = None
|
| 323 |
+
registry_json = None
|
| 324 |
+
|
| 325 |
+
if type(decoded) is not dict or frozenset(decoded) != _REGISTRY_KEYS:
|
| 326 |
+
raise AuthConfigurationError("credential registry has an invalid top-level shape")
|
| 327 |
+
if type(decoded["version"]) is not int or decoded["version"] != 1:
|
| 328 |
+
raise AuthConfigurationError("credential registry version must be 1")
|
| 329 |
+
raw_credentials = decoded["credentials"]
|
| 330 |
+
if type(raw_credentials) is not list or not raw_credentials:
|
| 331 |
+
raise AuthConfigurationError(
|
| 332 |
+
"credential registry must contain a non-empty credentials array"
|
| 333 |
+
)
|
| 334 |
+
|
| 335 |
+
credentials = []
|
| 336 |
+
for index, raw_credential in enumerate(raw_credentials):
|
| 337 |
+
credentials.append(_credential_from_mapping(raw_credential, index))
|
| 338 |
+
return _registry_from_credentials(credentials)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def parse_legacy_principal_registry(
|
| 342 |
+
registry_json: Union[str, bytes, bytearray],
|
| 343 |
+
*,
|
| 344 |
+
namespace: str,
|
| 345 |
+
) -> CredentialRegistry:
|
| 346 |
+
"""Map the former digest-only principal registry to stable scoped credentials."""
|
| 347 |
+
canonical_namespace = _validate_identifier("namespace", namespace)
|
| 348 |
+
raw_registry = registry_json
|
| 349 |
+
try:
|
| 350 |
+
if not isinstance(raw_registry, (str, bytes, bytearray)):
|
| 351 |
+
raise AuthConfigurationError("principal registry must be JSON text")
|
| 352 |
+
if not raw_registry:
|
| 353 |
+
raise AuthConfigurationError("principal registry must not be empty")
|
| 354 |
+
try:
|
| 355 |
+
decoded = json.loads(
|
| 356 |
+
raw_registry,
|
| 357 |
+
object_pairs_hook=_reject_duplicate_object_keys,
|
| 358 |
+
)
|
| 359 |
+
except AuthConfigurationError:
|
| 360 |
+
raise
|
| 361 |
+
except (TypeError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
| 362 |
+
raise AuthConfigurationError("principal registry is not valid JSON") from exc
|
| 363 |
+
finally:
|
| 364 |
+
raw_registry = None
|
| 365 |
+
registry_json = None
|
| 366 |
+
|
| 367 |
+
if type(decoded) is not dict or not decoded:
|
| 368 |
+
raise AuthConfigurationError(
|
| 369 |
+
"principal registry must contain at least one principal"
|
| 370 |
+
)
|
| 371 |
+
|
| 372 |
+
credentials = []
|
| 373 |
+
for index, (principal_id, raw_record) in enumerate(decoded.items()):
|
| 374 |
+
owner_id = _validate_identifier("principal_id", principal_id)
|
| 375 |
+
if (
|
| 376 |
+
type(raw_record) is not dict
|
| 377 |
+
or frozenset(raw_record) != _LEGACY_PRINCIPAL_KEYS
|
| 378 |
+
):
|
| 379 |
+
raise AuthConfigurationError(
|
| 380 |
+
f"principal registry entry {index} has an invalid shape"
|
| 381 |
+
)
|
| 382 |
+
roles = raw_record["roles"]
|
| 383 |
+
if type(roles) is not list or not roles:
|
| 384 |
+
raise AuthConfigurationError("principal roles must be a non-empty array")
|
| 385 |
+
if (
|
| 386 |
+
any(type(role) is not str or role not in _LEGACY_ROLE_SCOPES for role in roles)
|
| 387 |
+
or len(set(roles)) != len(roles)
|
| 388 |
+
):
|
| 389 |
+
raise AuthConfigurationError("principal roles are invalid")
|
| 390 |
+
scopes = frozenset().union(
|
| 391 |
+
*(_LEGACY_ROLE_SCOPES[role] for role in roles)
|
| 392 |
+
)
|
| 393 |
+
key_id = "legacy:" + hashlib.sha256(
|
| 394 |
+
owner_id.encode("utf-8")
|
| 395 |
+
).hexdigest()[:24]
|
| 396 |
+
credentials.append(
|
| 397 |
+
_Credential(
|
| 398 |
+
owner_id=owner_id,
|
| 399 |
+
namespace=canonical_namespace,
|
| 400 |
+
key_id=key_id,
|
| 401 |
+
token_digest=_parse_token_sha256(raw_record["token_sha256"]),
|
| 402 |
+
scopes=scopes,
|
| 403 |
+
revoked=False,
|
| 404 |
+
)
|
| 405 |
+
)
|
| 406 |
+
return _registry_from_credentials(credentials)
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def _legacy_registry(
|
| 410 |
+
*,
|
| 411 |
+
token: Optional[str],
|
| 412 |
+
owner_id: Optional[str],
|
| 413 |
+
namespace: Optional[str],
|
| 414 |
+
key_id: str,
|
| 415 |
+
scopes: Iterable[str],
|
| 416 |
+
) -> CredentialRegistry:
|
| 417 |
+
if token is None or owner_id is None or namespace is None:
|
| 418 |
+
raise AuthConfigurationError(
|
| 419 |
+
"legacy authentication requires token, owner_id, and namespace"
|
| 420 |
+
)
|
| 421 |
+
credential = _Credential(
|
| 422 |
+
owner_id=_validate_identifier("legacy owner_id", owner_id),
|
| 423 |
+
namespace=_validate_identifier("legacy namespace", namespace),
|
| 424 |
+
key_id=_validate_identifier("legacy key_id", key_id),
|
| 425 |
+
token_digest=_digest_registry_token(token),
|
| 426 |
+
scopes=_normalize_scopes(
|
| 427 |
+
scopes,
|
| 428 |
+
field_name="legacy scopes",
|
| 429 |
+
allow_empty=False,
|
| 430 |
+
require_list=False,
|
| 431 |
+
),
|
| 432 |
+
revoked=False,
|
| 433 |
+
)
|
| 434 |
+
token = None
|
| 435 |
+
return CredentialRegistry((credential,))
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def load_credential_registry(
|
| 439 |
+
registry_json: Optional[Union[str, bytes, bytearray]],
|
| 440 |
+
*,
|
| 441 |
+
principal_registry_json: Optional[Union[str, bytes, bytearray]] = None,
|
| 442 |
+
principal_registry_namespace: Optional[str] = None,
|
| 443 |
+
legacy_enabled: bool = False,
|
| 444 |
+
legacy_token: Optional[str] = None,
|
| 445 |
+
legacy_owner_id: Optional[str] = None,
|
| 446 |
+
legacy_namespace: Optional[str] = None,
|
| 447 |
+
legacy_key_id: str = "legacy",
|
| 448 |
+
legacy_scopes: Iterable[str] = (),
|
| 449 |
+
) -> CredentialRegistry:
|
| 450 |
+
"""Load registry JSON or an explicitly enabled, fully bound legacy credential."""
|
| 451 |
+
legacy_scopes = tuple(legacy_scopes)
|
| 452 |
+
configured_registries = sum(
|
| 453 |
+
value is not None for value in (registry_json, principal_registry_json)
|
| 454 |
+
)
|
| 455 |
+
if configured_registries > 1:
|
| 456 |
+
raise AuthConfigurationError(
|
| 457 |
+
"credential registries cannot be configured together"
|
| 458 |
+
)
|
| 459 |
+
legacy_values_present = any(
|
| 460 |
+
value is not None
|
| 461 |
+
for value in (legacy_token, legacy_owner_id, legacy_namespace)
|
| 462 |
+
) or legacy_key_id != "legacy" or bool(legacy_scopes)
|
| 463 |
+
if registry_json is not None:
|
| 464 |
+
if legacy_enabled is True or legacy_values_present:
|
| 465 |
+
raise AuthConfigurationError(
|
| 466 |
+
"registry and legacy authentication cannot be configured together"
|
| 467 |
+
)
|
| 468 |
+
return parse_credential_registry(registry_json)
|
| 469 |
+
if principal_registry_json is not None:
|
| 470 |
+
if legacy_enabled is True or legacy_values_present:
|
| 471 |
+
raise AuthConfigurationError(
|
| 472 |
+
"registry and legacy authentication cannot be configured together"
|
| 473 |
+
)
|
| 474 |
+
if principal_registry_namespace is None:
|
| 475 |
+
raise AuthConfigurationError(
|
| 476 |
+
"principal registry requires a namespace binding"
|
| 477 |
+
)
|
| 478 |
+
return parse_legacy_principal_registry(
|
| 479 |
+
principal_registry_json,
|
| 480 |
+
namespace=principal_registry_namespace,
|
| 481 |
+
)
|
| 482 |
+
if legacy_enabled is not True:
|
| 483 |
+
if legacy_values_present:
|
| 484 |
+
raise AuthConfigurationError("legacy authentication is not enabled")
|
| 485 |
+
raise AuthConfigurationError("credential registry is not configured")
|
| 486 |
+
return _legacy_registry(
|
| 487 |
+
token=legacy_token,
|
| 488 |
+
owner_id=legacy_owner_id,
|
| 489 |
+
namespace=legacy_namespace,
|
| 490 |
+
key_id=legacy_key_id,
|
| 491 |
+
scopes=legacy_scopes,
|
| 492 |
+
)
|
| 493 |
+
|
| 494 |
+
|
| 495 |
+
def _parse_bearer_header(authorization: Optional[str]) -> str:
|
| 496 |
+
if authorization is None:
|
| 497 |
+
raise AuthenticationError("missing_authorization")
|
| 498 |
+
if type(authorization) is not str:
|
| 499 |
+
raise AuthenticationError("invalid_authorization")
|
| 500 |
+
scheme, separator, token = authorization.partition(" ")
|
| 501 |
+
if (
|
| 502 |
+
not separator
|
| 503 |
+
or scheme.lower() != "bearer"
|
| 504 |
+
or not token
|
| 505 |
+
or len(token) > 4096
|
| 506 |
+
or any(character.isspace() for character in token)
|
| 507 |
+
):
|
| 508 |
+
token = None
|
| 509 |
+
raise AuthenticationError("invalid_authorization")
|
| 510 |
+
return token
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def authenticate_bearer(
|
| 514 |
+
authorization: Optional[str],
|
| 515 |
+
registry: CredentialRegistry,
|
| 516 |
+
*,
|
| 517 |
+
namespace: str,
|
| 518 |
+
required_scopes: Iterable[str] = (),
|
| 519 |
+
) -> Principal:
|
| 520 |
+
"""Authenticate through a parsed registry and return an immutable principal."""
|
| 521 |
+
if not isinstance(registry, CredentialRegistry):
|
| 522 |
+
raise AuthConfigurationError("registry must be a CredentialRegistry")
|
| 523 |
+
return registry.authenticate(
|
| 524 |
+
authorization,
|
| 525 |
+
namespace=namespace,
|
| 526 |
+
required_scopes=required_scopes,
|
| 527 |
+
)
|
gdw_drain.py
CHANGED
|
@@ -1,13 +1,10 @@
|
|
| 1 |
-
"""
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
-
import os
|
| 6 |
-
import uuid
|
| 7 |
-
from datetime import datetime, timezone
|
| 8 |
from typing import Any, Dict, Optional
|
| 9 |
|
| 10 |
-
from
|
| 11 |
from gdw_workspace import GDWWorkspace
|
| 12 |
|
| 13 |
|
|
@@ -17,62 +14,18 @@ def drain_effects(
|
|
| 17 |
limit: int = 100,
|
| 18 |
worker_id: Optional[str] = None,
|
| 19 |
) -> Dict[str, Any]:
|
| 20 |
-
|
| 21 |
-
if bounded < 1 or bounded > 1000:
|
| 22 |
-
raise ValueError("drain limit must be between 1 and 1000")
|
| 23 |
-
worker = worker_id or f"gdw-runtime-{os.getpid()}-{uuid.uuid4().hex[:12]}"
|
| 24 |
-
exported = 0
|
| 25 |
-
failed = 0
|
| 26 |
-
error_classes = []
|
| 27 |
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
workspace.validate_claimed_effect(row)
|
| 35 |
-
if row["kind"] == "proof_export":
|
| 36 |
-
artifact = export_proof_payload(
|
| 37 |
-
row["payload"],
|
| 38 |
-
artifact_identity=row["idempotency_key"],
|
| 39 |
-
owner_id=row["owner_id"],
|
| 40 |
-
)
|
| 41 |
-
elif row["kind"] == "receipt_projection":
|
| 42 |
-
artifact = export_receipt_projection(
|
| 43 |
-
row["payload"],
|
| 44 |
-
row["idempotency_key"],
|
| 45 |
-
owner_id=row["owner_id"],
|
| 46 |
-
)
|
| 47 |
-
else:
|
| 48 |
-
raise ValueError("unsupported effect kind")
|
| 49 |
-
workspace.mark_effect_exported(
|
| 50 |
-
row["idempotency_key"],
|
| 51 |
-
worker,
|
| 52 |
-
row["claim_token"],
|
| 53 |
-
artifact,
|
| 54 |
-
datetime.now(timezone.utc).isoformat(),
|
| 55 |
-
)
|
| 56 |
-
exported += 1
|
| 57 |
-
except Exception as exc:
|
| 58 |
-
workspace.release_effect(
|
| 59 |
-
row["idempotency_key"],
|
| 60 |
-
worker,
|
| 61 |
-
row["claim_token"],
|
| 62 |
-
f"{type(exc).__name__}: {exc}",
|
| 63 |
-
)
|
| 64 |
-
failed += 1
|
| 65 |
-
error_classes.append(type(exc).__name__)
|
| 66 |
-
break
|
| 67 |
-
|
| 68 |
-
integrity = workspace.integrity()
|
| 69 |
return {
|
| 70 |
-
"schema": "szl.gdw-effect-drain/
|
| 71 |
-
|
| 72 |
-
"failed": failed,
|
| 73 |
-
"error_classes": error_classes,
|
| 74 |
-
"pending_effects": integrity["pending_effects"],
|
| 75 |
"integrity_ok": integrity["ok"],
|
| 76 |
-
"
|
| 77 |
"credential_values_recorded": False,
|
| 78 |
}
|
|
|
|
| 1 |
+
"""Compatibility entry point for the single generation-fenced GDW drain."""
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
|
|
|
|
|
|
|
|
| 5 |
from typing import Any, Dict, Optional
|
| 6 |
|
| 7 |
+
from gdw_runtime import drain_once
|
| 8 |
from gdw_workspace import GDWWorkspace
|
| 9 |
|
| 10 |
|
|
|
|
| 14 |
limit: int = 100,
|
| 15 |
worker_id: Optional[str] = None,
|
| 16 |
) -> Dict[str, Any]:
|
| 17 |
+
"""Delegate to the runtime's only claim, export, and completion path."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
report = drain_once(
|
| 20 |
+
limit=limit,
|
| 21 |
+
worker_id=worker_id,
|
| 22 |
+
workspace=workspace,
|
| 23 |
+
)
|
| 24 |
+
integrity = workspace.integrity(global_scope=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
return {
|
| 26 |
+
"schema": "szl.gdw-effect-drain/v2",
|
| 27 |
+
**report,
|
|
|
|
|
|
|
|
|
|
| 28 |
"integrity_ok": integrity["ok"],
|
| 29 |
+
"database_generation_id": integrity["database_generation_id"],
|
| 30 |
"credential_values_recorded": False,
|
| 31 |
}
|
gdw_proofs.py
CHANGED
|
@@ -4,10 +4,14 @@ import hashlib
|
|
| 4 |
import json
|
| 5 |
import os
|
| 6 |
import tempfile
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
from typing import Any, Dict
|
| 9 |
|
| 10 |
|
|
|
|
|
|
|
|
|
|
| 11 |
def canonical_json(value: Any) -> str:
|
| 12 |
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
| 13 |
|
|
@@ -20,8 +24,9 @@ def build_proof_payload(
|
|
| 20 |
proposal_id: str,
|
| 21 |
request_id: str,
|
| 22 |
request_digest: str,
|
|
|
|
| 23 |
owner_id: str,
|
| 24 |
-
|
| 25 |
step: int,
|
| 26 |
before_hash: str,
|
| 27 |
after_hash: str,
|
|
@@ -38,8 +43,9 @@ def build_proof_payload(
|
|
| 38 |
"proposal_id": proposal_id,
|
| 39 |
"request_id": request_id,
|
| 40 |
"request_digest": request_digest,
|
|
|
|
| 41 |
"owner_id": owner_id,
|
| 42 |
-
"
|
| 43 |
"step_id": step,
|
| 44 |
"state_before_hash": before_hash,
|
| 45 |
"state_after_hash": after_hash,
|
|
@@ -64,46 +70,52 @@ def build_proof_payload(
|
|
| 64 |
return payload
|
| 65 |
|
| 66 |
|
| 67 |
-
def
|
| 68 |
root: Path,
|
| 69 |
filename: str,
|
| 70 |
payload: Dict[str, Any],
|
| 71 |
owner_id: str,
|
| 72 |
) -> Dict[str, Any]:
|
| 73 |
-
if not owner_id:
|
| 74 |
raise ValueError("owner_id is required for artifact isolation")
|
|
|
|
|
|
|
| 75 |
owner_scope = hashlib.sha256(owner_id.encode("utf-8")).hexdigest()[:32]
|
| 76 |
-
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
| 78 |
destination = owner_root / filename
|
| 79 |
encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
| 80 |
expected_sha256 = hashlib.sha256(encoded).hexdigest()
|
| 81 |
-
|
| 82 |
if destination.exists():
|
| 83 |
-
|
| 84 |
-
if existing != encoded:
|
| 85 |
raise FileExistsError(
|
| 86 |
-
|
| 87 |
)
|
| 88 |
return {
|
| 89 |
-
"status": "
|
| 90 |
"path": str(destination),
|
| 91 |
"sha256": expected_sha256,
|
|
|
|
| 92 |
"immutable": True,
|
| 93 |
"owner_scope": owner_scope,
|
| 94 |
}
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
|
|
|
|
|
|
|
|
|
| 102 |
if sum(1 for _ in owner_root.glob("*.json")) >= owner_limit:
|
| 103 |
raise RuntimeError("per-owner artifact quota exceeded")
|
| 104 |
if sum(1 for _ in root.glob("*/*.json")) >= global_limit:
|
| 105 |
raise RuntimeError("global artifact quota exceeded")
|
| 106 |
-
|
| 107 |
handle, temporary = tempfile.mkstemp(
|
| 108 |
prefix=".gdw-artifact-", suffix=".tmp", dir=owner_root
|
| 109 |
)
|
|
@@ -115,10 +127,10 @@ def _export_json_artifact(
|
|
| 115 |
try:
|
| 116 |
os.link(temporary, destination)
|
| 117 |
except FileExistsError:
|
| 118 |
-
|
| 119 |
-
if existing != encoded:
|
| 120 |
raise FileExistsError(
|
| 121 |
-
|
|
|
|
| 122 |
)
|
| 123 |
finally:
|
| 124 |
if os.path.exists(temporary):
|
|
@@ -127,14 +139,50 @@ def _export_json_artifact(
|
|
| 127 |
"status": "EXPORTED",
|
| 128 |
"path": str(destination),
|
| 129 |
"sha256": expected_sha256,
|
|
|
|
| 130 |
"immutable": True,
|
| 131 |
"owner_scope": owner_scope,
|
| 132 |
}
|
| 133 |
|
| 134 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
def export_proof_payload(
|
| 136 |
payload: Dict[str, Any],
|
| 137 |
-
|
|
|
|
| 138 |
owner_id: str | None = None,
|
| 139 |
) -> Dict[str, Any]:
|
| 140 |
root = Path(os.environ.get("GDW_PROOF_DIR", "output/proofs")).resolve()
|
|
@@ -146,36 +194,28 @@ def export_proof_payload(
|
|
| 146 |
unsigned_payload.pop("payload_sha256", None)
|
| 147 |
if claimed_digest != sha256_json(unsigned_payload):
|
| 148 |
raise ValueError("proof payload_sha256 does not match canonical payload")
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
len(identity) != 64
|
| 152 |
-
or any(ch not in "0123456789abcdef" for ch in identity)
|
| 153 |
-
):
|
| 154 |
-
raise ValueError("artifact_identity must be a lowercase SHA-256 digest")
|
| 155 |
artifact = _export_json_artifact(
|
| 156 |
root,
|
| 157 |
-
f"{
|
| 158 |
payload,
|
| 159 |
owner_id or str(payload.get("owner_id") or ""),
|
| 160 |
)
|
| 161 |
-
artifact["artifact_identity"] =
|
| 162 |
artifact.update({"status": "INPUT_EXPORTED", "formal_status": "NOT_RUN"})
|
| 163 |
return artifact
|
| 164 |
|
| 165 |
|
| 166 |
def export_receipt_projection(
|
| 167 |
payload: Dict[str, Any],
|
| 168 |
-
|
| 169 |
owner_id: str | None = None,
|
| 170 |
) -> Dict[str, Any]:
|
| 171 |
root = Path(
|
| 172 |
os.environ.get("GDW_RECEIPT_PROJECTION_DIR", "output/gdw/receipts")
|
| 173 |
).resolve()
|
| 174 |
-
|
| 175 |
-
len(idempotency_key) != 64
|
| 176 |
-
or any(ch not in "0123456789abcdef" for ch in idempotency_key)
|
| 177 |
-
):
|
| 178 |
-
raise ValueError("idempotency_key must be a lowercase SHA-256 digest")
|
| 179 |
claimed_digest = payload.get("receipt_hash")
|
| 180 |
unsigned_payload = dict(payload)
|
| 181 |
unsigned_payload.pop("receipt_hash", None)
|
|
@@ -183,11 +223,11 @@ def export_receipt_projection(
|
|
| 183 |
raise ValueError("receipt_hash does not match canonical receipt")
|
| 184 |
artifact = _export_json_artifact(
|
| 185 |
root,
|
| 186 |
-
f"{
|
| 187 |
payload,
|
| 188 |
owner_id or str(payload.get("owner_id") or ""),
|
| 189 |
)
|
| 190 |
-
artifact["artifact_identity"] =
|
| 191 |
artifact.update(
|
| 192 |
{
|
| 193 |
"status": "RECEIPT_PROJECTED",
|
|
|
|
| 4 |
import json
|
| 5 |
import os
|
| 6 |
import tempfile
|
| 7 |
+
import threading
|
| 8 |
from pathlib import Path
|
| 9 |
from typing import Any, Dict
|
| 10 |
|
| 11 |
|
| 12 |
+
_ARTIFACT_QUOTA_LOCK = threading.RLock()
|
| 13 |
+
|
| 14 |
+
|
| 15 |
def canonical_json(value: Any) -> str:
|
| 16 |
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
| 17 |
|
|
|
|
| 24 |
proposal_id: str,
|
| 25 |
request_id: str,
|
| 26 |
request_digest: str,
|
| 27 |
+
namespace: str,
|
| 28 |
owner_id: str,
|
| 29 |
+
database_generation_id: str,
|
| 30 |
step: int,
|
| 31 |
before_hash: str,
|
| 32 |
after_hash: str,
|
|
|
|
| 43 |
"proposal_id": proposal_id,
|
| 44 |
"request_id": request_id,
|
| 45 |
"request_digest": request_digest,
|
| 46 |
+
"namespace": namespace,
|
| 47 |
"owner_id": owner_id,
|
| 48 |
+
"database_generation_id": database_generation_id,
|
| 49 |
"step_id": step,
|
| 50 |
"state_before_hash": before_hash,
|
| 51 |
"state_after_hash": after_hash,
|
|
|
|
| 70 |
return payload
|
| 71 |
|
| 72 |
|
| 73 |
+
def _export_json_artifact_unlocked(
|
| 74 |
root: Path,
|
| 75 |
filename: str,
|
| 76 |
payload: Dict[str, Any],
|
| 77 |
owner_id: str,
|
| 78 |
) -> Dict[str, Any]:
|
| 79 |
+
if type(owner_id) is not str or not owner_id:
|
| 80 |
raise ValueError("owner_id is required for artifact isolation")
|
| 81 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 82 |
+
root = root.resolve()
|
| 83 |
owner_scope = hashlib.sha256(owner_id.encode("utf-8")).hexdigest()[:32]
|
| 84 |
+
owner_candidate = root / owner_scope
|
| 85 |
+
owner_candidate.mkdir(parents=True, exist_ok=True)
|
| 86 |
+
owner_root = owner_candidate.resolve()
|
| 87 |
+
if owner_root.parent != root or owner_root.name != owner_scope:
|
| 88 |
+
raise ValueError("artifact owner scope escapes the configured root")
|
| 89 |
destination = owner_root / filename
|
| 90 |
encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
| 91 |
expected_sha256 = hashlib.sha256(encoded).hexdigest()
|
|
|
|
| 92 |
if destination.exists():
|
| 93 |
+
if destination.read_bytes() != encoded:
|
|
|
|
| 94 |
raise FileExistsError(
|
| 95 |
+
"refusing to overwrite an existing non-identical GDW artifact"
|
| 96 |
)
|
| 97 |
return {
|
| 98 |
+
"status": "EXPORTED",
|
| 99 |
"path": str(destination),
|
| 100 |
"sha256": expected_sha256,
|
| 101 |
+
"reused": True,
|
| 102 |
"immutable": True,
|
| 103 |
"owner_scope": owner_scope,
|
| 104 |
}
|
| 105 |
+
owner_limit = _bounded_artifact_limit(
|
| 106 |
+
"GDW_OWNER_MAX_ARTIFACTS", default=10_000, maximum=100_000
|
| 107 |
+
)
|
| 108 |
+
global_limit = _bounded_artifact_limit(
|
| 109 |
+
"GDW_GLOBAL_MAX_ARTIFACTS", default=100_000, maximum=1_000_000
|
| 110 |
+
)
|
| 111 |
+
if global_limit < owner_limit:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
"GDW_GLOBAL_MAX_ARTIFACTS must be at least GDW_OWNER_MAX_ARTIFACTS"
|
| 114 |
+
)
|
| 115 |
if sum(1 for _ in owner_root.glob("*.json")) >= owner_limit:
|
| 116 |
raise RuntimeError("per-owner artifact quota exceeded")
|
| 117 |
if sum(1 for _ in root.glob("*/*.json")) >= global_limit:
|
| 118 |
raise RuntimeError("global artifact quota exceeded")
|
|
|
|
| 119 |
handle, temporary = tempfile.mkstemp(
|
| 120 |
prefix=".gdw-artifact-", suffix=".tmp", dir=owner_root
|
| 121 |
)
|
|
|
|
| 127 |
try:
|
| 128 |
os.link(temporary, destination)
|
| 129 |
except FileExistsError:
|
| 130 |
+
if destination.read_bytes() != encoded:
|
|
|
|
| 131 |
raise FileExistsError(
|
| 132 |
+
"refusing to overwrite a concurrently created "
|
| 133 |
+
"non-identical GDW artifact"
|
| 134 |
)
|
| 135 |
finally:
|
| 136 |
if os.path.exists(temporary):
|
|
|
|
| 139 |
"status": "EXPORTED",
|
| 140 |
"path": str(destination),
|
| 141 |
"sha256": expected_sha256,
|
| 142 |
+
"reused": False,
|
| 143 |
"immutable": True,
|
| 144 |
"owner_scope": owner_scope,
|
| 145 |
}
|
| 146 |
|
| 147 |
|
| 148 |
+
def _bounded_artifact_limit(name: str, *, default: int, maximum: int) -> int:
|
| 149 |
+
raw = os.environ.get(name)
|
| 150 |
+
try:
|
| 151 |
+
value = int(raw) if raw not in (None, "") else default
|
| 152 |
+
except (TypeError, ValueError) as exc:
|
| 153 |
+
raise ValueError(f"{name} must be an integer") from exc
|
| 154 |
+
if value < 1 or value > maximum:
|
| 155 |
+
raise ValueError(f"{name} must be between 1 and {maximum}")
|
| 156 |
+
return value
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _export_json_artifact(
|
| 160 |
+
root: Path,
|
| 161 |
+
filename: str,
|
| 162 |
+
payload: Dict[str, Any],
|
| 163 |
+
owner_id: str,
|
| 164 |
+
) -> Dict[str, Any]:
|
| 165 |
+
with _ARTIFACT_QUOTA_LOCK:
|
| 166 |
+
return _export_json_artifact_unlocked(
|
| 167 |
+
root,
|
| 168 |
+
filename,
|
| 169 |
+
payload,
|
| 170 |
+
owner_id,
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _validate_artifact_id(artifact_id: str) -> None:
|
| 175 |
+
if (
|
| 176 |
+
len(artifact_id) != 64
|
| 177 |
+
or any(ch not in "0123456789abcdef" for ch in artifact_id)
|
| 178 |
+
):
|
| 179 |
+
raise ValueError("artifact_id must be a lowercase SHA-256 digest")
|
| 180 |
+
|
| 181 |
+
|
| 182 |
def export_proof_payload(
|
| 183 |
payload: Dict[str, Any],
|
| 184 |
+
*,
|
| 185 |
+
artifact_id: str | None = None,
|
| 186 |
owner_id: str | None = None,
|
| 187 |
) -> Dict[str, Any]:
|
| 188 |
root = Path(os.environ.get("GDW_PROOF_DIR", "output/proofs")).resolve()
|
|
|
|
| 194 |
unsigned_payload.pop("payload_sha256", None)
|
| 195 |
if claimed_digest != sha256_json(unsigned_payload):
|
| 196 |
raise ValueError("proof payload_sha256 does not match canonical payload")
|
| 197 |
+
resolved_artifact_id = artifact_id or claimed_digest
|
| 198 |
+
_validate_artifact_id(resolved_artifact_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
artifact = _export_json_artifact(
|
| 200 |
root,
|
| 201 |
+
f"{resolved_artifact_id}.json",
|
| 202 |
payload,
|
| 203 |
owner_id or str(payload.get("owner_id") or ""),
|
| 204 |
)
|
| 205 |
+
artifact["artifact_identity"] = resolved_artifact_id
|
| 206 |
artifact.update({"status": "INPUT_EXPORTED", "formal_status": "NOT_RUN"})
|
| 207 |
return artifact
|
| 208 |
|
| 209 |
|
| 210 |
def export_receipt_projection(
|
| 211 |
payload: Dict[str, Any],
|
| 212 |
+
artifact_id: str,
|
| 213 |
owner_id: str | None = None,
|
| 214 |
) -> Dict[str, Any]:
|
| 215 |
root = Path(
|
| 216 |
os.environ.get("GDW_RECEIPT_PROJECTION_DIR", "output/gdw/receipts")
|
| 217 |
).resolve()
|
| 218 |
+
_validate_artifact_id(artifact_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
claimed_digest = payload.get("receipt_hash")
|
| 220 |
unsigned_payload = dict(payload)
|
| 221 |
unsigned_payload.pop("receipt_hash", None)
|
|
|
|
| 223 |
raise ValueError("receipt_hash does not match canonical receipt")
|
| 224 |
artifact = _export_json_artifact(
|
| 225 |
root,
|
| 226 |
+
f"{artifact_id}.json",
|
| 227 |
payload,
|
| 228 |
owner_id or str(payload.get("owner_id") or ""),
|
| 229 |
)
|
| 230 |
+
artifact["artifact_identity"] = artifact_id
|
| 231 |
artifact.update(
|
| 232 |
{
|
| 233 |
"status": "RECEIPT_PROJECTED",
|
gdw_runtime.py
ADDED
|
@@ -0,0 +1,601 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fail-closed GDW storage preparation and supervised outbox draining."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import runpy
|
| 6 |
+
import sqlite3
|
| 7 |
+
import sys
|
| 8 |
+
import tempfile
|
| 9 |
+
import threading
|
| 10 |
+
import uuid
|
| 11 |
+
from datetime import datetime, timezone
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any, Mapping, Optional
|
| 14 |
+
|
| 15 |
+
from gdw_proofs import (
|
| 16 |
+
export_proof_payload,
|
| 17 |
+
export_receipt_projection,
|
| 18 |
+
)
|
| 19 |
+
from gdw_workspace import GDWWorkspace
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
if __name__ == "__main__":
|
| 23 |
+
# Let route modules import the same stateful module while this file is the
|
| 24 |
+
# process entry point.
|
| 25 |
+
sys.modules.setdefault("gdw_runtime", sys.modules[__name__])
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
ALLOWED_JOURNAL_MODES = {"DELETE", "WAL"}
|
| 29 |
+
ALLOWED_SYNCHRONOUS_MODES = {"FULL", "NORMAL"}
|
| 30 |
+
_STATE_LOCK = threading.RLock()
|
| 31 |
+
_STATE: dict[str, Any] = {
|
| 32 |
+
"startup_state": "NOT_RUN",
|
| 33 |
+
"evidence_label": "UNAVAILABLE",
|
| 34 |
+
"drain": {
|
| 35 |
+
"enabled": False,
|
| 36 |
+
"running": False,
|
| 37 |
+
"last_outcome": "NOT_RUN",
|
| 38 |
+
"last_attempt_at": None,
|
| 39 |
+
"last_success_at": None,
|
| 40 |
+
"last_error": None,
|
| 41 |
+
"last_report": None,
|
| 42 |
+
"run_generation_id": None,
|
| 43 |
+
"success_run_generation_id": None,
|
| 44 |
+
"success_database_generation_id": None,
|
| 45 |
+
"max_staleness_seconds": None,
|
| 46 |
+
},
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class GDWRuntimeError(RuntimeError):
|
| 51 |
+
"""Fail-closed GDW production-runtime configuration error."""
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _now() -> str:
|
| 55 |
+
return datetime.now(timezone.utc).isoformat()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _enabled(value: Optional[str]) -> bool:
|
| 59 |
+
return (value or "").strip().lower() in {"1", "true", "yes", "on"}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _bounded_int(
|
| 63 |
+
value: Optional[str],
|
| 64 |
+
*,
|
| 65 |
+
default: int,
|
| 66 |
+
minimum: int,
|
| 67 |
+
maximum: int,
|
| 68 |
+
name: str,
|
| 69 |
+
) -> int:
|
| 70 |
+
try:
|
| 71 |
+
parsed = int(value) if value not in (None, "") else default
|
| 72 |
+
except (TypeError, ValueError) as exc:
|
| 73 |
+
raise GDWRuntimeError(f"{name} must be an integer") from exc
|
| 74 |
+
if parsed < minimum or parsed > maximum:
|
| 75 |
+
raise GDWRuntimeError(
|
| 76 |
+
f"{name} must be between {minimum} and {maximum}"
|
| 77 |
+
)
|
| 78 |
+
return parsed
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _path_within(path: Path, root: Path, *, name: str) -> Path:
|
| 82 |
+
candidate = path.resolve()
|
| 83 |
+
try:
|
| 84 |
+
candidate.relative_to(root)
|
| 85 |
+
except ValueError as exc:
|
| 86 |
+
raise GDWRuntimeError(
|
| 87 |
+
f"{name} must be contained by required mount {root}"
|
| 88 |
+
) from exc
|
| 89 |
+
return candidate
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _verify_writable_directory(path: Path) -> None:
|
| 93 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 94 |
+
handle, probe = tempfile.mkstemp(prefix=".gdw-write-probe-", dir=path)
|
| 95 |
+
try:
|
| 96 |
+
with os.fdopen(handle, "wb") as stream:
|
| 97 |
+
stream.write(b"ok")
|
| 98 |
+
stream.flush()
|
| 99 |
+
os.fsync(stream.fileno())
|
| 100 |
+
finally:
|
| 101 |
+
Path(probe).unlink(missing_ok=True)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def storage_contract(
|
| 105 |
+
environ: Optional[Mapping[str, str]] = None,
|
| 106 |
+
) -> dict[str, Any]:
|
| 107 |
+
"""Resolve and validate the declared GDW storage contract without writes."""
|
| 108 |
+
|
| 109 |
+
values = os.environ if environ is None else environ
|
| 110 |
+
database = Path(
|
| 111 |
+
values.get("GDW_DB_PATH", "output/gdw/gdw.sqlite3")
|
| 112 |
+
).resolve()
|
| 113 |
+
proof_dir = Path(
|
| 114 |
+
values.get("GDW_PROOF_DIR", "output/proofs")
|
| 115 |
+
).resolve()
|
| 116 |
+
receipt_dir = Path(
|
| 117 |
+
values.get("GDW_RECEIPT_PROJECTION_DIR", "output/gdw/receipts")
|
| 118 |
+
).resolve()
|
| 119 |
+
required_mount_text = (values.get("GDW_REQUIRED_MOUNT") or "").strip()
|
| 120 |
+
persistent_required = _enabled(
|
| 121 |
+
values.get("GDW_REQUIRE_PERSISTENT_STORAGE")
|
| 122 |
+
)
|
| 123 |
+
journal_mode = (
|
| 124 |
+
values.get("GDW_SQLITE_JOURNAL") or "WAL"
|
| 125 |
+
).strip().upper()
|
| 126 |
+
synchronous = (
|
| 127 |
+
values.get("GDW_SQLITE_SYNCHRONOUS") or "NORMAL"
|
| 128 |
+
).strip().upper()
|
| 129 |
+
proof_export_mode = (
|
| 130 |
+
values.get("GDW_PROOF_EXPORT_MODE") or "outbox"
|
| 131 |
+
).strip().lower()
|
| 132 |
+
|
| 133 |
+
if journal_mode not in ALLOWED_JOURNAL_MODES:
|
| 134 |
+
raise GDWRuntimeError(
|
| 135 |
+
"GDW_SQLITE_JOURNAL must be one of "
|
| 136 |
+
+ ",".join(sorted(ALLOWED_JOURNAL_MODES))
|
| 137 |
+
)
|
| 138 |
+
if synchronous not in ALLOWED_SYNCHRONOUS_MODES:
|
| 139 |
+
raise GDWRuntimeError(
|
| 140 |
+
"GDW_SQLITE_SYNCHRONOUS must be one of "
|
| 141 |
+
+ ",".join(sorted(ALLOWED_SYNCHRONOUS_MODES))
|
| 142 |
+
)
|
| 143 |
+
if proof_export_mode != "outbox":
|
| 144 |
+
raise GDWRuntimeError(
|
| 145 |
+
"GDW_PROOF_EXPORT_MODE must be 'outbox'; synchronous export "
|
| 146 |
+
"is not transaction-safe"
|
| 147 |
+
)
|
| 148 |
+
if persistent_required and not required_mount_text:
|
| 149 |
+
raise GDWRuntimeError(
|
| 150 |
+
"GDW_REQUIRED_MOUNT is required when persistent storage is required"
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
mount: Optional[Path] = None
|
| 154 |
+
mount_verified = False
|
| 155 |
+
if required_mount_text:
|
| 156 |
+
mount = Path(required_mount_text).resolve()
|
| 157 |
+
database = _path_within(database, mount, name="GDW_DB_PATH")
|
| 158 |
+
proof_dir = _path_within(proof_dir, mount, name="GDW_PROOF_DIR")
|
| 159 |
+
receipt_dir = _path_within(
|
| 160 |
+
receipt_dir,
|
| 161 |
+
mount,
|
| 162 |
+
name="GDW_RECEIPT_PROJECTION_DIR",
|
| 163 |
+
)
|
| 164 |
+
mount_verified = os.path.ismount(str(mount))
|
| 165 |
+
if not mount_verified:
|
| 166 |
+
raise GDWRuntimeError(
|
| 167 |
+
f"required GDW storage mount is not attached: {mount}"
|
| 168 |
+
)
|
| 169 |
+
if persistent_required and journal_mode != "DELETE":
|
| 170 |
+
raise GDWRuntimeError(
|
| 171 |
+
"persistent GDW storage requires GDW_SQLITE_JOURNAL=DELETE"
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
return {
|
| 175 |
+
"persistence_required": persistent_required,
|
| 176 |
+
"required_mount": str(mount) if mount else None,
|
| 177 |
+
"mount_verified": mount_verified,
|
| 178 |
+
"database_path": str(database),
|
| 179 |
+
"proof_dir": str(proof_dir),
|
| 180 |
+
"receipt_projection_dir": str(receipt_dir),
|
| 181 |
+
"journal_mode_requested": journal_mode,
|
| 182 |
+
"synchronous_requested": synchronous,
|
| 183 |
+
"proof_export_mode": proof_export_mode,
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def prepare_runtime(
|
| 188 |
+
environ: Optional[Mapping[str, str]] = None,
|
| 189 |
+
) -> dict[str, Any]:
|
| 190 |
+
"""Verify durable paths, initialise SQLite, and select the declared journal."""
|
| 191 |
+
|
| 192 |
+
contract = storage_contract(environ)
|
| 193 |
+
database = Path(contract["database_path"])
|
| 194 |
+
proof_dir = Path(contract["proof_dir"])
|
| 195 |
+
receipt_dir = Path(contract["receipt_projection_dir"])
|
| 196 |
+
|
| 197 |
+
try:
|
| 198 |
+
_verify_writable_directory(database.parent)
|
| 199 |
+
_verify_writable_directory(proof_dir)
|
| 200 |
+
_verify_writable_directory(receipt_dir)
|
| 201 |
+
workspace = GDWWorkspace(
|
| 202 |
+
str(database),
|
| 203 |
+
namespace=(os.environ.get("GDW_NAMESPACE") or "a11oy"),
|
| 204 |
+
owner_id=(
|
| 205 |
+
os.environ.get("GDW_SERVICE_OWNER_ID") or "gdw-runtime"
|
| 206 |
+
),
|
| 207 |
+
production=False,
|
| 208 |
+
)
|
| 209 |
+
connection = sqlite3.connect(str(database), timeout=30)
|
| 210 |
+
try:
|
| 211 |
+
selected = connection.execute(
|
| 212 |
+
f"PRAGMA journal_mode={contract['journal_mode_requested']}"
|
| 213 |
+
).fetchone()[0]
|
| 214 |
+
selected = str(selected).upper()
|
| 215 |
+
if selected != contract["journal_mode_requested"]:
|
| 216 |
+
raise GDWRuntimeError(
|
| 217 |
+
"SQLite journal mode mismatch: requested "
|
| 218 |
+
f"{contract['journal_mode_requested']}, observed {selected}"
|
| 219 |
+
)
|
| 220 |
+
connection.execute(
|
| 221 |
+
f"PRAGMA synchronous={contract['synchronous_requested']}"
|
| 222 |
+
)
|
| 223 |
+
observed_synchronous = int(
|
| 224 |
+
connection.execute("PRAGMA synchronous").fetchone()[0]
|
| 225 |
+
)
|
| 226 |
+
integrity = str(
|
| 227 |
+
connection.execute("PRAGMA integrity_check").fetchone()[0]
|
| 228 |
+
)
|
| 229 |
+
if integrity != "ok":
|
| 230 |
+
raise GDWRuntimeError(
|
| 231 |
+
f"GDW SQLite integrity check failed: {integrity}"
|
| 232 |
+
)
|
| 233 |
+
finally:
|
| 234 |
+
connection.close()
|
| 235 |
+
observed = {
|
| 236 |
+
**contract,
|
| 237 |
+
"journal_mode_observed": selected,
|
| 238 |
+
"synchronous_observed": observed_synchronous,
|
| 239 |
+
"sqlite_integrity": integrity,
|
| 240 |
+
"schema_version": workspace.schema_version(),
|
| 241 |
+
"database_generation_id": workspace.database_generation_id,
|
| 242 |
+
"workspace_path": str(workspace.path),
|
| 243 |
+
}
|
| 244 |
+
except GDWRuntimeError:
|
| 245 |
+
raise
|
| 246 |
+
except Exception as exc:
|
| 247 |
+
raise GDWRuntimeError(
|
| 248 |
+
f"GDW persistent runtime preparation failed: {type(exc).__name__}"
|
| 249 |
+
) from exc
|
| 250 |
+
|
| 251 |
+
with _STATE_LOCK:
|
| 252 |
+
_STATE.update(
|
| 253 |
+
{
|
| 254 |
+
"startup_state": "READY",
|
| 255 |
+
"evidence_label": "VERIFIED",
|
| 256 |
+
"storage": observed,
|
| 257 |
+
"prepared_at": _now(),
|
| 258 |
+
"error": None,
|
| 259 |
+
}
|
| 260 |
+
)
|
| 261 |
+
return observed
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _verify_effect_binding(
|
| 265 |
+
workspace: GDWWorkspace,
|
| 266 |
+
row: Mapping[str, Any],
|
| 267 |
+
) -> None:
|
| 268 |
+
errors = workspace.effect_binding_errors_for_row(dict(row))
|
| 269 |
+
if errors:
|
| 270 |
+
raise ValueError("invalid effect binding: " + ",".join(errors))
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _export_effect(
|
| 274 |
+
workspace: GDWWorkspace,
|
| 275 |
+
row: Mapping[str, Any],
|
| 276 |
+
) -> dict[str, Any]:
|
| 277 |
+
_verify_effect_binding(workspace, row)
|
| 278 |
+
artifact_id = str(row["intent_sha256"])
|
| 279 |
+
if row["kind"] == "proof_export":
|
| 280 |
+
return export_proof_payload(
|
| 281 |
+
row["payload"],
|
| 282 |
+
artifact_id=artifact_id,
|
| 283 |
+
owner_id=str(row["owner_id"]),
|
| 284 |
+
)
|
| 285 |
+
if row["kind"] == "receipt_projection":
|
| 286 |
+
return export_receipt_projection(
|
| 287 |
+
row["payload"],
|
| 288 |
+
artifact_id,
|
| 289 |
+
owner_id=str(row["owner_id"]),
|
| 290 |
+
)
|
| 291 |
+
raise ValueError(f"unsupported effect kind: {row['kind']}")
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def drain_once(
|
| 295 |
+
*,
|
| 296 |
+
limit: int = 100,
|
| 297 |
+
lease_seconds: int = 300,
|
| 298 |
+
worker_id: Optional[str] = None,
|
| 299 |
+
workspace: Optional[GDWWorkspace] = None,
|
| 300 |
+
) -> dict[str, Any]:
|
| 301 |
+
"""Run one bounded drain pass and leave failed rows retryable."""
|
| 302 |
+
|
| 303 |
+
bounded = _bounded_int(
|
| 304 |
+
str(limit), default=100, minimum=1, maximum=1000, name="limit"
|
| 305 |
+
)
|
| 306 |
+
lease = _bounded_int(
|
| 307 |
+
str(lease_seconds),
|
| 308 |
+
default=300,
|
| 309 |
+
minimum=1,
|
| 310 |
+
maximum=3600,
|
| 311 |
+
name="lease_seconds",
|
| 312 |
+
)
|
| 313 |
+
store = workspace or GDWWorkspace(
|
| 314 |
+
namespace=(os.environ.get("GDW_NAMESPACE") or "a11oy"),
|
| 315 |
+
owner_id=(os.environ.get("GDW_SERVICE_OWNER_ID") or "gdw-runtime"),
|
| 316 |
+
)
|
| 317 |
+
owner = worker_id or f"gdw-drain-{os.getpid()}-{uuid.uuid4().hex[:12]}"
|
| 318 |
+
exported = 0
|
| 319 |
+
failed = 0
|
| 320 |
+
errors = []
|
| 321 |
+
identities = (
|
| 322 |
+
[(store.namespace, store.owner_id)]
|
| 323 |
+
if workspace is not None
|
| 324 |
+
else store.pending_effect_identities()
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
+
for namespace, owner_id in identities:
|
| 328 |
+
remaining = bounded - exported - failed
|
| 329 |
+
if remaining <= 0:
|
| 330 |
+
break
|
| 331 |
+
if not store.production:
|
| 332 |
+
for row in store.pending_proofs(
|
| 333 |
+
remaining,
|
| 334 |
+
namespace=namespace,
|
| 335 |
+
owner_id=owner_id,
|
| 336 |
+
):
|
| 337 |
+
try:
|
| 338 |
+
artifact = export_proof_payload(
|
| 339 |
+
row["payload"],
|
| 340 |
+
owner_id=owner_id,
|
| 341 |
+
)
|
| 342 |
+
store.mark_proof_exported(
|
| 343 |
+
row["proposal_id"],
|
| 344 |
+
artifact,
|
| 345 |
+
_now(),
|
| 346 |
+
namespace=namespace,
|
| 347 |
+
owner_id=owner_id,
|
| 348 |
+
)
|
| 349 |
+
exported += 1
|
| 350 |
+
except Exception as exc:
|
| 351 |
+
failed += 1
|
| 352 |
+
errors.append(f"legacy:{type(exc).__name__}")
|
| 353 |
+
if exported + failed >= bounded:
|
| 354 |
+
break
|
| 355 |
+
|
| 356 |
+
remaining = bounded - exported - failed
|
| 357 |
+
if remaining <= 0:
|
| 358 |
+
break
|
| 359 |
+
rows = store.claim_effects(
|
| 360 |
+
owner,
|
| 361 |
+
limit=remaining,
|
| 362 |
+
lease_seconds=lease,
|
| 363 |
+
namespace=namespace,
|
| 364 |
+
owner_id=owner_id,
|
| 365 |
+
)
|
| 366 |
+
for row in rows:
|
| 367 |
+
try:
|
| 368 |
+
store.assert_effect_claim(
|
| 369 |
+
row["idempotency_key"],
|
| 370 |
+
owner,
|
| 371 |
+
row["claim_generation"],
|
| 372 |
+
namespace=namespace,
|
| 373 |
+
owner_id=owner_id,
|
| 374 |
+
)
|
| 375 |
+
artifact = _export_effect(store, row)
|
| 376 |
+
store.mark_effect_exported(
|
| 377 |
+
row["idempotency_key"],
|
| 378 |
+
owner,
|
| 379 |
+
row["claim_generation"],
|
| 380 |
+
artifact,
|
| 381 |
+
_now(),
|
| 382 |
+
namespace=namespace,
|
| 383 |
+
owner_id=owner_id,
|
| 384 |
+
)
|
| 385 |
+
exported += 1
|
| 386 |
+
except Exception as exc:
|
| 387 |
+
try:
|
| 388 |
+
store.release_effect(
|
| 389 |
+
row["idempotency_key"],
|
| 390 |
+
owner,
|
| 391 |
+
row["claim_generation"],
|
| 392 |
+
f"{type(exc).__name__}: {str(exc)[:240]}",
|
| 393 |
+
namespace=namespace,
|
| 394 |
+
owner_id=owner_id,
|
| 395 |
+
)
|
| 396 |
+
except RuntimeError:
|
| 397 |
+
errors.append(f"{row['kind']}:CLAIM_LOST")
|
| 398 |
+
failed += 1
|
| 399 |
+
errors.append(f"{row['kind']}:{type(exc).__name__}")
|
| 400 |
+
|
| 401 |
+
integrity = store.integrity(global_scope=True)
|
| 402 |
+
return {
|
| 403 |
+
"attempted": exported + failed,
|
| 404 |
+
"exported": exported,
|
| 405 |
+
"failed": failed,
|
| 406 |
+
"pending_effects": integrity["pending_effects"],
|
| 407 |
+
"legacy_pending_proofs": integrity["pending_proofs"],
|
| 408 |
+
"sqlite_integrity": integrity["sqlite_integrity"],
|
| 409 |
+
"errors": errors,
|
| 410 |
+
}
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def _set_drain_state(**values: Any) -> None:
|
| 414 |
+
with _STATE_LOCK:
|
| 415 |
+
drain = dict(_STATE["drain"])
|
| 416 |
+
drain.update(values)
|
| 417 |
+
_STATE["drain"] = drain
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
class OutboxSupervisor:
|
| 421 |
+
"""Single-process bounded outbox worker with retry backoff."""
|
| 422 |
+
|
| 423 |
+
def __init__(
|
| 424 |
+
self,
|
| 425 |
+
*,
|
| 426 |
+
enabled: bool,
|
| 427 |
+
interval_seconds: int,
|
| 428 |
+
retry_max_seconds: int,
|
| 429 |
+
batch_size: int,
|
| 430 |
+
lease_seconds: int,
|
| 431 |
+
) -> None:
|
| 432 |
+
self.enabled = enabled
|
| 433 |
+
self.interval_seconds = interval_seconds
|
| 434 |
+
self.retry_max_seconds = retry_max_seconds
|
| 435 |
+
self.batch_size = batch_size
|
| 436 |
+
self.lease_seconds = lease_seconds
|
| 437 |
+
self.worker_id = f"gdw-supervisor-{os.getpid()}-{uuid.uuid4().hex[:12]}"
|
| 438 |
+
self._stop = threading.Event()
|
| 439 |
+
self._thread: Optional[threading.Thread] = None
|
| 440 |
+
|
| 441 |
+
@classmethod
|
| 442 |
+
def from_environment(cls) -> "OutboxSupervisor":
|
| 443 |
+
return cls(
|
| 444 |
+
enabled=_enabled(os.environ.get("GDW_OUTBOX_ENABLED")),
|
| 445 |
+
interval_seconds=_bounded_int(
|
| 446 |
+
os.environ.get("GDW_OUTBOX_INTERVAL_SECONDS"),
|
| 447 |
+
default=5,
|
| 448 |
+
minimum=1,
|
| 449 |
+
maximum=3600,
|
| 450 |
+
name="GDW_OUTBOX_INTERVAL_SECONDS",
|
| 451 |
+
),
|
| 452 |
+
retry_max_seconds=_bounded_int(
|
| 453 |
+
os.environ.get("GDW_OUTBOX_RETRY_MAX_SECONDS"),
|
| 454 |
+
default=60,
|
| 455 |
+
minimum=1,
|
| 456 |
+
maximum=3600,
|
| 457 |
+
name="GDW_OUTBOX_RETRY_MAX_SECONDS",
|
| 458 |
+
),
|
| 459 |
+
batch_size=_bounded_int(
|
| 460 |
+
os.environ.get("GDW_OUTBOX_BATCH_SIZE"),
|
| 461 |
+
default=100,
|
| 462 |
+
minimum=1,
|
| 463 |
+
maximum=1000,
|
| 464 |
+
name="GDW_OUTBOX_BATCH_SIZE",
|
| 465 |
+
),
|
| 466 |
+
lease_seconds=_bounded_int(
|
| 467 |
+
os.environ.get("GDW_OUTBOX_LEASE_SECONDS"),
|
| 468 |
+
default=300,
|
| 469 |
+
minimum=1,
|
| 470 |
+
maximum=3600,
|
| 471 |
+
name="GDW_OUTBOX_LEASE_SECONDS",
|
| 472 |
+
),
|
| 473 |
+
)
|
| 474 |
+
|
| 475 |
+
def start(self) -> None:
|
| 476 |
+
_set_drain_state(enabled=self.enabled)
|
| 477 |
+
if not self.enabled:
|
| 478 |
+
_set_drain_state(last_outcome="DISABLED")
|
| 479 |
+
return
|
| 480 |
+
if self._thread and self._thread.is_alive():
|
| 481 |
+
return
|
| 482 |
+
self._thread = threading.Thread(
|
| 483 |
+
target=self._run,
|
| 484 |
+
name="gdw-outbox-supervisor",
|
| 485 |
+
daemon=True,
|
| 486 |
+
)
|
| 487 |
+
self._thread.start()
|
| 488 |
+
|
| 489 |
+
def _run(self) -> None:
|
| 490 |
+
run_generation_id = uuid.uuid4().hex
|
| 491 |
+
with _STATE_LOCK:
|
| 492 |
+
database_generation_id = str(
|
| 493 |
+
(_STATE.get("storage") or {}).get("database_generation_id") or ""
|
| 494 |
+
)
|
| 495 |
+
_set_drain_state(
|
| 496 |
+
running=True,
|
| 497 |
+
worker_id=self.worker_id,
|
| 498 |
+
last_outcome="STARTING",
|
| 499 |
+
last_attempt_at=None,
|
| 500 |
+
last_success_at=None,
|
| 501 |
+
last_error=None,
|
| 502 |
+
last_report=None,
|
| 503 |
+
run_generation_id=run_generation_id,
|
| 504 |
+
success_run_generation_id=None,
|
| 505 |
+
success_database_generation_id=None,
|
| 506 |
+
max_staleness_seconds=max(30, self.interval_seconds * 3),
|
| 507 |
+
)
|
| 508 |
+
delay = 0
|
| 509 |
+
retry_delay = self.interval_seconds
|
| 510 |
+
try:
|
| 511 |
+
while not self._stop.wait(delay):
|
| 512 |
+
attempted_at = _now()
|
| 513 |
+
_set_drain_state(last_attempt_at=attempted_at)
|
| 514 |
+
try:
|
| 515 |
+
report = drain_once(
|
| 516 |
+
limit=self.batch_size,
|
| 517 |
+
lease_seconds=self.lease_seconds,
|
| 518 |
+
worker_id=self.worker_id,
|
| 519 |
+
)
|
| 520 |
+
if report["failed"] or report["legacy_pending_proofs"]:
|
| 521 |
+
retry_delay = min(
|
| 522 |
+
self.retry_max_seconds,
|
| 523 |
+
max(self.interval_seconds, retry_delay * 2),
|
| 524 |
+
)
|
| 525 |
+
delay = retry_delay
|
| 526 |
+
_set_drain_state(
|
| 527 |
+
last_outcome="RETRY_SCHEDULED",
|
| 528 |
+
last_error=(
|
| 529 |
+
"bounded drain pass reported failures or "
|
| 530 |
+
"unmigrated legacy proofs"
|
| 531 |
+
),
|
| 532 |
+
last_report=report,
|
| 533 |
+
)
|
| 534 |
+
else:
|
| 535 |
+
retry_delay = self.interval_seconds
|
| 536 |
+
delay = self.interval_seconds
|
| 537 |
+
_set_drain_state(
|
| 538 |
+
last_outcome="SUCCEEDED",
|
| 539 |
+
last_success_at=_now(),
|
| 540 |
+
last_error=None,
|
| 541 |
+
last_report=report,
|
| 542 |
+
success_run_generation_id=run_generation_id,
|
| 543 |
+
success_database_generation_id=(
|
| 544 |
+
database_generation_id
|
| 545 |
+
),
|
| 546 |
+
)
|
| 547 |
+
except Exception as exc:
|
| 548 |
+
retry_delay = min(
|
| 549 |
+
self.retry_max_seconds,
|
| 550 |
+
max(self.interval_seconds, retry_delay * 2),
|
| 551 |
+
)
|
| 552 |
+
delay = retry_delay
|
| 553 |
+
_set_drain_state(
|
| 554 |
+
last_outcome="RETRY_SCHEDULED",
|
| 555 |
+
last_error=f"{type(exc).__name__}: {str(exc)[:240]}",
|
| 556 |
+
)
|
| 557 |
+
finally:
|
| 558 |
+
_set_drain_state(running=False)
|
| 559 |
+
|
| 560 |
+
def stop(self, timeout_seconds: float = 10.0) -> None:
|
| 561 |
+
self._stop.set()
|
| 562 |
+
if self._thread:
|
| 563 |
+
self._thread.join(timeout=max(0.0, timeout_seconds))
|
| 564 |
+
_set_drain_state(running=False)
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def runtime_health() -> dict[str, Any]:
|
| 568 |
+
"""Return secret-free observed inputs for the GDW health route."""
|
| 569 |
+
|
| 570 |
+
with _STATE_LOCK:
|
| 571 |
+
return json.loads(json.dumps(_STATE))
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def main() -> int:
|
| 575 |
+
try:
|
| 576 |
+
prepare_runtime()
|
| 577 |
+
except Exception as exc:
|
| 578 |
+
with _STATE_LOCK:
|
| 579 |
+
_STATE.update(
|
| 580 |
+
{
|
| 581 |
+
"startup_state": "BLOCKED",
|
| 582 |
+
"evidence_label": "VERIFIED",
|
| 583 |
+
"error": f"{type(exc).__name__}: {str(exc)[:240]}",
|
| 584 |
+
}
|
| 585 |
+
)
|
| 586 |
+
raise
|
| 587 |
+
|
| 588 |
+
supervisor = OutboxSupervisor.from_environment()
|
| 589 |
+
supervisor.start()
|
| 590 |
+
try:
|
| 591 |
+
runpy.run_path(
|
| 592 |
+
str(Path(__file__).with_name("serve.py")),
|
| 593 |
+
run_name="__main__",
|
| 594 |
+
)
|
| 595 |
+
finally:
|
| 596 |
+
supervisor.stop()
|
| 597 |
+
return 0
|
| 598 |
+
|
| 599 |
+
|
| 600 |
+
if __name__ == "__main__":
|
| 601 |
+
raise SystemExit(main())
|
gdw_workspace.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
routers/gdw_frontier.py
CHANGED
|
@@ -1,12 +1,12 @@
|
|
| 1 |
"""Authenticated Governed Delta Workspace API and benchmark surfaces."""
|
| 2 |
|
| 3 |
import hashlib
|
| 4 |
-
import hmac
|
| 5 |
import json
|
| 6 |
import os
|
| 7 |
import re
|
|
|
|
| 8 |
import time
|
| 9 |
-
from datetime import datetime,
|
| 10 |
from typing import List, Literal, Optional
|
| 11 |
|
| 12 |
from fastapi import Header, HTTPException, Request
|
|
@@ -14,17 +14,31 @@ from fastapi.responses import PlainTextResponse
|
|
| 14 |
from pydantic import BaseModel, Field, ValidationError
|
| 15 |
|
| 16 |
from gdw_attention import AttentionFeatures, choose_attention_mode
|
| 17 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
from gdw_proofs import build_proof_payload, sha256_json
|
|
|
|
| 19 |
from gdw_telemetry import GDWTelemetry
|
| 20 |
-
from gdw_workspace import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
from szl_sgh_scheduler import build_plan
|
| 22 |
|
| 23 |
|
| 24 |
_TELEMETRY = GDWTelemetry()
|
| 25 |
_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
| 26 |
_EXPERTS = {"planner", "retriever", "auditor", "verifier", "operator"}
|
| 27 |
-
|
|
|
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
class GDWStepRequest(BaseModel):
|
|
@@ -52,6 +66,24 @@ def _dump_model(model):
|
|
| 52 |
return model.dict()
|
| 53 |
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
def _now() -> str:
|
| 56 |
return datetime.now(timezone.utc).isoformat()
|
| 57 |
|
|
@@ -63,171 +95,241 @@ def _sha(value) -> str:
|
|
| 63 |
return hashlib.sha256(encoded).hexdigest()
|
| 64 |
|
| 65 |
|
| 66 |
-
def
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
or any(ch not in "0123456789abcdef" for ch in token_sha256)
|
| 88 |
-
or token_sha256 == hashlib.sha256(b"").hexdigest()
|
| 89 |
-
or token_sha256 in token_digests
|
| 90 |
-
):
|
| 91 |
-
raise RuntimeError("GDW principal token binding is invalid")
|
| 92 |
-
roles = record.get("roles")
|
| 93 |
-
if (
|
| 94 |
-
not isinstance(roles, list)
|
| 95 |
-
or not roles
|
| 96 |
-
or not set(roles).issubset(_PRINCIPAL_ROLES)
|
| 97 |
-
):
|
| 98 |
-
raise RuntimeError("GDW principal roles are invalid")
|
| 99 |
-
token_digests.add(token_sha256)
|
| 100 |
-
normalized[principal_id] = {
|
| 101 |
-
"principal_id": principal_id,
|
| 102 |
-
"token_sha256": token_sha256,
|
| 103 |
-
"roles": sorted(set(roles)),
|
| 104 |
}
|
| 105 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
|
| 108 |
-
def
|
| 109 |
authorization: Optional[str],
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
| 112 |
try:
|
| 113 |
-
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
raise HTTPException(
|
| 116 |
status_code=503,
|
| 117 |
-
detail="GDW
|
| 118 |
) from exc
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
if required_role and required_role not in principal["roles"]:
|
| 133 |
-
raise HTTPException(status_code=403, detail="principal role is insufficient")
|
| 134 |
-
return principal
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
def _bounded_config(name: str, default: int, maximum: int) -> int:
|
| 138 |
-
try:
|
| 139 |
-
value = int(os.environ.get(name, str(default)))
|
| 140 |
-
except Exception as exc:
|
| 141 |
-
raise RuntimeError(f"{name} must be an integer") from exc
|
| 142 |
-
if value < 1 or value > maximum:
|
| 143 |
-
raise RuntimeError(f"{name} must be between 1 and {maximum}")
|
| 144 |
-
return value
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
def _admission_limits() -> dict:
|
| 148 |
-
owner_requests = _bounded_config("GDW_OWNER_MAX_REQUESTS", 1000, 10000)
|
| 149 |
-
owner_sessions = _bounded_config("GDW_OWNER_MAX_SESSIONS", 100, 1000)
|
| 150 |
-
global_requests = _bounded_config(
|
| 151 |
-
"GDW_GLOBAL_MAX_REQUESTS", 100000, 1000000
|
| 152 |
)
|
| 153 |
-
global_sessions = _bounded_config(
|
| 154 |
-
"GDW_GLOBAL_MAX_SESSIONS", 10000, 100000
|
| 155 |
-
)
|
| 156 |
-
if global_requests < owner_requests or global_sessions < owner_sessions:
|
| 157 |
-
raise RuntimeError("GDW global quotas cannot be lower than owner quotas")
|
| 158 |
-
return {
|
| 159 |
-
"owner_requests": owner_requests,
|
| 160 |
-
"owner_sessions": owner_sessions,
|
| 161 |
-
"global_requests": global_requests,
|
| 162 |
-
"global_sessions": global_sessions,
|
| 163 |
-
}
|
| 164 |
|
| 165 |
|
| 166 |
-
def
|
| 167 |
-
|
| 168 |
-
|
|
|
|
| 169 |
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
"GDW_GLOBAL_MAX_ARTIFACTS", 100000, 1000000
|
| 176 |
-
)
|
| 177 |
-
if global_artifacts < owner_artifacts:
|
| 178 |
-
raise RuntimeError(
|
| 179 |
-
"GDW global artifact quota cannot be lower than owner quota"
|
| 180 |
)
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
"global_artifacts": global_artifacts,
|
| 184 |
-
"max_attempts": _bounded_config("GDW_MAX_EFFECT_ATTEMPTS", 20, 100),
|
| 185 |
-
}
|
| 186 |
|
| 187 |
|
| 188 |
-
def
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
|
| 211 |
-
def
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
except Exception as exc:
|
| 215 |
raise HTTPException(
|
| 216 |
status_code=503,
|
| 217 |
-
detail=
|
| 218 |
-
|
|
|
|
|
|
|
|
|
|
| 219 |
|
| 220 |
|
| 221 |
-
def
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
}
|
| 232 |
|
| 233 |
|
|
@@ -259,7 +361,7 @@ def _governance_gate(
|
|
| 259 |
payload_data: dict,
|
| 260 |
request_id: str,
|
| 261 |
request_digest: str,
|
| 262 |
-
|
| 263 |
) -> dict:
|
| 264 |
action = {
|
| 265 |
"tool": "execute",
|
|
@@ -269,17 +371,22 @@ def _governance_gate(
|
|
| 269 |
"target": payload_data["session_id"],
|
| 270 |
"request_id": request_id,
|
| 271 |
"request_digest": request_digest,
|
| 272 |
-
"
|
| 273 |
-
"
|
|
|
|
| 274 |
"text": payload_data["request"],
|
| 275 |
"high_impact": float(payload_data["risk_budget"]) >= 0.75,
|
| 276 |
"irreversible": False,
|
| 277 |
}
|
| 278 |
try:
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
except Exception as exc:
|
| 284 |
return {
|
| 285 |
"allowed": False,
|
|
@@ -287,7 +394,11 @@ def _governance_gate(
|
|
| 287 |
"reason_codes": ["DOCTRINE_GATE_UNAVAILABLE"],
|
| 288 |
"detail": type(exc).__name__,
|
| 289 |
"writer_is_judge": True,
|
| 290 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
}
|
| 292 |
|
| 293 |
try:
|
|
@@ -306,13 +417,17 @@ def _governance_gate(
|
|
| 306 |
"reason_codes": ["CODENAME_GATE_UNAVAILABLE"],
|
| 307 |
"detail": type(exc).__name__,
|
| 308 |
"writer_is_judge": True,
|
| 309 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
"colang": {
|
| 311 |
"decision": colang.get("decision"),
|
| 312 |
"fired_flows": colang.get("fired_flows", []),
|
| 313 |
"flows_evaluated": colang.get("flows_evaluated", []),
|
| 314 |
"policy_files": colang.get("policy_files", []),
|
| 315 |
-
"
|
| 316 |
},
|
| 317 |
}
|
| 318 |
|
|
@@ -324,15 +439,19 @@ def _governance_gate(
|
|
| 324 |
return {
|
| 325 |
"allowed": not reasons,
|
| 326 |
"decision": "ALLOW" if not reasons else "DENY",
|
| 327 |
-
"reason_codes": reasons or ["
|
| 328 |
"writer_is_judge": True,
|
| 329 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
"colang": {
|
| 331 |
"decision": colang.get("decision"),
|
| 332 |
"fired_flows": colang.get("fired_flows", []),
|
| 333 |
"flows_evaluated": colang.get("flows_evaluated", []),
|
| 334 |
"policy_files": colang.get("policy_files", []),
|
| 335 |
-
"
|
| 336 |
},
|
| 337 |
"codename_gate": {
|
| 338 |
"clean": not codename_hits,
|
|
@@ -346,14 +465,14 @@ def _atomic_receipt(
|
|
| 346 |
proposal_id: str,
|
| 347 |
request_id: str,
|
| 348 |
request_digest: str,
|
| 349 |
-
owner_id: str,
|
| 350 |
-
generation_id: str,
|
| 351 |
session_id: str,
|
| 352 |
step: int,
|
| 353 |
before_hash: str,
|
| 354 |
after_hash: str,
|
| 355 |
scheduler_mode: str,
|
| 356 |
governance: dict,
|
|
|
|
|
|
|
| 357 |
timestamp: str,
|
| 358 |
) -> dict:
|
| 359 |
receipt = {
|
|
@@ -362,9 +481,11 @@ def _atomic_receipt(
|
|
| 362 |
"proposal_id": proposal_id,
|
| 363 |
"request_id": request_id,
|
| 364 |
"request_digest": request_digest,
|
| 365 |
-
"owner_id": owner_id,
|
| 366 |
-
"generation_id": generation_id,
|
| 367 |
"session_id": session_id,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 368 |
"step": step,
|
| 369 |
"state_before_hash": before_hash,
|
| 370 |
"state_after_hash": after_hash,
|
|
@@ -377,81 +498,41 @@ def _atomic_receipt(
|
|
| 377 |
return receipt
|
| 378 |
|
| 379 |
|
| 380 |
-
def _effect_key(
|
| 381 |
-
*,
|
| 382 |
-
generation_id: str,
|
| 383 |
-
owner_id: str,
|
| 384 |
-
request_id: str,
|
| 385 |
-
request_digest: str,
|
| 386 |
-
kind: str,
|
| 387 |
-
canonical_identity: str,
|
| 388 |
-
payload_sha256: str,
|
| 389 |
-
) -> str:
|
| 390 |
-
return hashlib.sha256(
|
| 391 |
-
(
|
| 392 |
-
f"{generation_id}:{owner_id}:{request_id}:{request_digest}:"
|
| 393 |
-
f"{kind}:{canonical_identity}:{payload_sha256}"
|
| 394 |
-
).encode("utf-8")
|
| 395 |
-
).hexdigest()
|
| 396 |
-
|
| 397 |
-
|
| 398 |
def register(app, ns: str = "a11oy"):
|
| 399 |
prefix = f"/api/{ns}/v1/gdw"
|
| 400 |
|
| 401 |
@app.get(prefix + "/healthz")
|
| 402 |
@app.get("/v1/gdw/healthz")
|
| 403 |
def gdw_healthz():
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
"external_effects": "DISABLED",
|
| 423 |
-
"reason": f"semantic gate closed: {type(exc).__name__}",
|
| 424 |
-
"benchmark_claim": "UNMEASURED",
|
| 425 |
-
}
|
| 426 |
-
|
| 427 |
-
@app.post(prefix + "/drain")
|
| 428 |
-
@app.post("/v1/gdw/drain")
|
| 429 |
-
def gdw_drain(
|
| 430 |
-
limit: int = 100,
|
| 431 |
-
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 432 |
-
):
|
| 433 |
-
_authenticate(authorization, "admin")
|
| 434 |
-
workspace = _available_workspace()
|
| 435 |
-
try:
|
| 436 |
-
result = drain_effects(workspace, limit=limit)
|
| 437 |
-
except ValueError as exc:
|
| 438 |
-
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
| 439 |
-
except Exception as exc:
|
| 440 |
-
raise HTTPException(
|
| 441 |
-
status_code=503,
|
| 442 |
-
detail=f"GDW effect drain failed closed: {type(exc).__name__}",
|
| 443 |
-
) from exc
|
| 444 |
-
if result["failed"] or not result["integrity_ok"]:
|
| 445 |
-
raise HTTPException(status_code=503, detail=result)
|
| 446 |
-
return result
|
| 447 |
|
| 448 |
@app.get(prefix + "/bench/meta")
|
| 449 |
@app.get("/v1/gdw/bench/meta")
|
| 450 |
def gdw_bench_meta(
|
| 451 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 452 |
):
|
| 453 |
-
|
| 454 |
-
|
|
|
|
|
|
|
|
|
|
| 455 |
return {
|
| 456 |
"service": "gdw-frontier",
|
| 457 |
"implementation_status": "REAL",
|
|
@@ -469,8 +550,11 @@ def register(app, ns: str = "a11oy"):
|
|
| 469 |
def gdw_metrics(
|
| 470 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 471 |
):
|
| 472 |
-
|
| 473 |
-
|
|
|
|
|
|
|
|
|
|
| 474 |
return PlainTextResponse(
|
| 475 |
_TELEMETRY.render(),
|
| 476 |
media_type="text/plain; version=0.0.4; charset=utf-8",
|
|
@@ -481,9 +565,45 @@ def register(app, ns: str = "a11oy"):
|
|
| 481 |
def gdw_integrity(
|
| 482 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 483 |
):
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 487 |
|
| 488 |
@app.get(prefix + "/sessions/{session_id}")
|
| 489 |
@app.get("/v1/gdw/sessions/{session_id}")
|
|
@@ -491,57 +611,62 @@ def register(app, ns: str = "a11oy"):
|
|
| 491 |
session_id: str,
|
| 492 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 493 |
):
|
| 494 |
-
principal =
|
| 495 |
-
|
|
|
|
|
|
|
|
|
|
| 496 |
if not _ID_PATTERN.fullmatch(session_id):
|
| 497 |
raise HTTPException(status_code=422, detail="invalid session_id")
|
| 498 |
-
|
| 499 |
-
state = workspace.read_session(session_id, principal["principal_id"])
|
| 500 |
-
except PermissionError as exc:
|
| 501 |
-
raise HTTPException(status_code=403, detail=str(exc)) from exc
|
| 502 |
if state is None:
|
| 503 |
raise HTTPException(status_code=404, detail="session not found")
|
| 504 |
return state
|
| 505 |
|
| 506 |
-
|
| 507 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
async def gdw_step(
|
| 509 |
request: Request,
|
| 510 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 511 |
x_request_id: Optional[str] = Header(default=None, alias="X-Request-Id"),
|
| 512 |
):
|
| 513 |
started = time.perf_counter()
|
| 514 |
-
principal =
|
| 515 |
-
|
| 516 |
-
|
|
|
|
|
|
|
|
|
|
| 517 |
try:
|
| 518 |
raw_payload = await request.json()
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
exc.errors()
|
| 526 |
-
if isinstance(exc, ValidationError)
|
| 527 |
-
else "request body must be valid JSON"
|
| 528 |
-
)
|
| 529 |
-
raise HTTPException(status_code=422, detail=detail) from exc
|
| 530 |
request_id = _validate_identifiers(payload, x_request_id)
|
| 531 |
payload_data = _dump_model(payload)
|
| 532 |
request_digest = _sha(payload_data)
|
| 533 |
-
generation_id = workspace.generation_id()
|
| 534 |
-
limits = _admission_limits()
|
| 535 |
-
retention_seconds = _retention_seconds()
|
| 536 |
selected_mode = "unresolved"
|
| 537 |
decision = "ERROR"
|
| 538 |
receipt_hash = ""
|
| 539 |
|
| 540 |
try:
|
|
|
|
| 541 |
with workspace.transaction() as connection:
|
| 542 |
-
cached = workspace.cached_request(
|
| 543 |
-
connection, request_id, principal_id
|
| 544 |
-
)
|
| 545 |
if cached is not None:
|
| 546 |
cached_digest, cached_response = cached
|
| 547 |
if cached_digest != request_digest:
|
|
@@ -549,6 +674,18 @@ def register(app, ns: str = "a11oy"):
|
|
| 549 |
status_code=409,
|
| 550 |
detail="X-Request-Id was already used with different content",
|
| 551 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 552 |
cached_response["replayed"] = True
|
| 553 |
selected_mode = cached_response["scheduler_mode"]
|
| 554 |
decision = cached_response["decision"]
|
|
@@ -566,18 +703,14 @@ def register(app, ns: str = "a11oy"):
|
|
| 566 |
before_step = 0
|
| 567 |
before_hash = _sha(
|
| 568 |
{
|
|
|
|
|
|
|
| 569 |
"session_id": payload.session_id,
|
| 570 |
"step": 0,
|
| 571 |
"state": "GENESIS",
|
| 572 |
}
|
| 573 |
)
|
| 574 |
else:
|
| 575 |
-
workspace.require_object_owner(
|
| 576 |
-
connection,
|
| 577 |
-
"session",
|
| 578 |
-
payload.session_id,
|
| 579 |
-
principal_id,
|
| 580 |
-
)
|
| 581 |
before_step = previous["step"]
|
| 582 |
before_hash = previous["state_hash"]
|
| 583 |
|
|
@@ -598,48 +731,34 @@ def register(app, ns: str = "a11oy"):
|
|
| 598 |
selected_mode = routing["mode"]
|
| 599 |
precondition_decision = _decision(payload)
|
| 600 |
governance = _governance_gate(
|
| 601 |
-
payload_data,
|
| 602 |
-
request_id,
|
| 603 |
-
request_digest,
|
| 604 |
-
principal_id,
|
| 605 |
)
|
| 606 |
decision = precondition_decision
|
| 607 |
if decision == "ACCEPT" and not governance["allowed"]:
|
| 608 |
decision = "REJECT"
|
| 609 |
mutates = decision == "ACCEPT" and not payload.dry_run
|
| 610 |
step = before_step + 1 if mutates else before_step
|
| 611 |
-
|
| 612 |
-
expires_at = (
|
| 613 |
-
datetime.now(timezone.utc)
|
| 614 |
-
+ timedelta(seconds=retention_seconds)
|
| 615 |
-
).isoformat()
|
| 616 |
-
workspace.admit_request(
|
| 617 |
-
connection,
|
| 618 |
-
owner_id=principal_id,
|
| 619 |
-
request_id=request_id,
|
| 620 |
-
session_id=payload.session_id,
|
| 621 |
-
mutates=mutates,
|
| 622 |
-
created_at=timestamp,
|
| 623 |
-
expires_at=expires_at,
|
| 624 |
-
limits=limits,
|
| 625 |
-
)
|
| 626 |
proposal_id = sha256_json(
|
| 627 |
{
|
| 628 |
"schema": "szl.gdw.proposal-identity/v1",
|
| 629 |
-
"
|
| 630 |
-
"
|
|
|
|
| 631 |
"request_id": request_id,
|
| 632 |
"request_digest": request_digest,
|
| 633 |
"state_before_hash": before_hash,
|
| 634 |
"governance_evidence_sha256": sha256_json(governance),
|
| 635 |
}
|
| 636 |
)
|
|
|
|
| 637 |
|
| 638 |
if mutates:
|
| 639 |
state = {
|
|
|
|
|
|
|
| 640 |
"session_id": payload.session_id,
|
| 641 |
-
"
|
| 642 |
-
"generation_id": generation_id,
|
| 643 |
"step": step,
|
| 644 |
"previous_state_hash": before_hash,
|
| 645 |
"request_digest": request_digest,
|
|
@@ -659,14 +778,14 @@ def register(app, ns: str = "a11oy"):
|
|
| 659 |
proposal_id=proposal_id,
|
| 660 |
request_id=request_id,
|
| 661 |
request_digest=request_digest,
|
| 662 |
-
owner_id=principal_id,
|
| 663 |
-
generation_id=generation_id,
|
| 664 |
session_id=payload.session_id,
|
| 665 |
step=step,
|
| 666 |
before_hash=before_hash,
|
| 667 |
after_hash=after_hash,
|
| 668 |
scheduler_mode=selected_mode,
|
| 669 |
governance=governance,
|
|
|
|
|
|
|
| 670 |
timestamp=timestamp,
|
| 671 |
)
|
| 672 |
receipt_hash = receipt["receipt_hash"]
|
|
@@ -679,8 +798,9 @@ def register(app, ns: str = "a11oy"):
|
|
| 679 |
proposal_id=proposal_id,
|
| 680 |
request_id=request_id,
|
| 681 |
request_digest=request_digest,
|
| 682 |
-
|
| 683 |
-
|
|
|
|
| 684 |
step=step,
|
| 685 |
before_hash=before_hash,
|
| 686 |
after_hash=after_hash,
|
|
@@ -698,21 +818,16 @@ def register(app, ns: str = "a11oy"):
|
|
| 698 |
"GDW_PROOF_EXPORT_MODE must be 'outbox'; "
|
| 699 |
"synchronous external effects are not transaction-safe"
|
| 700 |
)
|
| 701 |
-
proof_payload_digest = sha256_json(proof_payload)
|
| 702 |
-
proof_effect_key = _effect_key(
|
| 703 |
-
generation_id=generation_id,
|
| 704 |
-
owner_id=principal_id,
|
| 705 |
-
request_id=request_id,
|
| 706 |
-
request_digest=request_digest,
|
| 707 |
-
kind="proof_export",
|
| 708 |
-
canonical_identity=proof_payload["payload_sha256"],
|
| 709 |
-
payload_sha256=proof_payload_digest,
|
| 710 |
-
)
|
| 711 |
proof_artifact = {
|
| 712 |
"status": "OUTBOX_PENDING",
|
| 713 |
"kind": "proof_export",
|
| 714 |
-
"idempotency_key":
|
| 715 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 716 |
"payload_sha256": proof_payload["payload_sha256"],
|
| 717 |
"formal_status": "NOT_RUN",
|
| 718 |
}
|
|
@@ -726,9 +841,14 @@ def register(app, ns: str = "a11oy"):
|
|
| 726 |
"benchmark_status": "UNMEASURED",
|
| 727 |
"proposal_id": proposal_id,
|
| 728 |
"request_id": request_id,
|
| 729 |
-
"
|
| 730 |
-
"
|
| 731 |
"session_id": payload.session_id,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
"decision": decision,
|
| 733 |
"step": step,
|
| 734 |
"state_hash": after_hash,
|
|
@@ -740,10 +860,6 @@ def register(app, ns: str = "a11oy"):
|
|
| 740 |
"kernel_execution": "NOT_EXECUTED_BY_CONTROL_API",
|
| 741 |
"dry_run": payload.dry_run,
|
| 742 |
"replayed": False,
|
| 743 |
-
"retention": {
|
| 744 |
-
"expires_at": expires_at,
|
| 745 |
-
"seconds": retention_seconds,
|
| 746 |
-
},
|
| 747 |
"audit": {
|
| 748 |
"governance": governance,
|
| 749 |
"precondition_decision": precondition_decision,
|
|
@@ -767,6 +883,7 @@ def register(app, ns: str = "a11oy"):
|
|
| 767 |
timestamp,
|
| 768 |
)
|
| 769 |
if receipt is not None:
|
|
|
|
| 770 |
workspace.save_receipt(
|
| 771 |
connection,
|
| 772 |
receipt_hash,
|
|
@@ -776,38 +893,34 @@ def register(app, ns: str = "a11oy"):
|
|
| 776 |
receipt,
|
| 777 |
timestamp,
|
| 778 |
)
|
| 779 |
-
receipt_payload_digest = sha256_json(receipt)
|
| 780 |
-
receipt_effect_key = _effect_key(
|
| 781 |
-
generation_id=generation_id,
|
| 782 |
-
owner_id=principal_id,
|
| 783 |
-
request_id=request_id,
|
| 784 |
-
request_digest=request_digest,
|
| 785 |
-
kind="receipt_projection",
|
| 786 |
-
canonical_identity=receipt_hash,
|
| 787 |
-
payload_sha256=receipt_payload_digest,
|
| 788 |
-
)
|
| 789 |
workspace.save_effect_outbox(
|
| 790 |
connection,
|
| 791 |
request_id,
|
| 792 |
"receipt_projection",
|
| 793 |
-
generation_id,
|
| 794 |
-
principal_id,
|
| 795 |
-
receipt_hash,
|
| 796 |
receipt,
|
| 797 |
-
|
| 798 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 799 |
timestamp,
|
| 800 |
)
|
| 801 |
workspace.save_effect_outbox(
|
| 802 |
connection,
|
| 803 |
request_id,
|
| 804 |
"proof_export",
|
| 805 |
-
generation_id,
|
| 806 |
-
principal_id,
|
| 807 |
-
proof_payload["payload_sha256"],
|
| 808 |
proof_payload,
|
| 809 |
-
|
| 810 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 811 |
timestamp,
|
| 812 |
)
|
| 813 |
|
|
@@ -820,10 +933,21 @@ def register(app, ns: str = "a11oy"):
|
|
| 820 |
return response
|
| 821 |
except HTTPException:
|
| 822 |
raise
|
| 823 |
-
except
|
| 824 |
-
raise HTTPException(
|
| 825 |
-
|
| 826 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 827 |
except Exception as exc:
|
| 828 |
_TELEMETRY.observe(
|
| 829 |
(time.perf_counter() - started) * 1000.0,
|
|
@@ -839,12 +963,13 @@ def register(app, ns: str = "a11oy"):
|
|
| 839 |
|
| 840 |
return {
|
| 841 |
"ok": True,
|
| 842 |
-
"state": "
|
| 843 |
"routes": [
|
| 844 |
prefix + "/healthz",
|
| 845 |
prefix + "/bench/meta",
|
| 846 |
prefix + "/metrics",
|
| 847 |
prefix + "/integrity",
|
|
|
|
| 848 |
prefix + "/drain",
|
| 849 |
prefix + "/sessions/{session_id}",
|
| 850 |
prefix + "/step",
|
|
|
|
| 1 |
"""Authenticated Governed Delta Workspace API and benchmark surfaces."""
|
| 2 |
|
| 3 |
import hashlib
|
|
|
|
| 4 |
import json
|
| 5 |
import os
|
| 6 |
import re
|
| 7 |
+
import threading
|
| 8 |
import time
|
| 9 |
+
from datetime import datetime, timezone
|
| 10 |
from typing import List, Literal, Optional
|
| 11 |
|
| 12 |
from fastapi import Header, HTTPException, Request
|
|
|
|
| 14 |
from pydantic import BaseModel, Field, ValidationError
|
| 15 |
|
| 16 |
from gdw_attention import AttentionFeatures, choose_attention_mode
|
| 17 |
+
from gdw_auth import (
|
| 18 |
+
AuthConfigurationError,
|
| 19 |
+
AuthenticationError,
|
| 20 |
+
Principal,
|
| 21 |
+
authenticate_bearer,
|
| 22 |
+
load_credential_registry,
|
| 23 |
+
)
|
| 24 |
from gdw_proofs import build_proof_payload, sha256_json
|
| 25 |
+
from gdw_runtime import drain_once, runtime_health
|
| 26 |
from gdw_telemetry import GDWTelemetry
|
| 27 |
+
from gdw_workspace import (
|
| 28 |
+
GDWConfigurationError,
|
| 29 |
+
GDWLifecycleError,
|
| 30 |
+
GDWQuotaExceeded,
|
| 31 |
+
GDWWorkspace,
|
| 32 |
+
)
|
| 33 |
from szl_sgh_scheduler import build_plan
|
| 34 |
|
| 35 |
|
| 36 |
_TELEMETRY = GDWTelemetry()
|
| 37 |
_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
| 38 |
_EXPERTS = {"planner", "retriever", "auditor", "verifier", "operator"}
|
| 39 |
+
_AUTH_LOCK = threading.RLock()
|
| 40 |
+
_AUTH_REGISTRY = None
|
| 41 |
+
_AUTH_FINGERPRINT = None
|
| 42 |
|
| 43 |
|
| 44 |
class GDWStepRequest(BaseModel):
|
|
|
|
| 66 |
return model.dict()
|
| 67 |
|
| 68 |
|
| 69 |
+
def _model_schema(model):
|
| 70 |
+
if hasattr(model, "model_json_schema"):
|
| 71 |
+
return model.model_json_schema()
|
| 72 |
+
return model.schema()
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _validate_step_payload(value) -> GDWStepRequest:
|
| 76 |
+
try:
|
| 77 |
+
if hasattr(GDWStepRequest, "model_validate"):
|
| 78 |
+
return GDWStepRequest.model_validate(value)
|
| 79 |
+
return GDWStepRequest.parse_obj(value)
|
| 80 |
+
except ValidationError as exc:
|
| 81 |
+
raise HTTPException(
|
| 82 |
+
status_code=422,
|
| 83 |
+
detail="invalid GDW step request",
|
| 84 |
+
) from exc
|
| 85 |
+
|
| 86 |
+
|
| 87 |
def _now() -> str:
|
| 88 |
return datetime.now(timezone.utc).isoformat()
|
| 89 |
|
|
|
|
| 95 |
return hashlib.sha256(encoded).hexdigest()
|
| 96 |
|
| 97 |
|
| 98 |
+
def _credential_registry():
|
| 99 |
+
global _AUTH_FINGERPRINT, _AUTH_REGISTRY
|
| 100 |
+
registry_json = os.environ.get("GDW_CREDENTIALS_JSON")
|
| 101 |
+
principal_registry_json = os.environ.get("GDW_PRINCIPALS_JSON")
|
| 102 |
+
legacy_enabled = os.environ.get(
|
| 103 |
+
"GDW_ALLOW_LEGACY_AUTH", ""
|
| 104 |
+
).strip().lower() in {"1", "true", "yes", "on"}
|
| 105 |
+
legacy_scopes = tuple(
|
| 106 |
+
value.strip()
|
| 107 |
+
for value in os.environ.get("GDW_LEGACY_SCOPES", "").split(",")
|
| 108 |
+
if value.strip()
|
| 109 |
+
)
|
| 110 |
+
fingerprint = _sha(
|
| 111 |
+
{
|
| 112 |
+
"registry": registry_json,
|
| 113 |
+
"principal_registry": principal_registry_json,
|
| 114 |
+
"legacy_enabled": legacy_enabled,
|
| 115 |
+
"legacy_token": os.environ.get("GDW_AUTH_TOKEN"),
|
| 116 |
+
"legacy_owner": os.environ.get("GDW_OWNER_ID"),
|
| 117 |
+
"legacy_namespace": os.environ.get("GDW_NAMESPACE"),
|
| 118 |
+
"legacy_scopes": legacy_scopes,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
}
|
| 120 |
+
)
|
| 121 |
+
with _AUTH_LOCK:
|
| 122 |
+
if _AUTH_REGISTRY is not None and _AUTH_FINGERPRINT == fingerprint:
|
| 123 |
+
return _AUTH_REGISTRY
|
| 124 |
+
registry = load_credential_registry(
|
| 125 |
+
registry_json,
|
| 126 |
+
principal_registry_json=principal_registry_json,
|
| 127 |
+
principal_registry_namespace=os.environ.get("GDW_NAMESPACE") or "a11oy",
|
| 128 |
+
legacy_enabled=legacy_enabled,
|
| 129 |
+
legacy_token=os.environ.get("GDW_AUTH_TOKEN"),
|
| 130 |
+
legacy_owner_id=os.environ.get("GDW_OWNER_ID"),
|
| 131 |
+
legacy_namespace=os.environ.get("GDW_NAMESPACE"),
|
| 132 |
+
legacy_scopes=legacy_scopes,
|
| 133 |
+
)
|
| 134 |
+
_AUTH_REGISTRY = registry
|
| 135 |
+
_AUTH_FINGERPRINT = fingerprint
|
| 136 |
+
return registry
|
| 137 |
|
| 138 |
|
| 139 |
+
def _authorise(
|
| 140 |
authorization: Optional[str],
|
| 141 |
+
*,
|
| 142 |
+
namespace: str,
|
| 143 |
+
required_scopes=(),
|
| 144 |
+
) -> Principal:
|
| 145 |
try:
|
| 146 |
+
return authenticate_bearer(
|
| 147 |
+
authorization,
|
| 148 |
+
_credential_registry(),
|
| 149 |
+
namespace=namespace,
|
| 150 |
+
required_scopes=required_scopes,
|
| 151 |
+
)
|
| 152 |
+
except AuthConfigurationError as exc:
|
| 153 |
raise HTTPException(
|
| 154 |
status_code=503,
|
| 155 |
+
detail="GDW credential registry is unavailable",
|
| 156 |
) from exc
|
| 157 |
+
except AuthenticationError as exc:
|
| 158 |
+
status = 403 if exc.code in {
|
| 159 |
+
"credential_revoked",
|
| 160 |
+
"foreign_namespace",
|
| 161 |
+
"missing_scopes",
|
| 162 |
+
} else 401
|
| 163 |
+
raise HTTPException(status_code=status, detail=exc.code) from exc
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _workspace(principal: Principal) -> GDWWorkspace:
|
| 167 |
+
return GDWWorkspace(
|
| 168 |
+
namespace=principal.namespace,
|
| 169 |
+
owner_id=principal.owner_id,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
|
| 173 |
+
def _governance_ready() -> bool:
|
| 174 |
+
try:
|
| 175 |
+
import szl_codename_gate
|
| 176 |
+
import szl_colang_policy
|
| 177 |
|
| 178 |
+
policy = szl_colang_policy.get_policy()
|
| 179 |
+
return bool(
|
| 180 |
+
policy.loaded
|
| 181 |
+
and policy.enforcement_ready
|
| 182 |
+
and callable(getattr(szl_codename_gate, "scan_text", None))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
)
|
| 184 |
+
except Exception:
|
| 185 |
+
return False
|
|
|
|
|
|
|
|
|
|
| 186 |
|
| 187 |
|
| 188 |
+
def _policy_bundle_sha256() -> Optional[str]:
|
| 189 |
+
try:
|
| 190 |
+
import szl_colang_policy
|
| 191 |
+
|
| 192 |
+
policy = szl_colang_policy.get_policy()
|
| 193 |
+
if not policy.enforcement_ready:
|
| 194 |
+
return None
|
| 195 |
+
return policy.bundle_sha256
|
| 196 |
+
except Exception:
|
| 197 |
+
return None
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _write_readiness(
|
| 201 |
+
namespace: str,
|
| 202 |
+
) -> tuple[bool, list[str], int, dict, bool]:
|
| 203 |
+
runtime = runtime_health()
|
| 204 |
+
production = os.environ.get(
|
| 205 |
+
"GDW_PRODUCTION_MODE", ""
|
| 206 |
+
).strip().lower() in {"1", "true", "yes", "on"}
|
| 207 |
+
blockers = []
|
| 208 |
+
if production:
|
| 209 |
+
storage = runtime.get("storage") or {}
|
| 210 |
+
drain = runtime.get("drain") or {}
|
| 211 |
+
if runtime.get("evidence_label") != "VERIFIED":
|
| 212 |
+
blockers.append("RUNTIME_EVIDENCE_UNVERIFIED")
|
| 213 |
+
if runtime.get("startup_state") != "READY":
|
| 214 |
+
blockers.append("RUNTIME_NOT_READY")
|
| 215 |
+
if storage.get("sqlite_integrity") != "ok":
|
| 216 |
+
blockers.append("SQLITE_INTEGRITY_UNVERIFIED")
|
| 217 |
+
if storage.get("schema_version") != GDWWorkspace.schema_version():
|
| 218 |
+
blockers.append("SCHEMA_VERSION_UNVERIFIED")
|
| 219 |
+
if not re.fullmatch(
|
| 220 |
+
r"[0-9a-f]{32}",
|
| 221 |
+
str(storage.get("database_generation_id") or ""),
|
| 222 |
+
):
|
| 223 |
+
blockers.append("DATABASE_GENERATION_UNVERIFIED")
|
| 224 |
+
if storage.get("proof_export_mode") != "outbox":
|
| 225 |
+
blockers.append("OUTBOX_MODE_UNVERIFIED")
|
| 226 |
+
if storage.get("journal_mode_observed") != storage.get(
|
| 227 |
+
"journal_mode_requested"
|
| 228 |
+
):
|
| 229 |
+
blockers.append("JOURNAL_MODE_MISMATCH")
|
| 230 |
+
if storage.get("persistence_required") is not True:
|
| 231 |
+
blockers.append("PERSISTENCE_NOT_REQUIRED")
|
| 232 |
+
if storage.get("mount_verified") is not True:
|
| 233 |
+
blockers.append("PERSISTENT_MOUNT_UNVERIFIED")
|
| 234 |
+
expected_synchronous = {"FULL": 2, "NORMAL": 1}.get(
|
| 235 |
+
storage.get("synchronous_requested")
|
| 236 |
+
)
|
| 237 |
+
if storage.get("synchronous_observed") != expected_synchronous:
|
| 238 |
+
blockers.append("SYNCHRONOUS_MODE_MISMATCH")
|
| 239 |
+
if not drain.get("enabled") or not drain.get("running"):
|
| 240 |
+
blockers.append("OUTBOX_SUPERVISOR_NOT_RUNNING")
|
| 241 |
+
if drain.get("last_outcome") != "SUCCEEDED":
|
| 242 |
+
blockers.append("OUTBOX_SUPERVISOR_NOT_HEALTHY")
|
| 243 |
+
if drain.get("success_run_generation_id") != drain.get(
|
| 244 |
+
"run_generation_id"
|
| 245 |
+
):
|
| 246 |
+
blockers.append("OUTBOX_SUPERVISOR_SUCCESS_STALE")
|
| 247 |
+
if drain.get("success_database_generation_id") != storage.get(
|
| 248 |
+
"database_generation_id"
|
| 249 |
+
):
|
| 250 |
+
blockers.append("OUTBOX_SUPERVISOR_DATABASE_STALE")
|
| 251 |
+
try:
|
| 252 |
+
success_at = datetime.fromisoformat(
|
| 253 |
+
str(drain.get("last_success_at") or "").replace("Z", "+00:00")
|
| 254 |
+
)
|
| 255 |
+
age = (datetime.now(timezone.utc) - success_at).total_seconds()
|
| 256 |
+
max_age = int(drain.get("max_staleness_seconds") or 0)
|
| 257 |
+
if max_age < 1 or age < 0 or age > max_age:
|
| 258 |
+
raise ValueError
|
| 259 |
+
except (TypeError, ValueError):
|
| 260 |
+
blockers.append("OUTBOX_SUPERVISOR_HEARTBEAT_STALE")
|
| 261 |
+
try:
|
| 262 |
+
credentials = _credential_registry()
|
| 263 |
+
credential_count = credentials.credential_count
|
| 264 |
+
except AuthConfigurationError:
|
| 265 |
+
credential_count = 0
|
| 266 |
+
blockers.append("CREDENTIAL_REGISTRY_UNAVAILABLE")
|
| 267 |
+
governance_ready = _governance_ready()
|
| 268 |
+
if not governance_ready:
|
| 269 |
+
blockers.append("GOVERNANCE_SOURCE_UNREADY")
|
| 270 |
+
return (
|
| 271 |
+
not blockers,
|
| 272 |
+
sorted(set(blockers)),
|
| 273 |
+
credential_count,
|
| 274 |
+
runtime,
|
| 275 |
+
governance_ready,
|
| 276 |
+
)
|
| 277 |
|
| 278 |
|
| 279 |
+
def _require_write_ready(namespace: str) -> None:
|
| 280 |
+
ready, blockers, _, _, _ = _write_readiness(namespace)
|
| 281 |
+
if not ready:
|
|
|
|
| 282 |
raise HTTPException(
|
| 283 |
status_code=503,
|
| 284 |
+
detail={
|
| 285 |
+
"reason": "GDW_WRITE_SURFACE_UNAVAILABLE",
|
| 286 |
+
"write_blockers": blockers,
|
| 287 |
+
},
|
| 288 |
+
)
|
| 289 |
|
| 290 |
|
| 291 |
+
def _public_runtime_health(runtime: dict) -> dict:
|
| 292 |
+
storage = runtime.get("storage")
|
| 293 |
+
public_storage = None
|
| 294 |
+
if isinstance(storage, dict):
|
| 295 |
+
public_storage = {
|
| 296 |
+
key: storage.get(key)
|
| 297 |
+
for key in (
|
| 298 |
+
"persistence_required",
|
| 299 |
+
"mount_verified",
|
| 300 |
+
"journal_mode_requested",
|
| 301 |
+
"synchronous_requested",
|
| 302 |
+
"proof_export_mode",
|
| 303 |
+
"journal_mode_observed",
|
| 304 |
+
"synchronous_observed",
|
| 305 |
+
"sqlite_integrity",
|
| 306 |
+
"schema_version",
|
| 307 |
+
"database_generation_id",
|
| 308 |
+
)
|
| 309 |
+
if key in storage
|
| 310 |
+
}
|
| 311 |
+
drain = runtime.get("drain")
|
| 312 |
+
public_drain = None
|
| 313 |
+
if isinstance(drain, dict):
|
| 314 |
+
public_drain = {
|
| 315 |
+
key: drain.get(key)
|
| 316 |
+
for key in (
|
| 317 |
+
"enabled",
|
| 318 |
+
"running",
|
| 319 |
+
"last_outcome",
|
| 320 |
+
"last_attempt_at",
|
| 321 |
+
"last_success_at",
|
| 322 |
+
"last_error",
|
| 323 |
+
)
|
| 324 |
+
if key in drain
|
| 325 |
}
|
| 326 |
+
return {
|
| 327 |
+
"startup_state": runtime.get("startup_state"),
|
| 328 |
+
"evidence_label": runtime.get("evidence_label"),
|
| 329 |
+
"storage": public_storage,
|
| 330 |
+
"drain": public_drain,
|
| 331 |
+
"prepared_at": runtime.get("prepared_at"),
|
| 332 |
+
"error": runtime.get("error"),
|
| 333 |
}
|
| 334 |
|
| 335 |
|
|
|
|
| 361 |
payload_data: dict,
|
| 362 |
request_id: str,
|
| 363 |
request_digest: str,
|
| 364 |
+
principal: Principal,
|
| 365 |
) -> dict:
|
| 366 |
action = {
|
| 367 |
"tool": "execute",
|
|
|
|
| 371 |
"target": payload_data["session_id"],
|
| 372 |
"request_id": request_id,
|
| 373 |
"request_digest": request_digest,
|
| 374 |
+
"principal": principal.owner_id,
|
| 375 |
+
"namespace": principal.namespace,
|
| 376 |
+
"credential_key_id": principal.key_id,
|
| 377 |
"text": payload_data["request"],
|
| 378 |
"high_impact": float(payload_data["risk_budget"]) >= 0.75,
|
| 379 |
"irreversible": False,
|
| 380 |
}
|
| 381 |
try:
|
| 382 |
+
import szl_colang_policy
|
| 383 |
+
|
| 384 |
+
policy = szl_colang_policy.get_policy()
|
| 385 |
+
if not policy.enforcement_ready:
|
| 386 |
+
raise RuntimeError("exact file-backed Colang policy is not ready")
|
| 387 |
+
colang = policy.evaluate(action)
|
| 388 |
+
if not colang.get("enforcement_ready"):
|
| 389 |
+
raise RuntimeError("Colang policy evaluation failed exact-source checks")
|
| 390 |
except Exception as exc:
|
| 391 |
return {
|
| 392 |
"allowed": False,
|
|
|
|
| 394 |
"reason_codes": ["DOCTRINE_GATE_UNAVAILABLE"],
|
| 395 |
"detail": type(exc).__name__,
|
| 396 |
"writer_is_judge": True,
|
| 397 |
+
"principal": {
|
| 398 |
+
"owner_id": principal.owner_id,
|
| 399 |
+
"namespace": principal.namespace,
|
| 400 |
+
"key_id": principal.key_id,
|
| 401 |
+
},
|
| 402 |
}
|
| 403 |
|
| 404 |
try:
|
|
|
|
| 417 |
"reason_codes": ["CODENAME_GATE_UNAVAILABLE"],
|
| 418 |
"detail": type(exc).__name__,
|
| 419 |
"writer_is_judge": True,
|
| 420 |
+
"principal": {
|
| 421 |
+
"owner_id": principal.owner_id,
|
| 422 |
+
"namespace": principal.namespace,
|
| 423 |
+
"key_id": principal.key_id,
|
| 424 |
+
},
|
| 425 |
"colang": {
|
| 426 |
"decision": colang.get("decision"),
|
| 427 |
"fired_flows": colang.get("fired_flows", []),
|
| 428 |
"flows_evaluated": colang.get("flows_evaluated", []),
|
| 429 |
"policy_files": colang.get("policy_files", []),
|
| 430 |
+
"bundle_sha256": colang.get("bundle_sha256"),
|
| 431 |
},
|
| 432 |
}
|
| 433 |
|
|
|
|
| 439 |
return {
|
| 440 |
"allowed": not reasons,
|
| 441 |
"decision": "ALLOW" if not reasons else "DENY",
|
| 442 |
+
"reason_codes": reasons or ["FILE_BACKED_GOVERNANCE_PASS"],
|
| 443 |
"writer_is_judge": True,
|
| 444 |
+
"principal": {
|
| 445 |
+
"owner_id": principal.owner_id,
|
| 446 |
+
"namespace": principal.namespace,
|
| 447 |
+
"key_id": principal.key_id,
|
| 448 |
+
},
|
| 449 |
"colang": {
|
| 450 |
"decision": colang.get("decision"),
|
| 451 |
"fired_flows": colang.get("fired_flows", []),
|
| 452 |
"flows_evaluated": colang.get("flows_evaluated", []),
|
| 453 |
"policy_files": colang.get("policy_files", []),
|
| 454 |
+
"bundle_sha256": colang.get("bundle_sha256"),
|
| 455 |
},
|
| 456 |
"codename_gate": {
|
| 457 |
"clean": not codename_hits,
|
|
|
|
| 465 |
proposal_id: str,
|
| 466 |
request_id: str,
|
| 467 |
request_digest: str,
|
|
|
|
|
|
|
| 468 |
session_id: str,
|
| 469 |
step: int,
|
| 470 |
before_hash: str,
|
| 471 |
after_hash: str,
|
| 472 |
scheduler_mode: str,
|
| 473 |
governance: dict,
|
| 474 |
+
principal: Principal,
|
| 475 |
+
database_generation_id: str,
|
| 476 |
timestamp: str,
|
| 477 |
) -> dict:
|
| 478 |
receipt = {
|
|
|
|
| 481 |
"proposal_id": proposal_id,
|
| 482 |
"request_id": request_id,
|
| 483 |
"request_digest": request_digest,
|
|
|
|
|
|
|
| 484 |
"session_id": session_id,
|
| 485 |
+
"owner_id": principal.owner_id,
|
| 486 |
+
"namespace": principal.namespace,
|
| 487 |
+
"database_generation_id": database_generation_id,
|
| 488 |
+
"credential_key_id": principal.key_id,
|
| 489 |
"step": step,
|
| 490 |
"state_before_hash": before_hash,
|
| 491 |
"state_after_hash": after_hash,
|
|
|
|
| 498 |
return receipt
|
| 499 |
|
| 500 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 501 |
def register(app, ns: str = "a11oy"):
|
| 502 |
prefix = f"/api/{ns}/v1/gdw"
|
| 503 |
|
| 504 |
@app.get(prefix + "/healthz")
|
| 505 |
@app.get("/v1/gdw/healthz")
|
| 506 |
def gdw_healthz():
|
| 507 |
+
(
|
| 508 |
+
write_ready,
|
| 509 |
+
blockers,
|
| 510 |
+
credential_count,
|
| 511 |
+
runtime,
|
| 512 |
+
governance_ready,
|
| 513 |
+
) = _write_readiness(ns)
|
| 514 |
+
public_runtime = _public_runtime_health(runtime)
|
| 515 |
+
return {
|
| 516 |
+
"service": "gdw-frontier",
|
| 517 |
+
"status": "REAL" if write_ready else "UNAVAILABLE",
|
| 518 |
+
"write_ready": write_ready,
|
| 519 |
+
"credential_count": credential_count,
|
| 520 |
+
"governance_ready": governance_ready,
|
| 521 |
+
"write_blockers": blockers,
|
| 522 |
+
"persistence": public_runtime,
|
| 523 |
+
"benchmark_claim": "UNMEASURED",
|
| 524 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 525 |
|
| 526 |
@app.get(prefix + "/bench/meta")
|
| 527 |
@app.get("/v1/gdw/bench/meta")
|
| 528 |
def gdw_bench_meta(
|
| 529 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 530 |
):
|
| 531 |
+
_authorise(
|
| 532 |
+
authorization,
|
| 533 |
+
namespace=ns,
|
| 534 |
+
required_scopes=("bench:read",),
|
| 535 |
+
)
|
| 536 |
return {
|
| 537 |
"service": "gdw-frontier",
|
| 538 |
"implementation_status": "REAL",
|
|
|
|
| 550 |
def gdw_metrics(
|
| 551 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 552 |
):
|
| 553 |
+
_authorise(
|
| 554 |
+
authorization,
|
| 555 |
+
namespace=ns,
|
| 556 |
+
required_scopes=("metrics:read",),
|
| 557 |
+
)
|
| 558 |
return PlainTextResponse(
|
| 559 |
_TELEMETRY.render(),
|
| 560 |
media_type="text/plain; version=0.0.4; charset=utf-8",
|
|
|
|
| 565 |
def gdw_integrity(
|
| 566 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 567 |
):
|
| 568 |
+
principal = _authorise(
|
| 569 |
+
authorization,
|
| 570 |
+
namespace=ns,
|
| 571 |
+
required_scopes=("integrity:read",),
|
| 572 |
+
)
|
| 573 |
+
return _workspace(principal).integrity()
|
| 574 |
+
|
| 575 |
+
@app.post(prefix + "/drain")
|
| 576 |
+
@app.post("/v1/gdw/drain")
|
| 577 |
+
def gdw_drain(
|
| 578 |
+
limit: int = 100,
|
| 579 |
+
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 580 |
+
):
|
| 581 |
+
principal = _authorise(
|
| 582 |
+
authorization,
|
| 583 |
+
namespace=ns,
|
| 584 |
+
required_scopes=("integrity:global",),
|
| 585 |
+
)
|
| 586 |
+
_require_write_ready(ns)
|
| 587 |
+
report = drain_once(limit=limit)
|
| 588 |
+
integrity = _workspace(principal).integrity(global_scope=True)
|
| 589 |
+
return {
|
| 590 |
+
"schema": "szl.gdw.drain-report/v1",
|
| 591 |
+
**report,
|
| 592 |
+
"integrity_ok": integrity["ok"],
|
| 593 |
+
"database_generation_id": integrity["database_generation_id"],
|
| 594 |
+
}
|
| 595 |
+
|
| 596 |
+
@app.get(prefix + "/integrity/global")
|
| 597 |
+
@app.get("/v1/gdw/integrity/global")
|
| 598 |
+
def gdw_global_integrity(
|
| 599 |
+
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 600 |
+
):
|
| 601 |
+
principal = _authorise(
|
| 602 |
+
authorization,
|
| 603 |
+
namespace=ns,
|
| 604 |
+
required_scopes=("integrity:global",),
|
| 605 |
+
)
|
| 606 |
+
return _workspace(principal).integrity(global_scope=True)
|
| 607 |
|
| 608 |
@app.get(prefix + "/sessions/{session_id}")
|
| 609 |
@app.get("/v1/gdw/sessions/{session_id}")
|
|
|
|
| 611 |
session_id: str,
|
| 612 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 613 |
):
|
| 614 |
+
principal = _authorise(
|
| 615 |
+
authorization,
|
| 616 |
+
namespace=ns,
|
| 617 |
+
required_scopes=("session:read",),
|
| 618 |
+
)
|
| 619 |
if not _ID_PATTERN.fullmatch(session_id):
|
| 620 |
raise HTTPException(status_code=422, detail="invalid session_id")
|
| 621 |
+
state = _workspace(principal).read_session(session_id)
|
|
|
|
|
|
|
|
|
|
| 622 |
if state is None:
|
| 623 |
raise HTTPException(status_code=404, detail="session not found")
|
| 624 |
return state
|
| 625 |
|
| 626 |
+
request_body_contract = {
|
| 627 |
+
"requestBody": {
|
| 628 |
+
"required": True,
|
| 629 |
+
"content": {
|
| 630 |
+
"application/json": {
|
| 631 |
+
"schema": _model_schema(GDWStepRequest),
|
| 632 |
+
}
|
| 633 |
+
},
|
| 634 |
+
}
|
| 635 |
+
}
|
| 636 |
+
|
| 637 |
+
@app.post(prefix + "/step", openapi_extra=request_body_contract)
|
| 638 |
+
@app.post("/v1/gdw/step", openapi_extra=request_body_contract)
|
| 639 |
async def gdw_step(
|
| 640 |
request: Request,
|
| 641 |
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 642 |
x_request_id: Optional[str] = Header(default=None, alias="X-Request-Id"),
|
| 643 |
):
|
| 644 |
started = time.perf_counter()
|
| 645 |
+
principal = _authorise(
|
| 646 |
+
authorization,
|
| 647 |
+
namespace=ns,
|
| 648 |
+
required_scopes=("step:write",),
|
| 649 |
+
)
|
| 650 |
+
_require_write_ready(ns)
|
| 651 |
try:
|
| 652 |
raw_payload = await request.json()
|
| 653 |
+
except Exception as exc:
|
| 654 |
+
raise HTTPException(
|
| 655 |
+
status_code=422,
|
| 656 |
+
detail="invalid GDW step request",
|
| 657 |
+
) from exc
|
| 658 |
+
payload = _validate_step_payload(raw_payload)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 659 |
request_id = _validate_identifiers(payload, x_request_id)
|
| 660 |
payload_data = _dump_model(payload)
|
| 661 |
request_digest = _sha(payload_data)
|
|
|
|
|
|
|
|
|
|
| 662 |
selected_mode = "unresolved"
|
| 663 |
decision = "ERROR"
|
| 664 |
receipt_hash = ""
|
| 665 |
|
| 666 |
try:
|
| 667 |
+
workspace = _workspace(principal)
|
| 668 |
with workspace.transaction() as connection:
|
| 669 |
+
cached = workspace.cached_request(connection, request_id)
|
|
|
|
|
|
|
| 670 |
if cached is not None:
|
| 671 |
cached_digest, cached_response = cached
|
| 672 |
if cached_digest != request_digest:
|
|
|
|
| 674 |
status_code=409,
|
| 675 |
detail="X-Request-Id was already used with different content",
|
| 676 |
)
|
| 677 |
+
current_bundle = _policy_bundle_sha256()
|
| 678 |
+
cached_bundle = (
|
| 679 |
+
cached_response.get("audit", {})
|
| 680 |
+
.get("governance", {})
|
| 681 |
+
.get("colang", {})
|
| 682 |
+
.get("bundle_sha256")
|
| 683 |
+
)
|
| 684 |
+
if not current_bundle or cached_bundle != current_bundle:
|
| 685 |
+
raise HTTPException(
|
| 686 |
+
status_code=409,
|
| 687 |
+
detail="policy snapshot changed; replay refused",
|
| 688 |
+
)
|
| 689 |
cached_response["replayed"] = True
|
| 690 |
selected_mode = cached_response["scheduler_mode"]
|
| 691 |
decision = cached_response["decision"]
|
|
|
|
| 703 |
before_step = 0
|
| 704 |
before_hash = _sha(
|
| 705 |
{
|
| 706 |
+
"namespace": principal.namespace,
|
| 707 |
+
"owner_id": principal.owner_id,
|
| 708 |
"session_id": payload.session_id,
|
| 709 |
"step": 0,
|
| 710 |
"state": "GENESIS",
|
| 711 |
}
|
| 712 |
)
|
| 713 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 714 |
before_step = previous["step"]
|
| 715 |
before_hash = previous["state_hash"]
|
| 716 |
|
|
|
|
| 731 |
selected_mode = routing["mode"]
|
| 732 |
precondition_decision = _decision(payload)
|
| 733 |
governance = _governance_gate(
|
| 734 |
+
payload_data, request_id, request_digest, principal
|
|
|
|
|
|
|
|
|
|
| 735 |
)
|
| 736 |
decision = precondition_decision
|
| 737 |
if decision == "ACCEPT" and not governance["allowed"]:
|
| 738 |
decision = "REJECT"
|
| 739 |
mutates = decision == "ACCEPT" and not payload.dry_run
|
| 740 |
step = before_step + 1 if mutates else before_step
|
| 741 |
+
database_generation_id = workspace.database_generation_id
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
proposal_id = sha256_json(
|
| 743 |
{
|
| 744 |
"schema": "szl.gdw.proposal-identity/v1",
|
| 745 |
+
"database_generation_id": database_generation_id,
|
| 746 |
+
"namespace": principal.namespace,
|
| 747 |
+
"owner_id": principal.owner_id,
|
| 748 |
"request_id": request_id,
|
| 749 |
"request_digest": request_digest,
|
| 750 |
"state_before_hash": before_hash,
|
| 751 |
"governance_evidence_sha256": sha256_json(governance),
|
| 752 |
}
|
| 753 |
)
|
| 754 |
+
timestamp = _now()
|
| 755 |
|
| 756 |
if mutates:
|
| 757 |
state = {
|
| 758 |
+
"namespace": principal.namespace,
|
| 759 |
+
"owner_id": principal.owner_id,
|
| 760 |
"session_id": payload.session_id,
|
| 761 |
+
"database_generation_id": database_generation_id,
|
|
|
|
| 762 |
"step": step,
|
| 763 |
"previous_state_hash": before_hash,
|
| 764 |
"request_digest": request_digest,
|
|
|
|
| 778 |
proposal_id=proposal_id,
|
| 779 |
request_id=request_id,
|
| 780 |
request_digest=request_digest,
|
|
|
|
|
|
|
| 781 |
session_id=payload.session_id,
|
| 782 |
step=step,
|
| 783 |
before_hash=before_hash,
|
| 784 |
after_hash=after_hash,
|
| 785 |
scheduler_mode=selected_mode,
|
| 786 |
governance=governance,
|
| 787 |
+
principal=principal,
|
| 788 |
+
database_generation_id=database_generation_id,
|
| 789 |
timestamp=timestamp,
|
| 790 |
)
|
| 791 |
receipt_hash = receipt["receipt_hash"]
|
|
|
|
| 798 |
proposal_id=proposal_id,
|
| 799 |
request_id=request_id,
|
| 800 |
request_digest=request_digest,
|
| 801 |
+
namespace=principal.namespace,
|
| 802 |
+
owner_id=principal.owner_id,
|
| 803 |
+
database_generation_id=database_generation_id,
|
| 804 |
step=step,
|
| 805 |
before_hash=before_hash,
|
| 806 |
after_hash=after_hash,
|
|
|
|
| 818 |
"GDW_PROOF_EXPORT_MODE must be 'outbox'; "
|
| 819 |
"synchronous external effects are not transaction-safe"
|
| 820 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 821 |
proof_artifact = {
|
| 822 |
"status": "OUTBOX_PENDING",
|
| 823 |
"kind": "proof_export",
|
| 824 |
+
"idempotency_key": workspace.scoped_effect_key(
|
| 825 |
+
principal.namespace,
|
| 826 |
+
principal.owner_id,
|
| 827 |
+
request_id,
|
| 828 |
+
"proof_export",
|
| 829 |
+
proof_payload["payload_sha256"],
|
| 830 |
+
),
|
| 831 |
"payload_sha256": proof_payload["payload_sha256"],
|
| 832 |
"formal_status": "NOT_RUN",
|
| 833 |
}
|
|
|
|
| 841 |
"benchmark_status": "UNMEASURED",
|
| 842 |
"proposal_id": proposal_id,
|
| 843 |
"request_id": request_id,
|
| 844 |
+
"request_digest": request_digest,
|
| 845 |
+
"database_generation_id": database_generation_id,
|
| 846 |
"session_id": payload.session_id,
|
| 847 |
+
"principal": {
|
| 848 |
+
"owner_id": principal.owner_id,
|
| 849 |
+
"namespace": principal.namespace,
|
| 850 |
+
"key_id": principal.key_id,
|
| 851 |
+
},
|
| 852 |
"decision": decision,
|
| 853 |
"step": step,
|
| 854 |
"state_hash": after_hash,
|
|
|
|
| 860 |
"kernel_execution": "NOT_EXECUTED_BY_CONTROL_API",
|
| 861 |
"dry_run": payload.dry_run,
|
| 862 |
"replayed": False,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 863 |
"audit": {
|
| 864 |
"governance": governance,
|
| 865 |
"precondition_decision": precondition_decision,
|
|
|
|
| 883 |
timestamp,
|
| 884 |
)
|
| 885 |
if receipt is not None:
|
| 886 |
+
receipt_payload_sha256 = sha256_json(receipt)
|
| 887 |
workspace.save_receipt(
|
| 888 |
connection,
|
| 889 |
receipt_hash,
|
|
|
|
| 893 |
receipt,
|
| 894 |
timestamp,
|
| 895 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 896 |
workspace.save_effect_outbox(
|
| 897 |
connection,
|
| 898 |
request_id,
|
| 899 |
"receipt_projection",
|
|
|
|
|
|
|
|
|
|
| 900 |
receipt,
|
| 901 |
+
receipt_payload_sha256,
|
| 902 |
+
workspace.scoped_effect_key(
|
| 903 |
+
principal.namespace,
|
| 904 |
+
principal.owner_id,
|
| 905 |
+
request_id,
|
| 906 |
+
"receipt_projection",
|
| 907 |
+
receipt_payload_sha256,
|
| 908 |
+
),
|
| 909 |
timestamp,
|
| 910 |
)
|
| 911 |
workspace.save_effect_outbox(
|
| 912 |
connection,
|
| 913 |
request_id,
|
| 914 |
"proof_export",
|
|
|
|
|
|
|
|
|
|
| 915 |
proof_payload,
|
| 916 |
+
proof_payload["payload_sha256"],
|
| 917 |
+
workspace.scoped_effect_key(
|
| 918 |
+
principal.namespace,
|
| 919 |
+
principal.owner_id,
|
| 920 |
+
request_id,
|
| 921 |
+
"proof_export",
|
| 922 |
+
proof_payload["payload_sha256"],
|
| 923 |
+
),
|
| 924 |
timestamp,
|
| 925 |
)
|
| 926 |
|
|
|
|
| 933 |
return response
|
| 934 |
except HTTPException:
|
| 935 |
raise
|
| 936 |
+
except GDWQuotaExceeded as exc:
|
| 937 |
+
raise HTTPException(
|
| 938 |
+
status_code=429,
|
| 939 |
+
detail="GDW quota exceeded",
|
| 940 |
+
) from exc
|
| 941 |
+
except GDWLifecycleError as exc:
|
| 942 |
+
raise HTTPException(
|
| 943 |
+
status_code=409,
|
| 944 |
+
detail="GDW object is outside its active lifecycle",
|
| 945 |
+
) from exc
|
| 946 |
+
except GDWConfigurationError as exc:
|
| 947 |
+
raise HTTPException(
|
| 948 |
+
status_code=503,
|
| 949 |
+
detail="GDW durable workspace is unavailable",
|
| 950 |
+
) from exc
|
| 951 |
except Exception as exc:
|
| 952 |
_TELEMETRY.observe(
|
| 953 |
(time.perf_counter() - started) * 1000.0,
|
|
|
|
| 963 |
|
| 964 |
return {
|
| 965 |
"ok": True,
|
| 966 |
+
"state": "REAL",
|
| 967 |
"routes": [
|
| 968 |
prefix + "/healthz",
|
| 969 |
prefix + "/bench/meta",
|
| 970 |
prefix + "/metrics",
|
| 971 |
prefix + "/integrity",
|
| 972 |
+
prefix + "/integrity/global",
|
| 973 |
prefix + "/drain",
|
| 974 |
prefix + "/sessions/{session_id}",
|
| 975 |
prefix + "/step",
|
serve.py
CHANGED
|
@@ -552,8 +552,9 @@ async def quantum_utility_receipt_replay(request: Request) -> JSONResponse:
|
|
| 552 |
|
| 553 |
# Governed Delta Workspace (wave 28): authenticated, deny-by-default state
|
| 554 |
# transitions over SQLite WAL, idempotency keys, the shared receipt substrate,
|
| 555 |
-
# structured theorem inputs, and honest benchmark metadata. Runtime
|
| 556 |
-
#
|
|
|
|
| 557 |
# Registered before both catch-alls.
|
| 558 |
try:
|
| 559 |
from routers import gdw_frontier as _gdw_frontier
|
|
|
|
| 552 |
|
| 553 |
# Governed Delta Workspace (wave 28): authenticated, deny-by-default state
|
| 554 |
# transitions over SQLite WAL, idempotency keys, the shared receipt substrate,
|
| 555 |
+
# structured theorem inputs, and honest benchmark metadata. Runtime writes are
|
| 556 |
+
# REAL only when the route's full readiness predicate passes; otherwise the
|
| 557 |
+
# surface is UNAVAILABLE. Benchmark claims remain UNMEASURED until a captured run.
|
| 558 |
# Registered before both catch-alls.
|
| 559 |
try:
|
| 560 |
from routers import gdw_frontier as _gdw_frontier
|
szl_colang_policy.py
CHANGED
|
@@ -36,7 +36,7 @@ import json
|
|
| 36 |
import os
|
| 37 |
import re
|
| 38 |
from pathlib import Path
|
| 39 |
-
from typing import
|
| 40 |
|
| 41 |
# Candidate locations for the policy dir (image: /app/policy/colang; repo-relative).
|
| 42 |
_POLICY_DIRS = [
|
|
@@ -49,7 +49,19 @@ _POLICY_DIRS = [
|
|
| 49 |
_FLOW_RE = re.compile(r"^define\s+flow\s+([A-Za-z0-9_]+)\s*$")
|
| 50 |
_REASON_RE = re.compile(r'with\s+reason\s+"([^"]+)"')
|
| 51 |
_PRED_RE = re.compile(r"\b([a-z_][a-z0-9_]*)\s*\(\s*\$action\s*\)")
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
# Threat / injection signatures the policy predicates actually scan for. Mirrors
|
| 54 |
# the in-image arena gate so policy enforcement is consistent across surfaces.
|
| 55 |
_INJECTION_SIGS = ["ignore previous", "ignore policy", "exfiltrate", "override",
|
|
@@ -61,9 +73,6 @@ _PII_SIGS = ["ssn", "social security", "card number", "full card", "pan ",
|
|
| 61 |
_EFFECTOR_TOOLS = {"issue_refund", "send_email", "reset_password", "assign_seat",
|
| 62 |
"apply_change", "run_sql", "engage", "release", "execute"}
|
| 63 |
_PAYLOAD_CEILING = 1_000_000
|
| 64 |
-
_ENFORCEMENT_CONTRACT_SCHEMA = "szl.colang-enforcement-contract/v1"
|
| 65 |
-
_ENFORCEMENT_EVALUATOR = "szl.colang-python-evaluator/v1"
|
| 66 |
-
_ENFORCEMENT_CONTRACT_FILENAME = "gdw_enforcement_contract.json"
|
| 67 |
|
| 68 |
|
| 69 |
def _resolve_dir() -> Optional[Path]:
|
|
@@ -213,6 +222,57 @@ _FLOW_LOGIC = {
|
|
| 213 |
"require_sensor_quorum":
|
| 214 |
lambda a: _is_threat_decision(a) and not _sensor_quorum_met(a),
|
| 215 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
|
| 218 |
def _parse_flows(text: str) -> list[dict]:
|
|
@@ -233,13 +293,41 @@ def _parse_flows(text: str) -> list[dict]:
|
|
| 233 |
rm = _REASON_RE.search(line)
|
| 234 |
if rm and not current["reason"]:
|
| 235 |
current["reason"] = rm.group(1)
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
current["guards"]
|
|
|
|
| 240 |
if current:
|
| 241 |
flows.append(current)
|
| 242 |
return flows
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
|
| 244 |
|
| 245 |
_NEMO_AVAILABLE = False
|
|
@@ -253,19 +341,43 @@ except Exception:
|
|
| 253 |
class ColangPolicy:
|
| 254 |
"""Loaded, file-backed, auditable Colang policy set."""
|
| 255 |
|
| 256 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
self.directory = directory or _resolve_dir()
|
|
|
|
|
|
|
|
|
|
| 258 |
self.files: list[dict] = []
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
self._load()
|
| 260 |
|
| 261 |
def _load(self) -> None:
|
| 262 |
self.files = []
|
|
|
|
|
|
|
|
|
|
|
|
|
| 263 |
if not self.directory:
|
|
|
|
| 264 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
for p in sorted(self.directory.glob("*.co")):
|
| 266 |
try:
|
| 267 |
raw = p.read_bytes()
|
| 268 |
-
text = raw.decode("utf-8", "
|
| 269 |
flows = _parse_flows(text)
|
| 270 |
pid = None
|
| 271 |
pver = None
|
|
@@ -285,13 +397,91 @@ class ColangPolicy:
|
|
| 285 |
"flows": flows,
|
| 286 |
"content": text,
|
| 287 |
})
|
| 288 |
-
except
|
| 289 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
|
| 291 |
@property
|
| 292 |
def loaded(self) -> bool:
|
| 293 |
return bool(self.files)
|
| 294 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
def all_flows(self) -> list[dict]:
|
| 296 |
out = []
|
| 297 |
for f in self.files:
|
|
@@ -300,165 +490,90 @@ class ColangPolicy:
|
|
| 300 |
"policy_version": f["policy_version"]})
|
| 301 |
return out
|
| 302 |
|
| 303 |
-
def
|
| 304 |
-
"""
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
self.
|
| 314 |
-
if self.directory
|
| 315 |
-
else None
|
| 316 |
-
)
|
| 317 |
-
contract: dict[str, Any] = {}
|
| 318 |
-
if contract_path is None or not contract_path.is_file():
|
| 319 |
-
reasons.append("ENFORCEMENT_CONTRACT_MISSING")
|
| 320 |
-
else:
|
| 321 |
-
try:
|
| 322 |
-
contract = json.loads(contract_path.read_text(encoding="utf-8"))
|
| 323 |
-
except Exception:
|
| 324 |
-
reasons.append("ENFORCEMENT_CONTRACT_INVALID")
|
| 325 |
-
|
| 326 |
-
if contract.get("schema") != _ENFORCEMENT_CONTRACT_SCHEMA:
|
| 327 |
-
reasons.append("ENFORCEMENT_CONTRACT_SCHEMA_MISMATCH")
|
| 328 |
-
if contract.get("evaluator") != _ENFORCEMENT_EVALUATOR:
|
| 329 |
-
reasons.append("ENFORCEMENT_EVALUATOR_MISMATCH")
|
| 330 |
-
|
| 331 |
-
expected_files = contract.get("files")
|
| 332 |
-
if not isinstance(expected_files, dict):
|
| 333 |
-
expected_files = {}
|
| 334 |
-
reasons.append("ENFORCEMENT_FILE_LOCK_INVALID")
|
| 335 |
-
actual_files = {item["name"]: item["sha256"] for item in self.files}
|
| 336 |
-
if expected_files != actual_files:
|
| 337 |
-
reasons.append("ENFORCEMENT_FILE_LOCK_MISMATCH")
|
| 338 |
-
|
| 339 |
-
unsupported_flows = sorted(
|
| 340 |
-
flow["name"]
|
| 341 |
-
for flow in self.all_flows()
|
| 342 |
-
if flow["name"] not in _FLOW_LOGIC
|
| 343 |
)
|
| 344 |
-
if
|
| 345 |
-
reasons.append("UNSUPPORTED_POLICY_FLOW")
|
| 346 |
-
|
| 347 |
-
return {
|
| 348 |
-
"valid": not reasons,
|
| 349 |
-
"schema": _ENFORCEMENT_CONTRACT_SCHEMA,
|
| 350 |
-
"evaluator": _ENFORCEMENT_EVALUATOR,
|
| 351 |
-
"contract_path": str(contract_path) if contract_path else None,
|
| 352 |
-
"contract_sha256": (
|
| 353 |
-
hashlib.sha256(contract_path.read_bytes()).hexdigest()
|
| 354 |
-
if contract_path and contract_path.is_file()
|
| 355 |
-
else None
|
| 356 |
-
),
|
| 357 |
-
"reason_codes": sorted(set(reasons)),
|
| 358 |
-
"unsupported_flows": unsupported_flows,
|
| 359 |
-
"files": actual_files,
|
| 360 |
-
}
|
| 361 |
-
|
| 362 |
-
def evaluate_strict(self, action: dict) -> dict:
|
| 363 |
-
"""Evaluate only an exact, supported, byte-locked policy generation."""
|
| 364 |
-
enforcement = self.enforcement_contract_status()
|
| 365 |
-
if not enforcement["valid"]:
|
| 366 |
return {
|
| 367 |
"allow": False,
|
| 368 |
"decision": "deny",
|
| 369 |
"fired_flows": [
|
| 370 |
{
|
| 371 |
-
"flow": "
|
| 372 |
-
"reason":
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 376 |
}
|
| 377 |
-
for reason in enforcement["reason_codes"]
|
| 378 |
],
|
| 379 |
-
"fired_count":
|
| 380 |
-
"flows_evaluated": [
|
| 381 |
-
|
| 382 |
-
],
|
| 383 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 384 |
"policy_files": [
|
| 385 |
{
|
| 386 |
-
"name":
|
| 387 |
-
"sha256":
|
| 388 |
-
"policy_id":
|
| 389 |
-
"policy_version":
|
| 390 |
}
|
| 391 |
-
for
|
| 392 |
],
|
| 393 |
-
"
|
|
|
|
|
|
|
|
|
|
| 394 |
}
|
| 395 |
-
action = action or {}
|
| 396 |
-
fired: list[dict] = []
|
| 397 |
-
evaluated: list[str] = []
|
| 398 |
-
evaluator_errors: list[str] = []
|
| 399 |
-
for flow in self.all_flows():
|
| 400 |
-
name = flow["name"]
|
| 401 |
-
logic = _FLOW_LOGIC[name]
|
| 402 |
-
evaluated.append(name)
|
| 403 |
-
try:
|
| 404 |
-
violated = bool(logic(action))
|
| 405 |
-
except Exception:
|
| 406 |
-
violated = True
|
| 407 |
-
evaluator_errors.append(name)
|
| 408 |
-
if violated:
|
| 409 |
-
fired.append({
|
| 410 |
-
"flow": name,
|
| 411 |
-
"reason": (
|
| 412 |
-
"POLICY_EVALUATOR_ERROR"
|
| 413 |
-
if name in evaluator_errors
|
| 414 |
-
else flow.get("reason") or name
|
| 415 |
-
),
|
| 416 |
-
"file": flow["file"],
|
| 417 |
-
"policy_id": flow["policy_id"],
|
| 418 |
-
"policy_version": flow["policy_version"],
|
| 419 |
-
})
|
| 420 |
-
allow = not fired
|
| 421 |
-
return {
|
| 422 |
-
"allow": allow,
|
| 423 |
-
"decision": "allow" if allow else "deny",
|
| 424 |
-
"fired_flows": fired,
|
| 425 |
-
"fired_count": len(fired),
|
| 426 |
-
"flows_evaluated": evaluated,
|
| 427 |
-
"matched_count": len(fired),
|
| 428 |
-
"policy_files": [
|
| 429 |
-
{
|
| 430 |
-
"name": item["name"],
|
| 431 |
-
"sha256": item["sha256"],
|
| 432 |
-
"policy_id": item["policy_id"],
|
| 433 |
-
"policy_version": item["policy_version"],
|
| 434 |
-
}
|
| 435 |
-
for item in self.files
|
| 436 |
-
],
|
| 437 |
-
"evaluator_errors": evaluator_errors,
|
| 438 |
-
"enforcement_contract": enforcement,
|
| 439 |
-
}
|
| 440 |
-
|
| 441 |
-
def evaluate(self, action: dict) -> dict:
|
| 442 |
-
"""Evaluate a proposed action against EVERY loaded flow. Returns which
|
| 443 |
-
flows fired (rule violated -> refuse) and the overall allow/deny. This is
|
| 444 |
-
the policy layer; serve.py calls it before signing an action receipt."""
|
| 445 |
-
action = action or {}
|
| 446 |
-
fired: list[dict] = []
|
| 447 |
-
evaluated: list[str] = []
|
| 448 |
for fl in self.all_flows():
|
| 449 |
name = fl["name"]
|
| 450 |
logic = _FLOW_LOGIC.get(name)
|
| 451 |
evaluated.append(name)
|
| 452 |
if logic is None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 453 |
continue
|
| 454 |
try:
|
| 455 |
violated = bool(logic(action))
|
| 456 |
except Exception:
|
| 457 |
-
violated =
|
|
|
|
| 458 |
if violated:
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
allow = len(fired) == 0
|
| 463 |
return {
|
| 464 |
"allow": allow,
|
|
@@ -466,6 +581,11 @@ class ColangPolicy:
|
|
| 466 |
"fired_flows": fired,
|
| 467 |
"fired_count": len(fired),
|
| 468 |
"flows_evaluated": evaluated,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
"matched_count": len(fired),
|
| 470 |
"policy_files": [{"name": f["name"], "sha256": f["sha256"],
|
| 471 |
"policy_id": f["policy_id"],
|
|
@@ -484,6 +604,18 @@ class ColangPolicy:
|
|
| 484 |
"nemoguardrails_runtime_present": _NEMO_AVAILABLE,
|
| 485 |
"file_count": len(self.files),
|
| 486 |
"flow_count": len(self.all_flows()),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 487 |
"files": self.files,
|
| 488 |
"honesty": (
|
| 489 |
"Policy is FILE-BACKED and version-controlled: each rule is a "
|
|
@@ -498,14 +630,11 @@ class ColangPolicy:
|
|
| 498 |
}
|
| 499 |
|
| 500 |
|
| 501 |
-
_SINGLETON: Optional[ColangPolicy] = None
|
| 502 |
-
|
| 503 |
-
|
| 504 |
def get_policy(reload: bool = False) -> ColangPolicy:
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
return
|
| 509 |
|
| 510 |
|
| 511 |
if __name__ == "__main__": # pragma: no cover
|
|
|
|
| 36 |
import os
|
| 37 |
import re
|
| 38 |
from pathlib import Path
|
| 39 |
+
from typing import Optional
|
| 40 |
|
| 41 |
# Candidate locations for the policy dir (image: /app/policy/colang; repo-relative).
|
| 42 |
_POLICY_DIRS = [
|
|
|
|
| 49 |
_FLOW_RE = re.compile(r"^define\s+flow\s+([A-Za-z0-9_]+)\s*$")
|
| 50 |
_REASON_RE = re.compile(r'with\s+reason\s+"([^"]+)"')
|
| 51 |
_PRED_RE = re.compile(r"\b([a-z_][a-z0-9_]*)\s*\(\s*\$action\s*\)")
|
| 52 |
+
_CONTRACT_NAME = "enforcement-contract.json"
|
| 53 |
+
_EVALUATOR_REGION_BEGIN = "# BEGIN EXACT POLICY EVALUATOR CONTRACT"
|
| 54 |
+
_EVALUATOR_REGION_END = "# END EXACT POLICY EVALUATOR CONTRACT"
|
| 55 |
+
|
| 56 |
+
# BEGIN EXACT POLICY EVALUATOR CONTRACT
|
| 57 |
+
_CANONICAL_POLICY_FILES = {
|
| 58 |
+
"killinchu_threat.co": (
|
| 59 |
+
"3c47dfd730d738f1ce0b722b5ccc21fb12dda70ccb288043ffe10b78479df9c2"
|
| 60 |
+
),
|
| 61 |
+
"roe_core.co": (
|
| 62 |
+
"77598deb89c4bebeb3682b30a0a01fcebc73a089183fa0a9a2877ad1fd732984"
|
| 63 |
+
),
|
| 64 |
+
}
|
| 65 |
# Threat / injection signatures the policy predicates actually scan for. Mirrors
|
| 66 |
# the in-image arena gate so policy enforcement is consistent across surfaces.
|
| 67 |
_INJECTION_SIGS = ["ignore previous", "ignore policy", "exfiltrate", "override",
|
|
|
|
| 73 |
_EFFECTOR_TOOLS = {"issue_refund", "send_email", "reset_password", "assign_seat",
|
| 74 |
"apply_change", "run_sql", "engage", "release", "execute"}
|
| 75 |
_PAYLOAD_CEILING = 1_000_000
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
|
| 78 |
def _resolve_dir() -> Optional[Path]:
|
|
|
|
| 222 |
"require_sensor_quorum":
|
| 223 |
lambda a: _is_threat_decision(a) and not _sensor_quorum_met(a),
|
| 224 |
}
|
| 225 |
+
_FLOW_GUARDS = {
|
| 226 |
+
"refuse_destructive_actions": [
|
| 227 |
+
"is_destructive",
|
| 228 |
+
"has_operator_authorization",
|
| 229 |
+
],
|
| 230 |
+
"refuse_pii_exfiltration": ["requests_pii_exfiltration"],
|
| 231 |
+
"refuse_prompt_injection": ["matches_injection_signature"],
|
| 232 |
+
"require_operator_approval_high_impact": [
|
| 233 |
+
"is_high_impact",
|
| 234 |
+
"has_operator_approval_event",
|
| 235 |
+
],
|
| 236 |
+
"enforce_payload_ceiling": ["payload_exceeds_ceiling"],
|
| 237 |
+
"policy_before_effect": ["is_effecting", "policy_evaluated_before"],
|
| 238 |
+
"no_autonomous_engagement": [
|
| 239 |
+
"is_engagement",
|
| 240 |
+
"has_human_authorization",
|
| 241 |
+
],
|
| 242 |
+
"require_calibrated_classifier": [
|
| 243 |
+
"is_automated_response",
|
| 244 |
+
"classifier_calibration_gate_pass",
|
| 245 |
+
],
|
| 246 |
+
"require_singleton_conformal_set": [
|
| 247 |
+
"is_automated_response",
|
| 248 |
+
"conformal_set_ambiguous",
|
| 249 |
+
],
|
| 250 |
+
"require_sensor_quorum": ["is_threat_decision", "sensor_quorum_met"],
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _load_enforcement_contract(
|
| 255 |
+
directory: Path,
|
| 256 |
+
trusted_policy_files: dict[str, str],
|
| 257 |
+
) -> dict:
|
| 258 |
+
path = directory / _CONTRACT_NAME
|
| 259 |
+
try:
|
| 260 |
+
contract = json.loads(path.read_text(encoding="utf-8"))
|
| 261 |
+
except (OSError, json.JSONDecodeError) as exc:
|
| 262 |
+
raise ValueError("policy enforcement contract is unavailable") from exc
|
| 263 |
+
if contract.get("schema") != "szl.colang-enforcement-contract/v1":
|
| 264 |
+
raise ValueError("policy enforcement contract schema is unsupported")
|
| 265 |
+
expected_files = contract.get("policy_files")
|
| 266 |
+
if not isinstance(expected_files, dict) or not expected_files:
|
| 267 |
+
raise ValueError("policy enforcement contract has no policy file bindings")
|
| 268 |
+
expected_contract = {
|
| 269 |
+
"schema": "szl.colang-enforcement-contract/v1",
|
| 270 |
+
"evaluator_region_sha256": _LOADED_EVALUATOR_SHA256,
|
| 271 |
+
"policy_files": trusted_policy_files,
|
| 272 |
+
}
|
| 273 |
+
if contract != expected_contract:
|
| 274 |
+
raise ValueError("policy enforcement contract is not rooted in trusted source")
|
| 275 |
+
return contract
|
| 276 |
|
| 277 |
|
| 278 |
def _parse_flows(text: str) -> list[dict]:
|
|
|
|
| 293 |
rm = _REASON_RE.search(line)
|
| 294 |
if rm and not current["reason"]:
|
| 295 |
current["reason"] = rm.group(1)
|
| 296 |
+
if line.strip().startswith("if "):
|
| 297 |
+
for pm in _PRED_RE.finditer(line):
|
| 298 |
+
g = pm.group(1)
|
| 299 |
+
if g not in current["guards"]:
|
| 300 |
+
current["guards"].append(g)
|
| 301 |
if current:
|
| 302 |
flows.append(current)
|
| 303 |
return flows
|
| 304 |
+
# END EXACT POLICY EVALUATOR CONTRACT
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _evaluator_region_sha256() -> str:
|
| 308 |
+
"""Hash the exact reviewed evaluator/parser region from loaded source."""
|
| 309 |
+
|
| 310 |
+
source = Path(__file__).read_text(encoding="utf-8").replace("\r\n", "\n")
|
| 311 |
+
pattern = re.compile(
|
| 312 |
+
r"^# BEGIN EXACT POLICY EVALUATOR CONTRACT\r?\n"
|
| 313 |
+
r"(?P<region>.*?)"
|
| 314 |
+
r"^# END EXACT POLICY EVALUATOR CONTRACT$",
|
| 315 |
+
re.MULTILINE | re.DOTALL,
|
| 316 |
+
)
|
| 317 |
+
match = pattern.search(source)
|
| 318 |
+
if match is None:
|
| 319 |
+
raise ValueError("policy evaluator contract markers are missing")
|
| 320 |
+
region = match.group("region")
|
| 321 |
+
if (
|
| 322 |
+
"_FLOW_LOGIC" not in region
|
| 323 |
+
or "def _matches_injection_signature" not in region
|
| 324 |
+
or "def _parse_flows" not in region
|
| 325 |
+
):
|
| 326 |
+
raise ValueError("policy evaluator contract region is incomplete")
|
| 327 |
+
return hashlib.sha256(region.encode("utf-8")).hexdigest()
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
_LOADED_EVALUATOR_SHA256 = _evaluator_region_sha256()
|
| 331 |
|
| 332 |
|
| 333 |
_NEMO_AVAILABLE = False
|
|
|
|
| 341 |
class ColangPolicy:
|
| 342 |
"""Loaded, file-backed, auditable Colang policy set."""
|
| 343 |
|
| 344 |
+
def __init__(
|
| 345 |
+
self,
|
| 346 |
+
directory: Optional[Path] = None,
|
| 347 |
+
*,
|
| 348 |
+
trusted_policy_files: Optional[dict[str, str]] = None,
|
| 349 |
+
) -> None:
|
| 350 |
self.directory = directory or _resolve_dir()
|
| 351 |
+
self.trusted_policy_files = dict(
|
| 352 |
+
trusted_policy_files or _CANONICAL_POLICY_FILES
|
| 353 |
+
)
|
| 354 |
self.files: list[dict] = []
|
| 355 |
+
self.contract: Optional[dict] = None
|
| 356 |
+
self.contract_errors: list[str] = []
|
| 357 |
+
self.validation_errors: list[str] = []
|
| 358 |
+
self._bundle_sha256: Optional[str] = None
|
| 359 |
self._load()
|
| 360 |
|
| 361 |
def _load(self) -> None:
|
| 362 |
self.files = []
|
| 363 |
+
self.contract = None
|
| 364 |
+
self.contract_errors = []
|
| 365 |
+
self.validation_errors = []
|
| 366 |
+
self._bundle_sha256 = None
|
| 367 |
if not self.directory:
|
| 368 |
+
self.contract_errors = ["policy_directory_unavailable"]
|
| 369 |
return
|
| 370 |
+
try:
|
| 371 |
+
self.contract = _load_enforcement_contract(
|
| 372 |
+
self.directory,
|
| 373 |
+
self.trusted_policy_files,
|
| 374 |
+
)
|
| 375 |
+
except ValueError:
|
| 376 |
+
self.contract_errors = ["enforcement_contract_unavailable"]
|
| 377 |
for p in sorted(self.directory.glob("*.co")):
|
| 378 |
try:
|
| 379 |
raw = p.read_bytes()
|
| 380 |
+
text = raw.decode("utf-8", "strict")
|
| 381 |
flows = _parse_flows(text)
|
| 382 |
pid = None
|
| 383 |
pver = None
|
|
|
|
| 397 |
"flows": flows,
|
| 398 |
"content": text,
|
| 399 |
})
|
| 400 |
+
except (OSError, UnicodeDecodeError):
|
| 401 |
+
self.validation_errors.append(f"policy_file_unreadable:{p.name}")
|
| 402 |
+
self.validation_errors.extend(self._flow_validation_errors())
|
| 403 |
+
self.validation_errors = sorted(set(self.validation_errors))
|
| 404 |
+
self.contract_errors.extend(self._snapshot_contract_errors())
|
| 405 |
+
self.contract_errors = sorted(set(self.contract_errors))
|
| 406 |
+
if not self.contract_errors and not self.validation_errors and self.contract:
|
| 407 |
+
self._bundle_sha256 = hashlib.sha256(
|
| 408 |
+
json.dumps(
|
| 409 |
+
self.contract,
|
| 410 |
+
sort_keys=True,
|
| 411 |
+
separators=(",", ":"),
|
| 412 |
+
).encode("utf-8")
|
| 413 |
+
).hexdigest()
|
| 414 |
+
|
| 415 |
+
def _flow_validation_errors(self) -> list[str]:
|
| 416 |
+
errors = []
|
| 417 |
+
observed_names = []
|
| 418 |
+
for source in self.files:
|
| 419 |
+
if not source["flows"]:
|
| 420 |
+
errors.append(f"policy_file_has_no_flows:{source['name']}")
|
| 421 |
+
if not source["policy_id"]:
|
| 422 |
+
errors.append(f"policy_id_missing:{source['name']}")
|
| 423 |
+
if not source["policy_version"]:
|
| 424 |
+
errors.append(f"policy_version_missing:{source['name']}")
|
| 425 |
+
for flow in source["flows"]:
|
| 426 |
+
name = flow["name"]
|
| 427 |
+
observed_names.append(name)
|
| 428 |
+
if not flow["reason"]:
|
| 429 |
+
errors.append(f"flow_reason_missing:{name}")
|
| 430 |
+
expected_guards = _FLOW_GUARDS.get(name)
|
| 431 |
+
if expected_guards is None:
|
| 432 |
+
errors.append(f"unsupported_flow:{name}")
|
| 433 |
+
elif flow["guards"] != expected_guards:
|
| 434 |
+
errors.append(f"flow_guard_mismatch:{name}")
|
| 435 |
+
duplicates = {
|
| 436 |
+
name for name in observed_names if observed_names.count(name) > 1
|
| 437 |
+
}
|
| 438 |
+
errors.extend(f"duplicate_flow:{name}" for name in sorted(duplicates))
|
| 439 |
+
return errors
|
| 440 |
+
|
| 441 |
+
def _snapshot_contract_errors(self) -> list[str]:
|
| 442 |
+
"""Validate the immutable bytes loaded into this policy snapshot."""
|
| 443 |
+
if not self.directory or not self.contract:
|
| 444 |
+
return self.contract_errors or ["enforcement_contract_unavailable"]
|
| 445 |
+
errors: list[str] = []
|
| 446 |
+
expected_files = self.contract["policy_files"]
|
| 447 |
+
observed_names = {source["name"] for source in self.files}
|
| 448 |
+
if observed_names != set(expected_files):
|
| 449 |
+
errors.append("policy_file_set_mismatch")
|
| 450 |
+
observed_hashes = {
|
| 451 |
+
source["name"]: source["sha256"] for source in self.files
|
| 452 |
+
}
|
| 453 |
+
for name, expected_sha in sorted(expected_files.items()):
|
| 454 |
+
if observed_hashes.get(name) != expected_sha:
|
| 455 |
+
errors.append(f"policy_file_digest_mismatch:{name}")
|
| 456 |
+
return sorted(set(errors))
|
| 457 |
|
| 458 |
@property
|
| 459 |
def loaded(self) -> bool:
|
| 460 |
return bool(self.files)
|
| 461 |
|
| 462 |
+
@property
|
| 463 |
+
def unsupported_flows(self) -> list[str]:
|
| 464 |
+
return sorted(
|
| 465 |
+
{
|
| 466 |
+
flow["name"]
|
| 467 |
+
for flow in self.all_flows()
|
| 468 |
+
if flow["name"] not in _FLOW_LOGIC
|
| 469 |
+
}
|
| 470 |
+
)
|
| 471 |
+
|
| 472 |
+
@property
|
| 473 |
+
def enforcement_ready(self) -> bool:
|
| 474 |
+
return (
|
| 475 |
+
self.loaded
|
| 476 |
+
and not self.unsupported_flows
|
| 477 |
+
and not self.contract_errors
|
| 478 |
+
and not self.validation_errors
|
| 479 |
+
)
|
| 480 |
+
|
| 481 |
+
@property
|
| 482 |
+
def bundle_sha256(self) -> Optional[str]:
|
| 483 |
+
return self._bundle_sha256
|
| 484 |
+
|
| 485 |
def all_flows(self) -> list[dict]:
|
| 486 |
out = []
|
| 487 |
for f in self.files:
|
|
|
|
| 490 |
"policy_version": f["policy_version"]})
|
| 491 |
return out
|
| 492 |
|
| 493 |
+
def evaluate(self, action: dict) -> dict:
|
| 494 |
+
"""Evaluate a proposed action against EVERY loaded flow. Returns which
|
| 495 |
+
flows fired (rule violated -> refuse) and the overall allow/deny. This is
|
| 496 |
+
the policy layer; serve.py calls it before signing an action receipt."""
|
| 497 |
+
action = action or {}
|
| 498 |
+
fired: list[dict] = []
|
| 499 |
+
evaluated: list[str] = []
|
| 500 |
+
unsupported: list[str] = []
|
| 501 |
+
evaluation_errors: list[str] = []
|
| 502 |
+
source_errors = sorted(
|
| 503 |
+
set(self.contract_errors + self.validation_errors)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 504 |
)
|
| 505 |
+
if source_errors:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 506 |
return {
|
| 507 |
"allow": False,
|
| 508 |
"decision": "deny",
|
| 509 |
"fired_flows": [
|
| 510 |
{
|
| 511 |
+
"flow": "exact_source_contract",
|
| 512 |
+
"reason": (
|
| 513 |
+
"POLICY_SOURCE_DRIFT"
|
| 514 |
+
if self.contract_errors
|
| 515 |
+
else "POLICY_SOURCE_INVALID"
|
| 516 |
+
),
|
| 517 |
+
"file": _CONTRACT_NAME,
|
| 518 |
+
"policy_id": "szl-colang-exact-source",
|
| 519 |
+
"policy_version": "1",
|
| 520 |
}
|
|
|
|
| 521 |
],
|
| 522 |
+
"fired_count": 1,
|
| 523 |
+
"flows_evaluated": [],
|
| 524 |
+
"unsupported_flows": self.unsupported_flows,
|
| 525 |
+
"evaluation_errors": [],
|
| 526 |
+
"validation_errors": list(self.validation_errors),
|
| 527 |
+
"source_contract_errors": source_errors,
|
| 528 |
+
"bundle_sha256": None,
|
| 529 |
+
"enforcement_ready": False,
|
| 530 |
+
"matched_count": 1,
|
| 531 |
"policy_files": [
|
| 532 |
{
|
| 533 |
+
"name": f["name"],
|
| 534 |
+
"sha256": f["sha256"],
|
| 535 |
+
"policy_id": f["policy_id"],
|
| 536 |
+
"policy_version": f["policy_version"],
|
| 537 |
}
|
| 538 |
+
for f in self.files
|
| 539 |
],
|
| 540 |
+
"honesty": (
|
| 541 |
+
"The exact reviewed policy/evaluator source contract did not "
|
| 542 |
+
"match current on-disk bytes; enforcement failed closed."
|
| 543 |
+
),
|
| 544 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 545 |
for fl in self.all_flows():
|
| 546 |
name = fl["name"]
|
| 547 |
logic = _FLOW_LOGIC.get(name)
|
| 548 |
evaluated.append(name)
|
| 549 |
if logic is None:
|
| 550 |
+
unsupported.append(name)
|
| 551 |
+
fired.append({
|
| 552 |
+
"flow": name,
|
| 553 |
+
"reason": "UNSUPPORTED_FLOW_FAIL_CLOSED",
|
| 554 |
+
"file": fl["file"],
|
| 555 |
+
"policy_id": fl["policy_id"],
|
| 556 |
+
"policy_version": fl["policy_version"],
|
| 557 |
+
})
|
| 558 |
continue
|
| 559 |
try:
|
| 560 |
violated = bool(logic(action))
|
| 561 |
except Exception:
|
| 562 |
+
violated = True
|
| 563 |
+
evaluation_errors.append(name)
|
| 564 |
if violated:
|
| 565 |
+
reason = (
|
| 566 |
+
"FLOW_EVALUATION_ERROR"
|
| 567 |
+
if name in evaluation_errors
|
| 568 |
+
else fl.get("reason") or name
|
| 569 |
+
)
|
| 570 |
+
fired.append({
|
| 571 |
+
"flow": name,
|
| 572 |
+
"reason": reason,
|
| 573 |
+
"file": fl["file"],
|
| 574 |
+
"policy_id": fl["policy_id"],
|
| 575 |
+
"policy_version": fl["policy_version"],
|
| 576 |
+
})
|
| 577 |
allow = len(fired) == 0
|
| 578 |
return {
|
| 579 |
"allow": allow,
|
|
|
|
| 581 |
"fired_flows": fired,
|
| 582 |
"fired_count": len(fired),
|
| 583 |
"flows_evaluated": evaluated,
|
| 584 |
+
"unsupported_flows": sorted(unsupported),
|
| 585 |
+
"evaluation_errors": sorted(evaluation_errors),
|
| 586 |
+
"source_contract_errors": [],
|
| 587 |
+
"bundle_sha256": self.bundle_sha256,
|
| 588 |
+
"enforcement_ready": not unsupported and not evaluation_errors,
|
| 589 |
"matched_count": len(fired),
|
| 590 |
"policy_files": [{"name": f["name"], "sha256": f["sha256"],
|
| 591 |
"policy_id": f["policy_id"],
|
|
|
|
| 604 |
"nemoguardrails_runtime_present": _NEMO_AVAILABLE,
|
| 605 |
"file_count": len(self.files),
|
| 606 |
"flow_count": len(self.all_flows()),
|
| 607 |
+
"enforcement_ready": self.enforcement_ready,
|
| 608 |
+
"source_contract_errors": list(self.contract_errors),
|
| 609 |
+
"validation_errors": list(self.validation_errors),
|
| 610 |
+
"bundle_sha256": self.bundle_sha256,
|
| 611 |
+
"enforcement_contract": {
|
| 612 |
+
"schema": self.contract.get("schema"),
|
| 613 |
+
"evaluator_region_sha256": self.contract.get(
|
| 614 |
+
"evaluator_region_sha256"
|
| 615 |
+
),
|
| 616 |
+
}
|
| 617 |
+
if self.contract
|
| 618 |
+
else None,
|
| 619 |
"files": self.files,
|
| 620 |
"honesty": (
|
| 621 |
"Policy is FILE-BACKED and version-controlled: each rule is a "
|
|
|
|
| 630 |
}
|
| 631 |
|
| 632 |
|
|
|
|
|
|
|
|
|
|
| 633 |
def get_policy(reload: bool = False) -> ColangPolicy:
|
| 634 |
+
# Policy files are tiny. A fresh immutable snapshot avoids stale singleton
|
| 635 |
+
# state and binds evaluation, evidence, and bundle identity to one read.
|
| 636 |
+
del reload
|
| 637 |
+
return ColangPolicy()
|
| 638 |
|
| 639 |
|
| 640 |
if __name__ == "__main__": # pragma: no cover
|