Spaces:
Running
Running
deploy(hf): sync szl-holdings/a11oy@900209d5b4a435f836fd6229b13e65c9f4e8c23b derived COPY set
Browse filesReusable Dockerfile-COPY-derived deploy from szl-holdings/a11oy 900209d5b4a435f836fd6229b13e65c9f4e8c23b.
Files: 1182 Pruned: 0
Derived from Dockerfile COPY sources (NO hand-maintained allowlist).
Signed-off-by: SZL Holdings <noreply@szlholdings.ai>
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
- gdw_workspace.py +1123 -2
- routers/gdw_frontier.py +200 -1
gdw_workspace.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""Tenant-safe SQLite state, idempotency, quota, and outbox storage for GDW."""
|
| 2 |
|
|
|
|
| 3 |
import contextlib
|
| 4 |
import hashlib
|
| 5 |
import json
|
|
@@ -13,8 +14,10 @@ from datetime import datetime, timedelta, timezone
|
|
| 13 |
from pathlib import Path
|
| 14 |
from typing import Any, Dict, Iterator, Optional, Tuple
|
| 15 |
|
|
|
|
| 16 |
|
| 17 |
-
|
|
|
|
| 18 |
_SCHEMA_LOCK = threading.RLock()
|
| 19 |
_PROCESS_WRITE_LOCK = threading.RLock()
|
| 20 |
_IDENTIFIER = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
@@ -278,6 +281,28 @@ _SCHEMA_STATEMENTS = (
|
|
| 278 |
)
|
| 279 |
""",
|
| 280 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
CREATE INDEX idx_sessions_lifecycle
|
| 282 |
ON session_state(namespace, owner_id, lifecycle, expires_at)
|
| 283 |
""",
|
|
@@ -500,6 +525,7 @@ class GDWWorkspace:
|
|
| 500 |
"ALTER TABLE effect_outbox RENAME TO effect_outbox_v2"
|
| 501 |
)
|
| 502 |
connection.execute(_SCHEMA_STATEMENTS[6])
|
|
|
|
| 503 |
connection.execute(
|
| 504 |
"ALTER TABLE schema_meta "
|
| 505 |
"ADD COLUMN database_generation_id TEXT"
|
|
@@ -794,6 +820,32 @@ class GDWWorkspace:
|
|
| 794 |
finally:
|
| 795 |
connection.execute("PRAGMA foreign_keys=ON")
|
| 796 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 797 |
@staticmethod
|
| 798 |
def _validate_schema(connection: sqlite3.Connection) -> str:
|
| 799 |
row = connection.execute(
|
|
@@ -816,6 +868,7 @@ class GDWWorkspace:
|
|
| 816 |
"receipts",
|
| 817 |
"proof_outbox",
|
| 818 |
"effect_outbox",
|
|
|
|
| 819 |
}
|
| 820 |
missing = required - GDWWorkspace._table_names(connection)
|
| 821 |
if missing:
|
|
@@ -834,6 +887,24 @@ class GDWWorkspace:
|
|
| 834 |
"intent_sha256",
|
| 835 |
"claim_generation",
|
| 836 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 837 |
}
|
| 838 |
for table, expected in expected_columns.items():
|
| 839 |
columns = {
|
|
@@ -895,6 +966,9 @@ class GDWWorkspace:
|
|
| 895 |
).fetchone()
|
| 896 |
if version is not None and int(version[0]) == 2:
|
| 897 |
self._migrate_v2(connection)
|
|
|
|
|
|
|
|
|
|
| 898 |
self.database_generation_id = self._validate_schema(connection)
|
| 899 |
finally:
|
| 900 |
connection.close()
|
|
@@ -2051,7 +2125,1033 @@ class GDWWorkspace:
|
|
| 2051 |
),
|
| 2052 |
)
|
| 2053 |
requeued += int(updated.rowcount)
|
| 2054 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2055 |
|
| 2056 |
def assert_effect_claim(
|
| 2057 |
self,
|
|
@@ -2393,6 +3493,7 @@ class GDWWorkspace:
|
|
| 2393 |
UNION SELECT namespace, owner_id FROM receipts
|
| 2394 |
UNION SELECT namespace, owner_id FROM proof_outbox
|
| 2395 |
UNION SELECT namespace, owner_id FROM effect_outbox
|
|
|
|
| 2396 |
ORDER BY namespace, owner_id
|
| 2397 |
"""
|
| 2398 |
).fetchall()
|
|
@@ -2460,6 +3561,9 @@ class GDWWorkspace:
|
|
| 2460 |
LENGTH(CAST(payload_json AS BLOB)) +
|
| 2461 |
COALESCE(LENGTH(CAST(artifact_json AS BLOB)), 0))
|
| 2462 |
FROM effect_outbox
|
|
|
|
|
|
|
|
|
|
| 2463 |
WHERE namespace = ? AND owner_id = ?), 0)
|
| 2464 |
""",
|
| 2465 |
(
|
|
@@ -2473,6 +3577,8 @@ class GDWWorkspace:
|
|
| 2473 |
owner_id,
|
| 2474 |
namespace,
|
| 2475 |
owner_id,
|
|
|
|
|
|
|
| 2476 |
),
|
| 2477 |
).fetchone()[0]
|
| 2478 |
)
|
|
@@ -2509,6 +3615,7 @@ class GDWWorkspace:
|
|
| 2509 |
UNION SELECT namespace, owner_id FROM receipts
|
| 2510 |
UNION SELECT namespace, owner_id FROM proof_outbox
|
| 2511 |
UNION SELECT namespace, owner_id FROM effect_outbox
|
|
|
|
| 2512 |
"""
|
| 2513 |
).fetchall()
|
| 2514 |
connection.execute("DELETE FROM usage")
|
|
@@ -2745,6 +3852,12 @@ class GDWWorkspace:
|
|
| 2745 |
)
|
| 2746 |
for table in _V1_TABLES
|
| 2747 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2748 |
effect_predicate = (
|
| 2749 |
"status IN ('PENDING', 'CLAIMED')"
|
| 2750 |
if global_scope
|
|
@@ -2813,6 +3926,7 @@ class GDWWorkspace:
|
|
| 2813 |
"invalid_request_digests": 0,
|
| 2814 |
"invalid_receipt_digests": 0,
|
| 2815 |
"invalid_proof_digests": 0,
|
|
|
|
| 2816 |
}
|
| 2817 |
scoped_suffix = (
|
| 2818 |
""
|
|
@@ -2946,6 +4060,13 @@ class GDWWorkspace:
|
|
| 2946 |
json.JSONDecodeError,
|
| 2947 |
):
|
| 2948 |
digest_violations["invalid_proof_digests"] += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2949 |
effect_rows = connection.execute(
|
| 2950 |
"""
|
| 2951 |
SELECT namespace, owner_id, idempotency_key,
|
|
|
|
| 1 |
"""Tenant-safe SQLite state, idempotency, quota, and outbox storage for GDW."""
|
| 2 |
|
| 3 |
+
import base64
|
| 4 |
import contextlib
|
| 5 |
import hashlib
|
| 6 |
import json
|
|
|
|
| 14 |
from pathlib import Path
|
| 15 |
from typing import Any, Dict, Iterator, Optional, Tuple
|
| 16 |
|
| 17 |
+
import szl_dsse
|
| 18 |
|
| 19 |
+
|
| 20 |
+
SCHEMA_VERSION = 4
|
| 21 |
_SCHEMA_LOCK = threading.RLock()
|
| 22 |
_PROCESS_WRITE_LOCK = threading.RLock()
|
| 23 |
_IDENTIFIER = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
|
|
| 281 |
)
|
| 282 |
""",
|
| 283 |
"""
|
| 284 |
+
CREATE TABLE effect_recovery_audit (
|
| 285 |
+
namespace TEXT NOT NULL,
|
| 286 |
+
owner_id TEXT NOT NULL,
|
| 287 |
+
recovery_id TEXT NOT NULL,
|
| 288 |
+
sequence INTEGER NOT NULL CHECK(sequence >= 0),
|
| 289 |
+
credential_key_id TEXT NOT NULL,
|
| 290 |
+
database_generation_id TEXT NOT NULL,
|
| 291 |
+
request_sha256 TEXT NOT NULL,
|
| 292 |
+
outcome_sha256 TEXT NOT NULL,
|
| 293 |
+
governance_sha256 TEXT NOT NULL,
|
| 294 |
+
receipt_sha256 TEXT NOT NULL,
|
| 295 |
+
previous_receipt_sha256 TEXT NOT NULL,
|
| 296 |
+
previous_chain_sha256 TEXT NOT NULL,
|
| 297 |
+
chain_sha256 TEXT NOT NULL,
|
| 298 |
+
dsse_envelope_sha256 TEXT NOT NULL,
|
| 299 |
+
report_json TEXT NOT NULL,
|
| 300 |
+
created_at TEXT NOT NULL,
|
| 301 |
+
PRIMARY KEY(namespace, owner_id, recovery_id),
|
| 302 |
+
UNIQUE(namespace, owner_id, sequence)
|
| 303 |
+
)
|
| 304 |
+
""",
|
| 305 |
+
"""
|
| 306 |
CREATE INDEX idx_sessions_lifecycle
|
| 307 |
ON session_state(namespace, owner_id, lifecycle, expires_at)
|
| 308 |
""",
|
|
|
|
| 525 |
"ALTER TABLE effect_outbox RENAME TO effect_outbox_v2"
|
| 526 |
)
|
| 527 |
connection.execute(_SCHEMA_STATEMENTS[6])
|
| 528 |
+
connection.execute(_SCHEMA_STATEMENTS[7])
|
| 529 |
connection.execute(
|
| 530 |
"ALTER TABLE schema_meta "
|
| 531 |
"ADD COLUMN database_generation_id TEXT"
|
|
|
|
| 820 |
finally:
|
| 821 |
connection.execute("PRAGMA foreign_keys=ON")
|
| 822 |
|
| 823 |
+
@staticmethod
|
| 824 |
+
def _migrate_v3(connection: sqlite3.Connection) -> None:
|
| 825 |
+
"""Add atomically bound recovery audit records without rebinding data."""
|
| 826 |
+
|
| 827 |
+
timestamp = _text_time()
|
| 828 |
+
connection.execute("BEGIN IMMEDIATE")
|
| 829 |
+
try:
|
| 830 |
+
version = connection.execute(
|
| 831 |
+
"SELECT schema_version FROM schema_meta WHERE schema_name = 'gdw'"
|
| 832 |
+
).fetchone()
|
| 833 |
+
if version is None or int(version[0]) != 3:
|
| 834 |
+
raise GDWSchemaError("GDW v3 migration source changed under lock")
|
| 835 |
+
connection.execute(_SCHEMA_STATEMENTS[7])
|
| 836 |
+
connection.execute(
|
| 837 |
+
"""
|
| 838 |
+
UPDATE schema_meta
|
| 839 |
+
SET schema_version = ?, upgraded_at = ?
|
| 840 |
+
WHERE schema_name = 'gdw'
|
| 841 |
+
""",
|
| 842 |
+
(SCHEMA_VERSION, timestamp),
|
| 843 |
+
)
|
| 844 |
+
connection.execute("COMMIT")
|
| 845 |
+
except Exception:
|
| 846 |
+
connection.execute("ROLLBACK")
|
| 847 |
+
raise
|
| 848 |
+
|
| 849 |
@staticmethod
|
| 850 |
def _validate_schema(connection: sqlite3.Connection) -> str:
|
| 851 |
row = connection.execute(
|
|
|
|
| 868 |
"receipts",
|
| 869 |
"proof_outbox",
|
| 870 |
"effect_outbox",
|
| 871 |
+
"effect_recovery_audit",
|
| 872 |
}
|
| 873 |
missing = required - GDWWorkspace._table_names(connection)
|
| 874 |
if missing:
|
|
|
|
| 887 |
"intent_sha256",
|
| 888 |
"claim_generation",
|
| 889 |
},
|
| 890 |
+
"effect_recovery_audit": {
|
| 891 |
+
"namespace",
|
| 892 |
+
"owner_id",
|
| 893 |
+
"recovery_id",
|
| 894 |
+
"sequence",
|
| 895 |
+
"credential_key_id",
|
| 896 |
+
"database_generation_id",
|
| 897 |
+
"request_sha256",
|
| 898 |
+
"outcome_sha256",
|
| 899 |
+
"governance_sha256",
|
| 900 |
+
"receipt_sha256",
|
| 901 |
+
"previous_receipt_sha256",
|
| 902 |
+
"previous_chain_sha256",
|
| 903 |
+
"chain_sha256",
|
| 904 |
+
"dsse_envelope_sha256",
|
| 905 |
+
"report_json",
|
| 906 |
+
"created_at",
|
| 907 |
+
},
|
| 908 |
}
|
| 909 |
for table, expected in expected_columns.items():
|
| 910 |
columns = {
|
|
|
|
| 966 |
).fetchone()
|
| 967 |
if version is not None and int(version[0]) == 2:
|
| 968 |
self._migrate_v2(connection)
|
| 969 |
+
version = (SCHEMA_VERSION,)
|
| 970 |
+
if version is not None and int(version[0]) == 3:
|
| 971 |
+
self._migrate_v3(connection)
|
| 972 |
self.database_generation_id = self._validate_schema(connection)
|
| 973 |
finally:
|
| 974 |
connection.close()
|
|
|
|
| 2125 |
),
|
| 2126 |
)
|
| 2127 |
requeued += int(updated.rowcount)
|
| 2128 |
+
return requeued
|
| 2129 |
+
|
| 2130 |
+
@staticmethod
|
| 2131 |
+
def _recoverable_publication_error(
|
| 2132 |
+
value: Any,
|
| 2133 |
+
*,
|
| 2134 |
+
expected_intent_sha256: str,
|
| 2135 |
+
) -> bool:
|
| 2136 |
+
match = re.fullmatch(
|
| 2137 |
+
r"OSError: \[Errno 95\] Operation not supported: "
|
| 2138 |
+
r"'(?P<stage>[^']*[\\/]\.gdw-artifact-[^'\\/]+\.tmp)' -> "
|
| 2139 |
+
r"'(?P<final>[^']*[\\/](?P<digest>[0-9a-f]{64})\.json)'",
|
| 2140 |
+
str(value or ""),
|
| 2141 |
+
)
|
| 2142 |
+
if match is None or match.group("digest") != expected_intent_sha256:
|
| 2143 |
+
return False
|
| 2144 |
+
stage = match.group("stage").replace("\\", "/")
|
| 2145 |
+
final = match.group("final").replace("\\", "/")
|
| 2146 |
+
return stage.rsplit("/", 1)[0] == final.rsplit("/", 1)[0]
|
| 2147 |
+
|
| 2148 |
+
@staticmethod
|
| 2149 |
+
def _validated_recovery_governance(
|
| 2150 |
+
governance: Any,
|
| 2151 |
+
*,
|
| 2152 |
+
namespace: str,
|
| 2153 |
+
owner_id: str,
|
| 2154 |
+
credential_key_id: str,
|
| 2155 |
+
recovery_id: str,
|
| 2156 |
+
source_revision: str,
|
| 2157 |
+
database_generation_id: str,
|
| 2158 |
+
limit: int,
|
| 2159 |
+
) -> Tuple[Dict[str, Any], str, str]:
|
| 2160 |
+
try:
|
| 2161 |
+
canonical = json.loads(_json_text(governance))
|
| 2162 |
+
if type(canonical) is not dict or set(canonical) != {
|
| 2163 |
+
"schema",
|
| 2164 |
+
"decision",
|
| 2165 |
+
"binding",
|
| 2166 |
+
"binding_sha256",
|
| 2167 |
+
"policy_gateway",
|
| 2168 |
+
}:
|
| 2169 |
+
raise ValueError("recovery governance shape mismatch")
|
| 2170 |
+
expected_binding = {
|
| 2171 |
+
"schema": "szl.gdw.transient-effect-recovery-authorization/v1",
|
| 2172 |
+
"action_type": "gdw.transient-effect-recovery",
|
| 2173 |
+
"namespace": namespace,
|
| 2174 |
+
"owner_id": owner_id,
|
| 2175 |
+
"credential_key_id": credential_key_id,
|
| 2176 |
+
"recovery_id": recovery_id,
|
| 2177 |
+
"source_revision": source_revision,
|
| 2178 |
+
"database_generation_id": database_generation_id,
|
| 2179 |
+
"limit": limit,
|
| 2180 |
+
"failure_class": "hf-hard-link-enotsup/v1",
|
| 2181 |
+
}
|
| 2182 |
+
binding_sha256 = hashlib.sha256(
|
| 2183 |
+
_json_text(expected_binding).encode("utf-8")
|
| 2184 |
+
).hexdigest()
|
| 2185 |
+
expected_witnesses = [
|
| 2186 |
+
{
|
| 2187 |
+
"id": f"principal:{namespace}:{owner_id}:{credential_key_id}",
|
| 2188 |
+
"role": "operator",
|
| 2189 |
+
"attested": True,
|
| 2190 |
+
},
|
| 2191 |
+
{
|
| 2192 |
+
"id": f"workload:szl-holdings/a11oy@{source_revision}",
|
| 2193 |
+
"role": "workload",
|
| 2194 |
+
"attested": True,
|
| 2195 |
+
},
|
| 2196 |
+
]
|
| 2197 |
+
gateway = canonical["policy_gateway"]
|
| 2198 |
+
if (
|
| 2199 |
+
canonical["schema"]
|
| 2200 |
+
!= "szl.gdw.transient-effect-recovery-governance/v1"
|
| 2201 |
+
or canonical["decision"] != "ALLOW"
|
| 2202 |
+
or canonical["binding"] != expected_binding
|
| 2203 |
+
or canonical["binding_sha256"] != binding_sha256
|
| 2204 |
+
or type(gateway) is not dict
|
| 2205 |
+
or set(gateway) != {
|
| 2206 |
+
"decision",
|
| 2207 |
+
"gate",
|
| 2208 |
+
"receipt_hash",
|
| 2209 |
+
"receipt_signed",
|
| 2210 |
+
"receipts_in_eq_out",
|
| 2211 |
+
"action_id",
|
| 2212 |
+
"witnesses",
|
| 2213 |
+
}
|
| 2214 |
+
or gateway["decision"] != "ALLOW"
|
| 2215 |
+
or gateway["gate"] != "ThresholdPolicySeverity"
|
| 2216 |
+
or re.fullmatch(
|
| 2217 |
+
r"[0-9a-f]{64}", str(gateway["receipt_hash"] or "")
|
| 2218 |
+
)
|
| 2219 |
+
is None
|
| 2220 |
+
or gateway["receipt_signed"] is not True
|
| 2221 |
+
or gateway["receipts_in_eq_out"] is not True
|
| 2222 |
+
or gateway["action_id"] != f"gdw-recovery:{binding_sha256}"
|
| 2223 |
+
or gateway["witnesses"] != expected_witnesses
|
| 2224 |
+
):
|
| 2225 |
+
raise ValueError("recovery governance binding mismatch")
|
| 2226 |
+
governance_sha256 = hashlib.sha256(
|
| 2227 |
+
_json_text(canonical).encode("utf-8")
|
| 2228 |
+
).hexdigest()
|
| 2229 |
+
return canonical, binding_sha256, governance_sha256
|
| 2230 |
+
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
| 2231 |
+
raise GDWConfigurationError(
|
| 2232 |
+
"transient effect recovery governance is invalid"
|
| 2233 |
+
) from exc
|
| 2234 |
+
|
| 2235 |
+
@staticmethod
|
| 2236 |
+
def _validated_recovery_audit(row: sqlite3.Row) -> Dict[str, Any]:
|
| 2237 |
+
try:
|
| 2238 |
+
report = json.loads(row["report_json"])
|
| 2239 |
+
if type(report) is not dict:
|
| 2240 |
+
raise ValueError("recovery report must be an object")
|
| 2241 |
+
outcome_fields = {
|
| 2242 |
+
"schema",
|
| 2243 |
+
"status",
|
| 2244 |
+
"recovery_id",
|
| 2245 |
+
"source_revision",
|
| 2246 |
+
"requested_limit",
|
| 2247 |
+
"failure_class",
|
| 2248 |
+
"database_generation_id",
|
| 2249 |
+
"inspected_pending_effects",
|
| 2250 |
+
"eligible_effects",
|
| 2251 |
+
"rescheduled_effects",
|
| 2252 |
+
"attempts_before",
|
| 2253 |
+
"attempts_after",
|
| 2254 |
+
"selection",
|
| 2255 |
+
"selection_sha256",
|
| 2256 |
+
"sqlite_integrity",
|
| 2257 |
+
"claimed_effects",
|
| 2258 |
+
"dead_letter_effects",
|
| 2259 |
+
"invalid_effect_bindings",
|
| 2260 |
+
"invalid_exported_artifacts",
|
| 2261 |
+
"invalid_recovery_audits",
|
| 2262 |
+
"credential_values_recorded",
|
| 2263 |
+
}
|
| 2264 |
+
if set(report) != outcome_fields | {
|
| 2265 |
+
"governance",
|
| 2266 |
+
"audit_receipt",
|
| 2267 |
+
"replayed",
|
| 2268 |
+
}:
|
| 2269 |
+
raise ValueError("recovery report shape mismatch")
|
| 2270 |
+
if (
|
| 2271 |
+
type(report["governance"]) is not dict
|
| 2272 |
+
or type(report["audit_receipt"]) is not dict
|
| 2273 |
+
):
|
| 2274 |
+
raise ValueError("recovery receipt must be an object")
|
| 2275 |
+
outcome = {field: report[field] for field in outcome_fields}
|
| 2276 |
+
receipt = dict(report["audit_receipt"])
|
| 2277 |
+
receipt_payload_fields = {
|
| 2278 |
+
"schema",
|
| 2279 |
+
"operator",
|
| 2280 |
+
"recovery_id",
|
| 2281 |
+
"source_revision",
|
| 2282 |
+
"database_generation_id",
|
| 2283 |
+
"request_sha256",
|
| 2284 |
+
"outcome_sha256",
|
| 2285 |
+
"governance_sha256",
|
| 2286 |
+
"selection_sha256",
|
| 2287 |
+
"rescheduled_effects",
|
| 2288 |
+
"attempts_before",
|
| 2289 |
+
"attempts_after",
|
| 2290 |
+
"sequence",
|
| 2291 |
+
"previous_receipt_sha256",
|
| 2292 |
+
"previous_chain_sha256",
|
| 2293 |
+
"atomic_with_mutation",
|
| 2294 |
+
"created_at",
|
| 2295 |
+
"credential_values_recorded",
|
| 2296 |
+
}
|
| 2297 |
+
receipt_fields = receipt_payload_fields | {
|
| 2298 |
+
"receipt_status",
|
| 2299 |
+
"receipt_sha256",
|
| 2300 |
+
"dsse_envelope_sha256",
|
| 2301 |
+
"chain_sha256",
|
| 2302 |
+
"dsse_envelope",
|
| 2303 |
+
}
|
| 2304 |
+
if set(receipt) != receipt_fields:
|
| 2305 |
+
raise ValueError("recovery receipt shape mismatch")
|
| 2306 |
+
receipt_payload = {
|
| 2307 |
+
field: receipt[field] for field in receipt_payload_fields
|
| 2308 |
+
}
|
| 2309 |
+
if (
|
| 2310 |
+
type(receipt.get("created_at")) is not str
|
| 2311 |
+
or _text_time(receipt["created_at"]) != receipt["created_at"]
|
| 2312 |
+
):
|
| 2313 |
+
raise ValueError("recovery receipt timestamp mismatch")
|
| 2314 |
+
recovery_created_at = _normalise_time(receipt["created_at"])
|
| 2315 |
+
|
| 2316 |
+
def is_digest(value: Any, length: int = 64) -> bool:
|
| 2317 |
+
return type(value) is str and re.fullmatch(
|
| 2318 |
+
rf"[0-9a-f]{{{length}}}", value
|
| 2319 |
+
) is not None
|
| 2320 |
+
|
| 2321 |
+
observed_outcome_sha256 = hashlib.sha256(
|
| 2322 |
+
_json_text(outcome).encode("utf-8")
|
| 2323 |
+
).hexdigest()
|
| 2324 |
+
observed_receipt_sha256 = hashlib.sha256(
|
| 2325 |
+
_json_text(receipt_payload).encode("utf-8")
|
| 2326 |
+
).hexdigest()
|
| 2327 |
+
envelope = receipt["dsse_envelope"]
|
| 2328 |
+
if type(envelope) is not dict:
|
| 2329 |
+
raise ValueError("recovery DSSE envelope shape mismatch")
|
| 2330 |
+
observed_envelope_sha256 = hashlib.sha256(
|
| 2331 |
+
_json_text(envelope).encode("utf-8")
|
| 2332 |
+
).hexdigest()
|
| 2333 |
+
decoded_payload = json.loads(
|
| 2334 |
+
base64.b64decode(
|
| 2335 |
+
str(envelope.get("payload") or ""),
|
| 2336 |
+
validate=True,
|
| 2337 |
+
).decode("utf-8")
|
| 2338 |
+
)
|
| 2339 |
+
if (
|
| 2340 |
+
envelope.get("payloadType") != szl_dsse.KHIPU_PAYLOAD_TYPE
|
| 2341 |
+
or decoded_payload != receipt_payload
|
| 2342 |
+
):
|
| 2343 |
+
raise ValueError("recovery DSSE payload binding mismatch")
|
| 2344 |
+
signed = envelope.get("signed") is True
|
| 2345 |
+
if signed:
|
| 2346 |
+
if (
|
| 2347 |
+
receipt["receipt_status"] != "SIGNED_KHIPU_DSSE"
|
| 2348 |
+
or szl_dsse.verify_envelope(envelope).get("verified") is not True
|
| 2349 |
+
):
|
| 2350 |
+
raise ValueError("recovery DSSE signature is invalid")
|
| 2351 |
+
elif (
|
| 2352 |
+
receipt["receipt_status"] != "UNSIGNED_KHIPU_DSSE"
|
| 2353 |
+
or envelope.get("signed") is not False
|
| 2354 |
+
or envelope.get("signatures") != []
|
| 2355 |
+
or "UNSIGNED" not in str(envelope.get("honesty") or "")
|
| 2356 |
+
):
|
| 2357 |
+
raise ValueError("recovery unsigned DSSE evidence is dishonest")
|
| 2358 |
+
observed_chain_sha256 = hashlib.sha256(
|
| 2359 |
+
_json_text(
|
| 2360 |
+
{
|
| 2361 |
+
"previous_chain_sha256": receipt["previous_chain_sha256"],
|
| 2362 |
+
"receipt_sha256": observed_receipt_sha256,
|
| 2363 |
+
"receipt_status": receipt["receipt_status"],
|
| 2364 |
+
"dsse_envelope_sha256": observed_envelope_sha256,
|
| 2365 |
+
}
|
| 2366 |
+
).encode("utf-8")
|
| 2367 |
+
).hexdigest()
|
| 2368 |
+
expected_operator = {
|
| 2369 |
+
"namespace": row["namespace"],
|
| 2370 |
+
"owner_id": row["owner_id"],
|
| 2371 |
+
"credential_key_id": row["credential_key_id"],
|
| 2372 |
+
}
|
| 2373 |
+
expected_request = {
|
| 2374 |
+
"schema": "szl.gdw.transient-effect-recovery-request/v1",
|
| 2375 |
+
"namespace": row["namespace"],
|
| 2376 |
+
"owner_id": row["owner_id"],
|
| 2377 |
+
"credential_key_id": row["credential_key_id"],
|
| 2378 |
+
"recovery_id": row["recovery_id"],
|
| 2379 |
+
"source_revision": outcome.get("source_revision"),
|
| 2380 |
+
"database_generation_id": row["database_generation_id"],
|
| 2381 |
+
"limit": outcome.get("requested_limit"),
|
| 2382 |
+
"failure_class": outcome.get("failure_class"),
|
| 2383 |
+
"governance_binding_sha256": report["governance"].get(
|
| 2384 |
+
"binding_sha256"
|
| 2385 |
+
),
|
| 2386 |
+
}
|
| 2387 |
+
observed_request_sha256 = hashlib.sha256(
|
| 2388 |
+
_json_text(expected_request).encode("utf-8")
|
| 2389 |
+
).hexdigest()
|
| 2390 |
+
(
|
| 2391 |
+
canonical_governance,
|
| 2392 |
+
_,
|
| 2393 |
+
observed_governance_sha256,
|
| 2394 |
+
) = GDWWorkspace._validated_recovery_governance(
|
| 2395 |
+
report["governance"],
|
| 2396 |
+
namespace=row["namespace"],
|
| 2397 |
+
owner_id=row["owner_id"],
|
| 2398 |
+
credential_key_id=row["credential_key_id"],
|
| 2399 |
+
recovery_id=row["recovery_id"],
|
| 2400 |
+
source_revision=outcome.get("source_revision"),
|
| 2401 |
+
database_generation_id=row["database_generation_id"],
|
| 2402 |
+
limit=outcome.get("requested_limit"),
|
| 2403 |
+
)
|
| 2404 |
+
if canonical_governance != report["governance"]:
|
| 2405 |
+
raise ValueError("recovery governance is not canonical")
|
| 2406 |
+
selection = outcome["selection"]
|
| 2407 |
+
count_fields = (
|
| 2408 |
+
"inspected_pending_effects",
|
| 2409 |
+
"eligible_effects",
|
| 2410 |
+
"rescheduled_effects",
|
| 2411 |
+
"attempts_before",
|
| 2412 |
+
"attempts_after",
|
| 2413 |
+
"claimed_effects",
|
| 2414 |
+
"dead_letter_effects",
|
| 2415 |
+
"invalid_effect_bindings",
|
| 2416 |
+
"invalid_exported_artifacts",
|
| 2417 |
+
"invalid_recovery_audits",
|
| 2418 |
+
)
|
| 2419 |
+
counts = {field: outcome[field] for field in count_fields}
|
| 2420 |
+
if (
|
| 2421 |
+
type(selection) is not list
|
| 2422 |
+
or any(type(value) is not int or value < 0 for value in counts.values())
|
| 2423 |
+
):
|
| 2424 |
+
raise ValueError("recovery accounting is invalid")
|
| 2425 |
+
selection_fields = {
|
| 2426 |
+
"namespace",
|
| 2427 |
+
"owner_id",
|
| 2428 |
+
"idempotency_key",
|
| 2429 |
+
"database_generation_id",
|
| 2430 |
+
"request_id",
|
| 2431 |
+
"kind",
|
| 2432 |
+
"receipt_hash",
|
| 2433 |
+
"payload_sha256",
|
| 2434 |
+
"intent_sha256",
|
| 2435 |
+
"attempts",
|
| 2436 |
+
"max_attempts",
|
| 2437 |
+
"next_attempt_at",
|
| 2438 |
+
"claim_generation",
|
| 2439 |
+
"last_error_sha256",
|
| 2440 |
+
}
|
| 2441 |
+
for item in selection:
|
| 2442 |
+
if type(item) is not dict or set(item) != selection_fields:
|
| 2443 |
+
raise ValueError("recovery selection shape mismatch")
|
| 2444 |
+
if any(
|
| 2445 |
+
_IDENTIFIER.fullmatch(str(item.get(field) or "")) is None
|
| 2446 |
+
for field in (
|
| 2447 |
+
"namespace",
|
| 2448 |
+
"owner_id",
|
| 2449 |
+
"idempotency_key",
|
| 2450 |
+
"request_id",
|
| 2451 |
+
)
|
| 2452 |
+
):
|
| 2453 |
+
raise ValueError("recovery selection identity mismatch")
|
| 2454 |
+
if (
|
| 2455 |
+
item.get("database_generation_id")
|
| 2456 |
+
!= row["database_generation_id"]
|
| 2457 |
+
or item.get("kind")
|
| 2458 |
+
not in {"receipt_projection", "proof_export"}
|
| 2459 |
+
or (
|
| 2460 |
+
item.get("receipt_hash") is not None
|
| 2461 |
+
and not is_digest(item.get("receipt_hash"))
|
| 2462 |
+
)
|
| 2463 |
+
or any(
|
| 2464 |
+
not is_digest(item.get(field))
|
| 2465 |
+
for field in (
|
| 2466 |
+
"payload_sha256",
|
| 2467 |
+
"intent_sha256",
|
| 2468 |
+
"last_error_sha256",
|
| 2469 |
+
)
|
| 2470 |
+
)
|
| 2471 |
+
or type(item.get("attempts")) is not int
|
| 2472 |
+
or type(item.get("max_attempts")) is not int
|
| 2473 |
+
or not 0 < item["attempts"] < item["max_attempts"]
|
| 2474 |
+
or type(item.get("claim_generation")) is not int
|
| 2475 |
+
or item["claim_generation"] < 0
|
| 2476 |
+
or type(item.get("next_attempt_at")) is not str
|
| 2477 |
+
or _text_time(item["next_attempt_at"])
|
| 2478 |
+
!= item["next_attempt_at"]
|
| 2479 |
+
or _normalise_time(item["next_attempt_at"])
|
| 2480 |
+
<= recovery_created_at
|
| 2481 |
+
):
|
| 2482 |
+
raise ValueError("recovery selection binding mismatch")
|
| 2483 |
+
observed_selection_sha256 = hashlib.sha256(
|
| 2484 |
+
_json_text(selection).encode("utf-8")
|
| 2485 |
+
).hexdigest()
|
| 2486 |
+
status = outcome["status"]
|
| 2487 |
+
status_contract = (
|
| 2488 |
+
(
|
| 2489 |
+
status == "RESCHEDULED"
|
| 2490 |
+
and counts["rescheduled_effects"] > 0
|
| 2491 |
+
and counts["eligible_effects"]
|
| 2492 |
+
== counts["rescheduled_effects"]
|
| 2493 |
+
and counts["claimed_effects"] == 0
|
| 2494 |
+
)
|
| 2495 |
+
or (
|
| 2496 |
+
status == "NO_ELIGIBLE_EFFECTS"
|
| 2497 |
+
and counts["eligible_effects"] == 0
|
| 2498 |
+
and counts["rescheduled_effects"] == 0
|
| 2499 |
+
and counts["claimed_effects"] == 0
|
| 2500 |
+
)
|
| 2501 |
+
or (
|
| 2502 |
+
status == "DEFERRED_ACTIVE_CLAIM"
|
| 2503 |
+
and counts["eligible_effects"] == 0
|
| 2504 |
+
and counts["rescheduled_effects"] == 0
|
| 2505 |
+
and counts["claimed_effects"] > 0
|
| 2506 |
+
)
|
| 2507 |
+
)
|
| 2508 |
+
if (
|
| 2509 |
+
report["replayed"] is not False
|
| 2510 |
+
or outcome.get("schema")
|
| 2511 |
+
!= "szl.gdw.transient-effect-recovery/v2"
|
| 2512 |
+
or receipt.get("schema")
|
| 2513 |
+
!= "szl.gdw.transient-effect-recovery-receipt/v2"
|
| 2514 |
+
or receipt.get("operator") != expected_operator
|
| 2515 |
+
or any(
|
| 2516 |
+
_IDENTIFIER.fullmatch(str(value or "")) is None
|
| 2517 |
+
for value in expected_operator.values()
|
| 2518 |
+
)
|
| 2519 |
+
or receipt.get("recovery_id") != row["recovery_id"]
|
| 2520 |
+
or outcome.get("recovery_id") != row["recovery_id"]
|
| 2521 |
+
or re.fullmatch(
|
| 2522 |
+
r"[0-9a-f]{40}",
|
| 2523 |
+
str(outcome.get("source_revision") or ""),
|
| 2524 |
+
)
|
| 2525 |
+
is None
|
| 2526 |
+
or receipt.get("source_revision")
|
| 2527 |
+
!= outcome.get("source_revision")
|
| 2528 |
+
or receipt.get("database_generation_id")
|
| 2529 |
+
!= row["database_generation_id"]
|
| 2530 |
+
or outcome.get("database_generation_id")
|
| 2531 |
+
!= row["database_generation_id"]
|
| 2532 |
+
or outcome.get("failure_class") != "hf-hard-link-enotsup/v1"
|
| 2533 |
+
or type(outcome.get("requested_limit")) is not int
|
| 2534 |
+
or not 1 <= outcome["requested_limit"] <= 1_000
|
| 2535 |
+
or outcome.get("sqlite_integrity") != "ok"
|
| 2536 |
+
or outcome.get("credential_values_recorded") is not False
|
| 2537 |
+
or receipt.get("credential_values_recorded") is not False
|
| 2538 |
+
or receipt.get("atomic_with_mutation") is not True
|
| 2539 |
+
or type(receipt.get("sequence")) is not int
|
| 2540 |
+
or receipt["sequence"] < 0
|
| 2541 |
+
or receipt["sequence"] != row["sequence"]
|
| 2542 |
+
or not is_digest(receipt.get("previous_receipt_sha256"))
|
| 2543 |
+
or receipt.get("previous_receipt_sha256")
|
| 2544 |
+
!= row["previous_receipt_sha256"]
|
| 2545 |
+
or not is_digest(receipt.get("previous_chain_sha256"))
|
| 2546 |
+
or receipt.get("previous_chain_sha256")
|
| 2547 |
+
!= row["previous_chain_sha256"]
|
| 2548 |
+
or counts["inspected_pending_effects"]
|
| 2549 |
+
< max(counts["eligible_effects"], counts["claimed_effects"])
|
| 2550 |
+
or counts["attempts_before"] != counts["attempts_after"]
|
| 2551 |
+
or counts["attempts_before"]
|
| 2552 |
+
!= sum(item["attempts"] for item in selection)
|
| 2553 |
+
or len(selection) != counts["rescheduled_effects"]
|
| 2554 |
+
or any(
|
| 2555 |
+
counts[field] != 0
|
| 2556 |
+
for field in (
|
| 2557 |
+
"dead_letter_effects",
|
| 2558 |
+
"invalid_effect_bindings",
|
| 2559 |
+
"invalid_exported_artifacts",
|
| 2560 |
+
"invalid_recovery_audits",
|
| 2561 |
+
)
|
| 2562 |
+
)
|
| 2563 |
+
or not status_contract
|
| 2564 |
+
or observed_selection_sha256 != outcome.get("selection_sha256")
|
| 2565 |
+
or receipt.get("created_at") != row["created_at"]
|
| 2566 |
+
or receipt.get("request_sha256") != row["request_sha256"]
|
| 2567 |
+
or observed_request_sha256 != row["request_sha256"]
|
| 2568 |
+
or receipt.get("outcome_sha256") != row["outcome_sha256"]
|
| 2569 |
+
or observed_outcome_sha256 != row["outcome_sha256"]
|
| 2570 |
+
or receipt.get("governance_sha256")
|
| 2571 |
+
!= row["governance_sha256"]
|
| 2572 |
+
or observed_governance_sha256 != row["governance_sha256"]
|
| 2573 |
+
or receipt.get("selection_sha256")
|
| 2574 |
+
!= outcome.get("selection_sha256")
|
| 2575 |
+
or receipt.get("rescheduled_effects")
|
| 2576 |
+
!= outcome.get("rescheduled_effects")
|
| 2577 |
+
or receipt.get("attempts_before")
|
| 2578 |
+
!= outcome.get("attempts_before")
|
| 2579 |
+
or receipt.get("attempts_after")
|
| 2580 |
+
!= outcome.get("attempts_after")
|
| 2581 |
+
or receipt.get("receipt_sha256") != row["receipt_sha256"]
|
| 2582 |
+
or observed_receipt_sha256 != row["receipt_sha256"]
|
| 2583 |
+
or receipt.get("dsse_envelope_sha256")
|
| 2584 |
+
!= row["dsse_envelope_sha256"]
|
| 2585 |
+
or observed_envelope_sha256 != row["dsse_envelope_sha256"]
|
| 2586 |
+
or receipt.get("chain_sha256") != row["chain_sha256"]
|
| 2587 |
+
or observed_chain_sha256 != row["chain_sha256"]
|
| 2588 |
+
):
|
| 2589 |
+
raise ValueError("recovery audit binding mismatch")
|
| 2590 |
+
report["audit_receipt"] = receipt
|
| 2591 |
+
report["replayed"] = False
|
| 2592 |
+
return report
|
| 2593 |
+
except (
|
| 2594 |
+
AttributeError,
|
| 2595 |
+
KeyError,
|
| 2596 |
+
OverflowError,
|
| 2597 |
+
TypeError,
|
| 2598 |
+
ValueError,
|
| 2599 |
+
json.JSONDecodeError,
|
| 2600 |
+
) as exc:
|
| 2601 |
+
raise GDWConfigurationError(
|
| 2602 |
+
"transient effect recovery audit is invalid"
|
| 2603 |
+
) from exc
|
| 2604 |
+
|
| 2605 |
+
def _recovery_audit_chain_errors(
|
| 2606 |
+
self,
|
| 2607 |
+
connection: sqlite3.Connection,
|
| 2608 |
+
*,
|
| 2609 |
+
namespace: Optional[str] = None,
|
| 2610 |
+
owner_id: Optional[str] = None,
|
| 2611 |
+
) -> int:
|
| 2612 |
+
predicate = ""
|
| 2613 |
+
params: Tuple[Any, ...] = ()
|
| 2614 |
+
if namespace is not None or owner_id is not None:
|
| 2615 |
+
if namespace is None or owner_id is None:
|
| 2616 |
+
raise GDWConfigurationError(
|
| 2617 |
+
"recovery audit chain scope must include namespace and owner"
|
| 2618 |
+
)
|
| 2619 |
+
predicate = " WHERE namespace = ? AND owner_id = ?"
|
| 2620 |
+
params = (namespace, owner_id)
|
| 2621 |
+
rows = connection.execute(
|
| 2622 |
+
"SELECT * FROM effect_recovery_audit"
|
| 2623 |
+
+ predicate
|
| 2624 |
+
+ " ORDER BY namespace, owner_id, sequence",
|
| 2625 |
+
params,
|
| 2626 |
+
).fetchall()
|
| 2627 |
+
errors = 0
|
| 2628 |
+
identity: Optional[Tuple[str, str]] = None
|
| 2629 |
+
expected_sequence = 0
|
| 2630 |
+
previous_receipt_sha256 = "0" * 64
|
| 2631 |
+
previous_chain_sha256 = "0" * 64
|
| 2632 |
+
for row in rows:
|
| 2633 |
+
row_identity = (row["namespace"], row["owner_id"])
|
| 2634 |
+
if row_identity != identity:
|
| 2635 |
+
identity = row_identity
|
| 2636 |
+
expected_sequence = 0
|
| 2637 |
+
previous_receipt_sha256 = "0" * 64
|
| 2638 |
+
previous_chain_sha256 = "0" * 64
|
| 2639 |
+
row_invalid = False
|
| 2640 |
+
try:
|
| 2641 |
+
self._validated_recovery_audit(row)
|
| 2642 |
+
except GDWConfigurationError:
|
| 2643 |
+
row_invalid = True
|
| 2644 |
+
if (
|
| 2645 |
+
row["sequence"] != expected_sequence
|
| 2646 |
+
or row["previous_receipt_sha256"]
|
| 2647 |
+
!= previous_receipt_sha256
|
| 2648 |
+
or row["previous_chain_sha256"] != previous_chain_sha256
|
| 2649 |
+
):
|
| 2650 |
+
row_invalid = True
|
| 2651 |
+
errors += int(row_invalid)
|
| 2652 |
+
expected_sequence += 1
|
| 2653 |
+
previous_receipt_sha256 = str(row["receipt_sha256"])
|
| 2654 |
+
previous_chain_sha256 = str(row["chain_sha256"])
|
| 2655 |
+
return errors
|
| 2656 |
+
|
| 2657 |
+
def recover_retry_scheduled_effects(
|
| 2658 |
+
self,
|
| 2659 |
+
*,
|
| 2660 |
+
recovery_id: str,
|
| 2661 |
+
credential_key_id: str,
|
| 2662 |
+
expected_source_revision: str,
|
| 2663 |
+
expected_database_generation_id: str,
|
| 2664 |
+
governance: Dict[str, Any],
|
| 2665 |
+
now: Optional[Any] = None,
|
| 2666 |
+
limit: int = 100,
|
| 2667 |
+
) -> Dict[str, Any]:
|
| 2668 |
+
"""Make only integrity-bound legacy HF publication failures due now."""
|
| 2669 |
+
|
| 2670 |
+
canonical_recovery_id = _checked_identity(recovery_id, "recovery_id")
|
| 2671 |
+
canonical_key_id = _checked_identity(
|
| 2672 |
+
credential_key_id,
|
| 2673 |
+
"credential_key_id",
|
| 2674 |
+
)
|
| 2675 |
+
source_revision = str(expected_source_revision or "").strip().lower()
|
| 2676 |
+
generation_id = str(
|
| 2677 |
+
expected_database_generation_id or ""
|
| 2678 |
+
).strip().lower()
|
| 2679 |
+
if re.fullmatch(r"[0-9a-f]{40}", source_revision) is None:
|
| 2680 |
+
raise GDWConfigurationError(
|
| 2681 |
+
"expected_source_revision must be a full Git SHA"
|
| 2682 |
+
)
|
| 2683 |
+
if (
|
| 2684 |
+
re.fullmatch(r"[0-9a-f]{32}", generation_id) is None
|
| 2685 |
+
or generation_id != self.database_generation_id
|
| 2686 |
+
):
|
| 2687 |
+
raise GDWConfigurationError(
|
| 2688 |
+
"transient effect recovery database generation mismatch"
|
| 2689 |
+
)
|
| 2690 |
+
if type(limit) is not int or not 1 <= limit <= 1_000:
|
| 2691 |
+
raise GDWConfigurationError(
|
| 2692 |
+
"transient effect recovery limit must be between 1 and 1000"
|
| 2693 |
+
)
|
| 2694 |
+
|
| 2695 |
+
current = _normalise_time(now)
|
| 2696 |
+
now_text = current.isoformat()
|
| 2697 |
+
(
|
| 2698 |
+
canonical_governance,
|
| 2699 |
+
governance_binding_sha256,
|
| 2700 |
+
governance_sha256,
|
| 2701 |
+
) = self._validated_recovery_governance(
|
| 2702 |
+
governance,
|
| 2703 |
+
namespace=self.namespace,
|
| 2704 |
+
owner_id=self.owner_id,
|
| 2705 |
+
credential_key_id=canonical_key_id,
|
| 2706 |
+
recovery_id=canonical_recovery_id,
|
| 2707 |
+
source_revision=source_revision,
|
| 2708 |
+
database_generation_id=generation_id,
|
| 2709 |
+
limit=limit,
|
| 2710 |
+
)
|
| 2711 |
+
request = {
|
| 2712 |
+
"schema": "szl.gdw.transient-effect-recovery-request/v1",
|
| 2713 |
+
"namespace": self.namespace,
|
| 2714 |
+
"owner_id": self.owner_id,
|
| 2715 |
+
"credential_key_id": canonical_key_id,
|
| 2716 |
+
"recovery_id": canonical_recovery_id,
|
| 2717 |
+
"source_revision": source_revision,
|
| 2718 |
+
"database_generation_id": generation_id,
|
| 2719 |
+
"limit": limit,
|
| 2720 |
+
"failure_class": "hf-hard-link-enotsup/v1",
|
| 2721 |
+
"governance_binding_sha256": governance_binding_sha256,
|
| 2722 |
+
}
|
| 2723 |
+
request_sha256 = hashlib.sha256(
|
| 2724 |
+
_json_text(request).encode("utf-8")
|
| 2725 |
+
).hexdigest()
|
| 2726 |
+
empty_selection_sha256 = hashlib.sha256(b"[]").hexdigest()
|
| 2727 |
+
transient_error = (
|
| 2728 |
+
"OSError: [Errno 95] Operation not supported: "
|
| 2729 |
+
"'%/.gdw-artifact-%.tmp' -> '%"
|
| 2730 |
+
)
|
| 2731 |
+
|
| 2732 |
+
with self.transaction() as connection:
|
| 2733 |
+
cached = connection.execute(
|
| 2734 |
+
"""
|
| 2735 |
+
SELECT * FROM effect_recovery_audit
|
| 2736 |
+
WHERE namespace = ? AND owner_id = ? AND recovery_id = ?
|
| 2737 |
+
""",
|
| 2738 |
+
(self.namespace, self.owner_id, canonical_recovery_id),
|
| 2739 |
+
).fetchone()
|
| 2740 |
+
if cached is not None:
|
| 2741 |
+
if cached["request_sha256"] != request_sha256:
|
| 2742 |
+
raise GDWConfigurationError(
|
| 2743 |
+
"recovery_id was already used with different content"
|
| 2744 |
+
)
|
| 2745 |
+
report = self._validated_recovery_audit(cached)
|
| 2746 |
+
if self._recovery_audit_chain_errors(
|
| 2747 |
+
connection,
|
| 2748 |
+
namespace=self.namespace,
|
| 2749 |
+
owner_id=self.owner_id,
|
| 2750 |
+
):
|
| 2751 |
+
raise GDWConfigurationError(
|
| 2752 |
+
"transient effect recovery audit chain is invalid"
|
| 2753 |
+
)
|
| 2754 |
+
report["replayed"] = True
|
| 2755 |
+
return report
|
| 2756 |
+
|
| 2757 |
+
before = self.integrity(
|
| 2758 |
+
global_scope=True,
|
| 2759 |
+
connection=connection,
|
| 2760 |
+
)
|
| 2761 |
+
if (
|
| 2762 |
+
before.get("ok") is not True
|
| 2763 |
+
or before.get("sqlite_integrity") != "ok"
|
| 2764 |
+
or before.get("dead_letter_effects") != 0
|
| 2765 |
+
or before.get("pending_proofs") != 0
|
| 2766 |
+
or before.get("invalid_effect_bindings") != 0
|
| 2767 |
+
or before.get("invalid_exported_artifacts") != 0
|
| 2768 |
+
or before.get("invalid_recovery_audits") != 0
|
| 2769 |
+
):
|
| 2770 |
+
raise GDWConfigurationError(
|
| 2771 |
+
"transient effect recovery refused by global integrity"
|
| 2772 |
+
)
|
| 2773 |
+
|
| 2774 |
+
def record(outcome: Dict[str, Any]) -> Dict[str, Any]:
|
| 2775 |
+
outcome_sha256 = hashlib.sha256(
|
| 2776 |
+
_json_text(outcome).encode("utf-8")
|
| 2777 |
+
).hexdigest()
|
| 2778 |
+
previous = connection.execute(
|
| 2779 |
+
"""
|
| 2780 |
+
SELECT sequence, receipt_sha256, chain_sha256
|
| 2781 |
+
FROM effect_recovery_audit
|
| 2782 |
+
WHERE namespace = ? AND owner_id = ?
|
| 2783 |
+
ORDER BY sequence DESC
|
| 2784 |
+
LIMIT 1
|
| 2785 |
+
""",
|
| 2786 |
+
(self.namespace, self.owner_id),
|
| 2787 |
+
).fetchone()
|
| 2788 |
+
sequence = 0 if previous is None else int(previous["sequence"]) + 1
|
| 2789 |
+
previous_receipt_sha256 = (
|
| 2790 |
+
"0" * 64 if previous is None else previous["receipt_sha256"]
|
| 2791 |
+
)
|
| 2792 |
+
previous_chain_sha256 = (
|
| 2793 |
+
"0" * 64 if previous is None else previous["chain_sha256"]
|
| 2794 |
+
)
|
| 2795 |
+
receipt_payload = {
|
| 2796 |
+
"schema": "szl.gdw.transient-effect-recovery-receipt/v2",
|
| 2797 |
+
"operator": {
|
| 2798 |
+
"namespace": self.namespace,
|
| 2799 |
+
"owner_id": self.owner_id,
|
| 2800 |
+
"credential_key_id": canonical_key_id,
|
| 2801 |
+
},
|
| 2802 |
+
"recovery_id": canonical_recovery_id,
|
| 2803 |
+
"source_revision": source_revision,
|
| 2804 |
+
"database_generation_id": generation_id,
|
| 2805 |
+
"request_sha256": request_sha256,
|
| 2806 |
+
"outcome_sha256": outcome_sha256,
|
| 2807 |
+
"governance_sha256": governance_sha256,
|
| 2808 |
+
"selection_sha256": outcome["selection_sha256"],
|
| 2809 |
+
"rescheduled_effects": outcome["rescheduled_effects"],
|
| 2810 |
+
"attempts_before": outcome["attempts_before"],
|
| 2811 |
+
"attempts_after": outcome["attempts_after"],
|
| 2812 |
+
"sequence": sequence,
|
| 2813 |
+
"previous_receipt_sha256": previous_receipt_sha256,
|
| 2814 |
+
"previous_chain_sha256": previous_chain_sha256,
|
| 2815 |
+
"atomic_with_mutation": True,
|
| 2816 |
+
"created_at": now_text,
|
| 2817 |
+
"credential_values_recorded": False,
|
| 2818 |
+
}
|
| 2819 |
+
receipt_sha256 = hashlib.sha256(
|
| 2820 |
+
_json_text(receipt_payload).encode("utf-8")
|
| 2821 |
+
).hexdigest()
|
| 2822 |
+
dsse_envelope = szl_dsse.sign_payload(
|
| 2823 |
+
receipt_payload,
|
| 2824 |
+
szl_dsse.KHIPU_PAYLOAD_TYPE,
|
| 2825 |
+
)
|
| 2826 |
+
signed = dsse_envelope.get("signed") is True
|
| 2827 |
+
signatures = dsse_envelope.get("signatures")
|
| 2828 |
+
if (
|
| 2829 |
+
type(signatures) is not list
|
| 2830 |
+
or (signed and len(signatures) != 1)
|
| 2831 |
+
or (not signed and signatures != [])
|
| 2832 |
+
):
|
| 2833 |
+
raise GDWConfigurationError(
|
| 2834 |
+
"transient recovery DSSE signer returned invalid evidence"
|
| 2835 |
+
)
|
| 2836 |
+
receipt_status = (
|
| 2837 |
+
"SIGNED_KHIPU_DSSE" if signed else "UNSIGNED_KHIPU_DSSE"
|
| 2838 |
+
)
|
| 2839 |
+
dsse_envelope_sha256 = hashlib.sha256(
|
| 2840 |
+
_json_text(dsse_envelope).encode("utf-8")
|
| 2841 |
+
).hexdigest()
|
| 2842 |
+
chain_sha256 = hashlib.sha256(
|
| 2843 |
+
_json_text(
|
| 2844 |
+
{
|
| 2845 |
+
"previous_chain_sha256": previous_chain_sha256,
|
| 2846 |
+
"receipt_sha256": receipt_sha256,
|
| 2847 |
+
"receipt_status": receipt_status,
|
| 2848 |
+
"dsse_envelope_sha256": dsse_envelope_sha256,
|
| 2849 |
+
}
|
| 2850 |
+
).encode("utf-8")
|
| 2851 |
+
).hexdigest()
|
| 2852 |
+
receipt = {
|
| 2853 |
+
**receipt_payload,
|
| 2854 |
+
"receipt_status": receipt_status,
|
| 2855 |
+
"receipt_sha256": receipt_sha256,
|
| 2856 |
+
"dsse_envelope_sha256": dsse_envelope_sha256,
|
| 2857 |
+
"chain_sha256": chain_sha256,
|
| 2858 |
+
"dsse_envelope": dsse_envelope,
|
| 2859 |
+
}
|
| 2860 |
+
report = {
|
| 2861 |
+
**outcome,
|
| 2862 |
+
"governance": canonical_governance,
|
| 2863 |
+
"audit_receipt": receipt,
|
| 2864 |
+
"replayed": False,
|
| 2865 |
+
}
|
| 2866 |
+
report_text = _json_text(report)
|
| 2867 |
+
self._reserve_usage(
|
| 2868 |
+
connection,
|
| 2869 |
+
self.namespace,
|
| 2870 |
+
self.owner_id,
|
| 2871 |
+
stored_bytes=_byte_len(report_text),
|
| 2872 |
+
)
|
| 2873 |
+
connection.execute(
|
| 2874 |
+
"""
|
| 2875 |
+
INSERT INTO effect_recovery_audit(
|
| 2876 |
+
namespace, owner_id, recovery_id, sequence,
|
| 2877 |
+
credential_key_id, database_generation_id,
|
| 2878 |
+
request_sha256, outcome_sha256, governance_sha256,
|
| 2879 |
+
receipt_sha256, previous_receipt_sha256,
|
| 2880 |
+
previous_chain_sha256, chain_sha256,
|
| 2881 |
+
dsse_envelope_sha256, report_json, created_at
|
| 2882 |
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
| 2883 |
+
""",
|
| 2884 |
+
(
|
| 2885 |
+
self.namespace,
|
| 2886 |
+
self.owner_id,
|
| 2887 |
+
canonical_recovery_id,
|
| 2888 |
+
sequence,
|
| 2889 |
+
canonical_key_id,
|
| 2890 |
+
generation_id,
|
| 2891 |
+
request_sha256,
|
| 2892 |
+
outcome_sha256,
|
| 2893 |
+
governance_sha256,
|
| 2894 |
+
receipt_sha256,
|
| 2895 |
+
previous_receipt_sha256,
|
| 2896 |
+
previous_chain_sha256,
|
| 2897 |
+
chain_sha256,
|
| 2898 |
+
dsse_envelope_sha256,
|
| 2899 |
+
report_text,
|
| 2900 |
+
now_text,
|
| 2901 |
+
),
|
| 2902 |
+
)
|
| 2903 |
+
return report
|
| 2904 |
+
|
| 2905 |
+
if before.get("claimed_effects") != 0:
|
| 2906 |
+
return record(
|
| 2907 |
+
{
|
| 2908 |
+
"schema": "szl.gdw.transient-effect-recovery/v2",
|
| 2909 |
+
"status": "DEFERRED_ACTIVE_CLAIM",
|
| 2910 |
+
"recovery_id": canonical_recovery_id,
|
| 2911 |
+
"source_revision": source_revision,
|
| 2912 |
+
"requested_limit": limit,
|
| 2913 |
+
"failure_class": "hf-hard-link-enotsup/v1",
|
| 2914 |
+
"database_generation_id": generation_id,
|
| 2915 |
+
"inspected_pending_effects": before["pending_effects"],
|
| 2916 |
+
"eligible_effects": 0,
|
| 2917 |
+
"rescheduled_effects": 0,
|
| 2918 |
+
"attempts_before": 0,
|
| 2919 |
+
"attempts_after": 0,
|
| 2920 |
+
"selection": [],
|
| 2921 |
+
"selection_sha256": empty_selection_sha256,
|
| 2922 |
+
"sqlite_integrity": before["sqlite_integrity"],
|
| 2923 |
+
"claimed_effects": before["claimed_effects"],
|
| 2924 |
+
"dead_letter_effects": before["dead_letter_effects"],
|
| 2925 |
+
"invalid_effect_bindings": before[
|
| 2926 |
+
"invalid_effect_bindings"
|
| 2927 |
+
],
|
| 2928 |
+
"invalid_exported_artifacts": before[
|
| 2929 |
+
"invalid_exported_artifacts"
|
| 2930 |
+
],
|
| 2931 |
+
"invalid_recovery_audits": before[
|
| 2932 |
+
"invalid_recovery_audits"
|
| 2933 |
+
],
|
| 2934 |
+
"credential_values_recorded": False,
|
| 2935 |
+
}
|
| 2936 |
+
)
|
| 2937 |
+
|
| 2938 |
+
eligibility = """
|
| 2939 |
+
status = 'PENDING'
|
| 2940 |
+
AND attempts > 0
|
| 2941 |
+
AND attempts < max_attempts
|
| 2942 |
+
AND next_attempt_at > ?
|
| 2943 |
+
AND database_generation_id = ?
|
| 2944 |
+
AND kind IN ('receipt_projection', 'proof_export')
|
| 2945 |
+
AND last_error LIKE ?
|
| 2946 |
+
AND lease_owner IS NULL
|
| 2947 |
+
AND lease_until IS NULL
|
| 2948 |
+
AND artifact_json IS NULL
|
| 2949 |
+
AND exported_at IS NULL
|
| 2950 |
+
AND tombstoned_at IS NULL
|
| 2951 |
+
"""
|
| 2952 |
+
eligible_total = int(
|
| 2953 |
+
connection.execute(
|
| 2954 |
+
f"SELECT COUNT(*) FROM effect_outbox WHERE {eligibility}",
|
| 2955 |
+
(now_text, generation_id, transient_error),
|
| 2956 |
+
).fetchone()[0]
|
| 2957 |
+
)
|
| 2958 |
+
if eligible_total > limit:
|
| 2959 |
+
raise GDWConfigurationError(
|
| 2960 |
+
"transient effect recovery exceeds the bounded limit"
|
| 2961 |
+
)
|
| 2962 |
+
rows = connection.execute(
|
| 2963 |
+
f"""
|
| 2964 |
+
SELECT * FROM effect_outbox
|
| 2965 |
+
WHERE {eligibility}
|
| 2966 |
+
ORDER BY next_attempt_at, created_at, idempotency_key
|
| 2967 |
+
LIMIT ?
|
| 2968 |
+
""",
|
| 2969 |
+
(now_text, generation_id, transient_error, limit),
|
| 2970 |
+
).fetchall()
|
| 2971 |
+
selection = []
|
| 2972 |
+
original_rows = {}
|
| 2973 |
+
for row in rows:
|
| 2974 |
+
try:
|
| 2975 |
+
payload = json.loads(row["payload_json"])
|
| 2976 |
+
except (TypeError, json.JSONDecodeError) as exc:
|
| 2977 |
+
raise GDWConfigurationError(
|
| 2978 |
+
"transient effect recovery found invalid payload"
|
| 2979 |
+
) from exc
|
| 2980 |
+
candidate = dict(row)
|
| 2981 |
+
candidate["payload"] = payload
|
| 2982 |
+
if not self._recoverable_publication_error(
|
| 2983 |
+
row["last_error"],
|
| 2984 |
+
expected_intent_sha256=row["intent_sha256"],
|
| 2985 |
+
):
|
| 2986 |
+
raise GDWConfigurationError(
|
| 2987 |
+
"transient effect recovery found unknown error"
|
| 2988 |
+
)
|
| 2989 |
+
if self._connection_effect_binding_errors(connection, candidate):
|
| 2990 |
+
raise GDWConfigurationError(
|
| 2991 |
+
"transient effect recovery found invalid binding"
|
| 2992 |
+
)
|
| 2993 |
+
original_rows[
|
| 2994 |
+
(row["namespace"], row["owner_id"], row["idempotency_key"])
|
| 2995 |
+
] = dict(row)
|
| 2996 |
+
selection.append(
|
| 2997 |
+
{
|
| 2998 |
+
"namespace": row["namespace"],
|
| 2999 |
+
"owner_id": row["owner_id"],
|
| 3000 |
+
"idempotency_key": row["idempotency_key"],
|
| 3001 |
+
"database_generation_id": row[
|
| 3002 |
+
"database_generation_id"
|
| 3003 |
+
],
|
| 3004 |
+
"request_id": row["request_id"],
|
| 3005 |
+
"kind": row["kind"],
|
| 3006 |
+
"receipt_hash": row["receipt_hash"],
|
| 3007 |
+
"payload_sha256": row["payload_sha256"],
|
| 3008 |
+
"intent_sha256": row["intent_sha256"],
|
| 3009 |
+
"attempts": int(row["attempts"]),
|
| 3010 |
+
"max_attempts": int(row["max_attempts"]),
|
| 3011 |
+
"next_attempt_at": row["next_attempt_at"],
|
| 3012 |
+
"claim_generation": int(row["claim_generation"]),
|
| 3013 |
+
"last_error_sha256": hashlib.sha256(
|
| 3014 |
+
str(row["last_error"]).encode("utf-8")
|
| 3015 |
+
).hexdigest(),
|
| 3016 |
+
}
|
| 3017 |
+
)
|
| 3018 |
+
|
| 3019 |
+
selection_sha256 = hashlib.sha256(
|
| 3020 |
+
_json_text(selection).encode("utf-8")
|
| 3021 |
+
).hexdigest()
|
| 3022 |
+
attempts_before = sum(item["attempts"] for item in selection)
|
| 3023 |
+
for row, item in zip(rows, selection):
|
| 3024 |
+
updated = connection.execute(
|
| 3025 |
+
"""
|
| 3026 |
+
UPDATE effect_outbox
|
| 3027 |
+
SET next_attempt_at = ?
|
| 3028 |
+
WHERE namespace = ? AND owner_id = ?
|
| 3029 |
+
AND idempotency_key = ?
|
| 3030 |
+
AND database_generation_id = ?
|
| 3031 |
+
AND request_id = ? AND kind = ?
|
| 3032 |
+
AND receipt_hash IS ?
|
| 3033 |
+
AND payload_json = ? AND payload_sha256 = ?
|
| 3034 |
+
AND intent_sha256 = ?
|
| 3035 |
+
AND status = 'PENDING'
|
| 3036 |
+
AND attempts = ? AND max_attempts = ?
|
| 3037 |
+
AND next_attempt_at = ? AND claim_generation = ?
|
| 3038 |
+
AND last_error = ?
|
| 3039 |
+
AND lease_owner IS NULL AND lease_until IS NULL
|
| 3040 |
+
AND artifact_json IS NULL AND exported_at IS NULL
|
| 3041 |
+
AND tombstoned_at IS NULL
|
| 3042 |
+
""",
|
| 3043 |
+
(
|
| 3044 |
+
now_text,
|
| 3045 |
+
item["namespace"],
|
| 3046 |
+
item["owner_id"],
|
| 3047 |
+
item["idempotency_key"],
|
| 3048 |
+
item["database_generation_id"],
|
| 3049 |
+
item["request_id"],
|
| 3050 |
+
item["kind"],
|
| 3051 |
+
item["receipt_hash"],
|
| 3052 |
+
row["payload_json"],
|
| 3053 |
+
item["payload_sha256"],
|
| 3054 |
+
item["intent_sha256"],
|
| 3055 |
+
item["attempts"],
|
| 3056 |
+
item["max_attempts"],
|
| 3057 |
+
item["next_attempt_at"],
|
| 3058 |
+
item["claim_generation"],
|
| 3059 |
+
row["last_error"],
|
| 3060 |
+
),
|
| 3061 |
+
)
|
| 3062 |
+
if updated.rowcount != 1:
|
| 3063 |
+
raise GDWConfigurationError(
|
| 3064 |
+
"transient effect changed during recovery"
|
| 3065 |
+
)
|
| 3066 |
+
|
| 3067 |
+
after = self.integrity(global_scope=True, connection=connection)
|
| 3068 |
+
attempts_after = 0
|
| 3069 |
+
for item in selection:
|
| 3070 |
+
persisted = connection.execute(
|
| 3071 |
+
"""
|
| 3072 |
+
SELECT * FROM effect_outbox
|
| 3073 |
+
WHERE namespace = ? AND owner_id = ? AND idempotency_key = ?
|
| 3074 |
+
""",
|
| 3075 |
+
(
|
| 3076 |
+
item["namespace"],
|
| 3077 |
+
item["owner_id"],
|
| 3078 |
+
item["idempotency_key"],
|
| 3079 |
+
),
|
| 3080 |
+
).fetchone()
|
| 3081 |
+
if persisted is None:
|
| 3082 |
+
raise GDWConfigurationError(
|
| 3083 |
+
"transient effect disappeared during recovery"
|
| 3084 |
+
)
|
| 3085 |
+
original = original_rows[
|
| 3086 |
+
(
|
| 3087 |
+
item["namespace"],
|
| 3088 |
+
item["owner_id"],
|
| 3089 |
+
item["idempotency_key"],
|
| 3090 |
+
)
|
| 3091 |
+
]
|
| 3092 |
+
changed = {
|
| 3093 |
+
field
|
| 3094 |
+
for field in persisted.keys()
|
| 3095 |
+
if field != "next_attempt_at"
|
| 3096 |
+
and persisted[field] != original[field]
|
| 3097 |
+
}
|
| 3098 |
+
if (
|
| 3099 |
+
changed
|
| 3100 |
+
or persisted["next_attempt_at"] != now_text
|
| 3101 |
+
or persisted["status"] != "PENDING"
|
| 3102 |
+
):
|
| 3103 |
+
raise GDWConfigurationError(
|
| 3104 |
+
"transient effect accounting changed during recovery"
|
| 3105 |
+
)
|
| 3106 |
+
attempts_after += int(persisted["attempts"])
|
| 3107 |
+
if (
|
| 3108 |
+
after.get("ok") is not True
|
| 3109 |
+
or after.get("sqlite_integrity") != "ok"
|
| 3110 |
+
or after.get("pending_effects") != before.get("pending_effects")
|
| 3111 |
+
or after.get("claimed_effects") != 0
|
| 3112 |
+
or after.get("dead_letter_effects") != 0
|
| 3113 |
+
or after.get("invalid_effect_bindings") != 0
|
| 3114 |
+
or after.get("invalid_exported_artifacts") != 0
|
| 3115 |
+
or after.get("invalid_recovery_audits") != 0
|
| 3116 |
+
or attempts_after != attempts_before
|
| 3117 |
+
):
|
| 3118 |
+
raise GDWConfigurationError(
|
| 3119 |
+
"transient effect recovery changed protected accounting"
|
| 3120 |
+
)
|
| 3121 |
+
|
| 3122 |
+
return record(
|
| 3123 |
+
{
|
| 3124 |
+
"schema": "szl.gdw.transient-effect-recovery/v2",
|
| 3125 |
+
"status": (
|
| 3126 |
+
"RESCHEDULED" if rows else "NO_ELIGIBLE_EFFECTS"
|
| 3127 |
+
),
|
| 3128 |
+
"recovery_id": canonical_recovery_id,
|
| 3129 |
+
"source_revision": source_revision,
|
| 3130 |
+
"requested_limit": limit,
|
| 3131 |
+
"failure_class": "hf-hard-link-enotsup/v1",
|
| 3132 |
+
"database_generation_id": generation_id,
|
| 3133 |
+
"inspected_pending_effects": before["pending_effects"],
|
| 3134 |
+
"eligible_effects": eligible_total,
|
| 3135 |
+
"rescheduled_effects": len(rows),
|
| 3136 |
+
"attempts_before": attempts_before,
|
| 3137 |
+
"attempts_after": attempts_after,
|
| 3138 |
+
"selection": selection,
|
| 3139 |
+
"selection_sha256": selection_sha256,
|
| 3140 |
+
"sqlite_integrity": after["sqlite_integrity"],
|
| 3141 |
+
"claimed_effects": after["claimed_effects"],
|
| 3142 |
+
"dead_letter_effects": after["dead_letter_effects"],
|
| 3143 |
+
"invalid_effect_bindings": after[
|
| 3144 |
+
"invalid_effect_bindings"
|
| 3145 |
+
],
|
| 3146 |
+
"invalid_exported_artifacts": after[
|
| 3147 |
+
"invalid_exported_artifacts"
|
| 3148 |
+
],
|
| 3149 |
+
"invalid_recovery_audits": after[
|
| 3150 |
+
"invalid_recovery_audits"
|
| 3151 |
+
],
|
| 3152 |
+
"credential_values_recorded": False,
|
| 3153 |
+
}
|
| 3154 |
+
)
|
| 3155 |
|
| 3156 |
def assert_effect_claim(
|
| 3157 |
self,
|
|
|
|
| 3493 |
UNION SELECT namespace, owner_id FROM receipts
|
| 3494 |
UNION SELECT namespace, owner_id FROM proof_outbox
|
| 3495 |
UNION SELECT namespace, owner_id FROM effect_outbox
|
| 3496 |
+
UNION SELECT namespace, owner_id FROM effect_recovery_audit
|
| 3497 |
ORDER BY namespace, owner_id
|
| 3498 |
"""
|
| 3499 |
).fetchall()
|
|
|
|
| 3561 |
LENGTH(CAST(payload_json AS BLOB)) +
|
| 3562 |
COALESCE(LENGTH(CAST(artifact_json AS BLOB)), 0))
|
| 3563 |
FROM effect_outbox
|
| 3564 |
+
WHERE namespace = ? AND owner_id = ?), 0) +
|
| 3565 |
+
COALESCE((SELECT SUM(LENGTH(CAST(report_json AS BLOB)))
|
| 3566 |
+
FROM effect_recovery_audit
|
| 3567 |
WHERE namespace = ? AND owner_id = ?), 0)
|
| 3568 |
""",
|
| 3569 |
(
|
|
|
|
| 3577 |
owner_id,
|
| 3578 |
namespace,
|
| 3579 |
owner_id,
|
| 3580 |
+
namespace,
|
| 3581 |
+
owner_id,
|
| 3582 |
),
|
| 3583 |
).fetchone()[0]
|
| 3584 |
)
|
|
|
|
| 3615 |
UNION SELECT namespace, owner_id FROM receipts
|
| 3616 |
UNION SELECT namespace, owner_id FROM proof_outbox
|
| 3617 |
UNION SELECT namespace, owner_id FROM effect_outbox
|
| 3618 |
+
UNION SELECT namespace, owner_id FROM effect_recovery_audit
|
| 3619 |
"""
|
| 3620 |
).fetchall()
|
| 3621 |
connection.execute("DELETE FROM usage")
|
|
|
|
| 3852 |
)
|
| 3853 |
for table in _V1_TABLES
|
| 3854 |
}
|
| 3855 |
+
counts["effect_recovery_audit"] = int(
|
| 3856 |
+
connection.execute(
|
| 3857 |
+
"SELECT COUNT(*) FROM effect_recovery_audit" + predicate,
|
| 3858 |
+
params,
|
| 3859 |
+
).fetchone()[0]
|
| 3860 |
+
)
|
| 3861 |
effect_predicate = (
|
| 3862 |
"status IN ('PENDING', 'CLAIMED')"
|
| 3863 |
if global_scope
|
|
|
|
| 3926 |
"invalid_request_digests": 0,
|
| 3927 |
"invalid_receipt_digests": 0,
|
| 3928 |
"invalid_proof_digests": 0,
|
| 3929 |
+
"invalid_recovery_audits": 0,
|
| 3930 |
}
|
| 3931 |
scoped_suffix = (
|
| 3932 |
""
|
|
|
|
| 4060 |
json.JSONDecodeError,
|
| 4061 |
):
|
| 4062 |
digest_violations["invalid_proof_digests"] += 1
|
| 4063 |
+
digest_violations["invalid_recovery_audits"] += (
|
| 4064 |
+
self._recovery_audit_chain_errors(
|
| 4065 |
+
connection,
|
| 4066 |
+
namespace=None if global_scope else ns,
|
| 4067 |
+
owner_id=None if global_scope else owner,
|
| 4068 |
+
)
|
| 4069 |
+
)
|
| 4070 |
effect_rows = connection.execute(
|
| 4071 |
"""
|
| 4072 |
SELECT namespace, owner_id, idempotency_key,
|
routers/gdw_frontier.py
CHANGED
|
@@ -12,7 +12,7 @@ from typing import List, Literal, Optional
|
|
| 12 |
from urllib.request import Request as UrlRequest
|
| 13 |
from urllib.request import urlopen
|
| 14 |
|
| 15 |
-
from fastapi import Header, HTTPException, Request
|
| 16 |
from fastapi.responses import PlainTextResponse
|
| 17 |
from pydantic import BaseModel, Field, ValidationError
|
| 18 |
|
|
@@ -352,6 +352,59 @@ def _require_write_ready(namespace: str) -> None:
|
|
| 352 |
)
|
| 353 |
|
| 354 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
def _public_runtime_health(runtime: dict) -> dict:
|
| 356 |
storage = runtime.get("storage")
|
| 357 |
public_storage = None
|
|
@@ -567,6 +620,70 @@ def _canonical_policy_evaluate(action: dict) -> dict:
|
|
| 567 |
return result
|
| 568 |
|
| 569 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 570 |
def _risk_severity(risk_budget: float) -> str:
|
| 571 |
if risk_budget < 0.25:
|
| 572 |
return "low"
|
|
@@ -894,6 +1011,87 @@ def register(app, ns: str = "a11oy"):
|
|
| 894 |
"database_generation_id": integrity["database_generation_id"],
|
| 895 |
}
|
| 896 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 897 |
@app.get(prefix + "/integrity/global")
|
| 898 |
@app.get("/v1/gdw/integrity/global")
|
| 899 |
def gdw_global_integrity(
|
|
@@ -1344,6 +1542,7 @@ def register(app, ns: str = "a11oy"):
|
|
| 1344 |
prefix + "/integrity",
|
| 1345 |
prefix + "/integrity/global",
|
| 1346 |
prefix + "/drain",
|
|
|
|
| 1347 |
prefix + "/sessions/{session_id}",
|
| 1348 |
prefix + "/step",
|
| 1349 |
],
|
|
|
|
| 12 |
from urllib.request import Request as UrlRequest
|
| 13 |
from urllib.request import urlopen
|
| 14 |
|
| 15 |
+
from fastapi import Header, HTTPException, Query, Request
|
| 16 |
from fastapi.responses import PlainTextResponse
|
| 17 |
from pydantic import BaseModel, Field, ValidationError
|
| 18 |
|
|
|
|
| 352 |
)
|
| 353 |
|
| 354 |
|
| 355 |
+
def _require_transient_recovery_runtime(
|
| 356 |
+
namespace: str,
|
| 357 |
+
expected_source_revision: Optional[str],
|
| 358 |
+
) -> str:
|
| 359 |
+
expected = str(expected_source_revision or "").strip().lower()
|
| 360 |
+
observed_source = os.environ.get("SZL_GIT_SHA", "").strip().lower()
|
| 361 |
+
if (
|
| 362 |
+
re.fullmatch(r"[0-9a-f]{40}", expected) is None
|
| 363 |
+
or observed_source != expected
|
| 364 |
+
):
|
| 365 |
+
raise HTTPException(
|
| 366 |
+
status_code=409,
|
| 367 |
+
detail="GDW recovery source revision mismatch",
|
| 368 |
+
)
|
| 369 |
+
runtime = runtime_health()
|
| 370 |
+
storage = runtime.get("storage") or {}
|
| 371 |
+
drain = runtime.get("drain") or {}
|
| 372 |
+
database_generation_id = str(
|
| 373 |
+
storage.get("database_generation_id") or ""
|
| 374 |
+
)
|
| 375 |
+
runtime_ready = (
|
| 376 |
+
runtime.get("startup_state") == "READY"
|
| 377 |
+
and runtime.get("evidence_label") == "VERIFIED"
|
| 378 |
+
and storage.get("persistence_required") is True
|
| 379 |
+
and storage.get("mount_verified") is True
|
| 380 |
+
and storage.get("journal_mode_requested") == "DELETE"
|
| 381 |
+
and storage.get("journal_mode_observed") == "DELETE"
|
| 382 |
+
and storage.get("synchronous_requested") == "FULL"
|
| 383 |
+
and storage.get("synchronous_observed") == 2
|
| 384 |
+
and storage.get("sqlite_integrity") == "ok"
|
| 385 |
+
and storage.get("proof_export_mode") == "outbox"
|
| 386 |
+
and storage.get("schema_version") == GDWWorkspace.schema_version()
|
| 387 |
+
and re.fullmatch(
|
| 388 |
+
r"[0-9a-f]{32}",
|
| 389 |
+
database_generation_id,
|
| 390 |
+
)
|
| 391 |
+
is not None
|
| 392 |
+
and drain.get("enabled") is True
|
| 393 |
+
and drain.get("running") is True
|
| 394 |
+
and _governance_ready()
|
| 395 |
+
)
|
| 396 |
+
try:
|
| 397 |
+
policy_ready = _canonical_policy_ready()
|
| 398 |
+
except Exception:
|
| 399 |
+
policy_ready = False
|
| 400 |
+
if not runtime_ready or not policy_ready:
|
| 401 |
+
raise HTTPException(
|
| 402 |
+
status_code=503,
|
| 403 |
+
detail="GDW recovery runtime contract is unavailable",
|
| 404 |
+
)
|
| 405 |
+
return database_generation_id
|
| 406 |
+
|
| 407 |
+
|
| 408 |
def _public_runtime_health(runtime: dict) -> dict:
|
| 409 |
storage = runtime.get("storage")
|
| 410 |
public_storage = None
|
|
|
|
| 620 |
return result
|
| 621 |
|
| 622 |
|
| 623 |
+
def _transient_recovery_governance(
|
| 624 |
+
*,
|
| 625 |
+
principal: Principal,
|
| 626 |
+
recovery_id: str,
|
| 627 |
+
source_revision: str,
|
| 628 |
+
database_generation_id: str,
|
| 629 |
+
limit: int,
|
| 630 |
+
) -> dict:
|
| 631 |
+
"""Obtain an explicit signed canonical-policy allow for one recovery call."""
|
| 632 |
+
|
| 633 |
+
binding = {
|
| 634 |
+
"schema": "szl.gdw.transient-effect-recovery-authorization/v1",
|
| 635 |
+
"action_type": "gdw.transient-effect-recovery",
|
| 636 |
+
"namespace": principal.namespace,
|
| 637 |
+
"owner_id": principal.owner_id,
|
| 638 |
+
"credential_key_id": principal.key_id,
|
| 639 |
+
"recovery_id": recovery_id,
|
| 640 |
+
"source_revision": source_revision,
|
| 641 |
+
"database_generation_id": database_generation_id,
|
| 642 |
+
"limit": limit,
|
| 643 |
+
"failure_class": "hf-hard-link-enotsup/v1",
|
| 644 |
+
}
|
| 645 |
+
binding_sha256 = _sha(binding)
|
| 646 |
+
action = {
|
| 647 |
+
"actionId": f"gdw-recovery:{binding_sha256}",
|
| 648 |
+
"severity": "high",
|
| 649 |
+
"decisionClass": "ordinary",
|
| 650 |
+
"confidence": 1.0,
|
| 651 |
+
"witnesses": [
|
| 652 |
+
{
|
| 653 |
+
"id": (
|
| 654 |
+
f"principal:{principal.namespace}:"
|
| 655 |
+
f"{principal.owner_id}:{principal.key_id}"
|
| 656 |
+
),
|
| 657 |
+
"role": "operator",
|
| 658 |
+
"attested": True,
|
| 659 |
+
},
|
| 660 |
+
{
|
| 661 |
+
"id": f"workload:szl-holdings/a11oy@{source_revision}",
|
| 662 |
+
"role": "workload",
|
| 663 |
+
"attested": True,
|
| 664 |
+
},
|
| 665 |
+
],
|
| 666 |
+
}
|
| 667 |
+
result = _canonical_policy_evaluate(action)
|
| 668 |
+
if result.get("decision") != "allow":
|
| 669 |
+
raise PermissionError("canonical policy denied transient recovery")
|
| 670 |
+
return {
|
| 671 |
+
"schema": "szl.gdw.transient-effect-recovery-governance/v1",
|
| 672 |
+
"decision": "ALLOW",
|
| 673 |
+
"binding": binding,
|
| 674 |
+
"binding_sha256": binding_sha256,
|
| 675 |
+
"policy_gateway": {
|
| 676 |
+
"decision": "ALLOW",
|
| 677 |
+
"gate": result["gate"],
|
| 678 |
+
"receipt_hash": result["receipt_hash"],
|
| 679 |
+
"receipt_signed": True,
|
| 680 |
+
"receipts_in_eq_out": True,
|
| 681 |
+
"action_id": action["actionId"],
|
| 682 |
+
"witnesses": action["witnesses"],
|
| 683 |
+
},
|
| 684 |
+
}
|
| 685 |
+
|
| 686 |
+
|
| 687 |
def _risk_severity(risk_budget: float) -> str:
|
| 688 |
if risk_budget < 0.25:
|
| 689 |
return "low"
|
|
|
|
| 1011 |
"database_generation_id": integrity["database_generation_id"],
|
| 1012 |
}
|
| 1013 |
|
| 1014 |
+
@app.post(prefix + "/recovery/transient-effects")
|
| 1015 |
+
@app.post("/v1/gdw/recovery/transient-effects")
|
| 1016 |
+
def gdw_recover_transient_effects(
|
| 1017 |
+
limit: int = Query(default=100, ge=1, le=1_000),
|
| 1018 |
+
authorization: Optional[str] = Header(default=None, alias="Authorization"),
|
| 1019 |
+
expected_source_revision: Optional[str] = Header(
|
| 1020 |
+
default=None,
|
| 1021 |
+
alias="X-Expected-Source-Revision",
|
| 1022 |
+
),
|
| 1023 |
+
idempotency_key: Optional[str] = Header(
|
| 1024 |
+
default=None,
|
| 1025 |
+
alias="Idempotency-Key",
|
| 1026 |
+
),
|
| 1027 |
+
):
|
| 1028 |
+
principal = _authorise(
|
| 1029 |
+
authorization,
|
| 1030 |
+
namespace=ns,
|
| 1031 |
+
required_scopes=("effects:recover", "integrity:global"),
|
| 1032 |
+
)
|
| 1033 |
+
runtime_generation = _require_transient_recovery_runtime(
|
| 1034 |
+
ns,
|
| 1035 |
+
expected_source_revision,
|
| 1036 |
+
)
|
| 1037 |
+
if not idempotency_key or not _ID_PATTERN.fullmatch(idempotency_key):
|
| 1038 |
+
raise HTTPException(
|
| 1039 |
+
status_code=422,
|
| 1040 |
+
detail=(
|
| 1041 |
+
"Idempotency-Key must be 1-128 canonical identifier "
|
| 1042 |
+
"characters"
|
| 1043 |
+
),
|
| 1044 |
+
)
|
| 1045 |
+
workspace = _workspace(principal)
|
| 1046 |
+
if workspace.database_generation_id != runtime_generation:
|
| 1047 |
+
raise HTTPException(
|
| 1048 |
+
status_code=503,
|
| 1049 |
+
detail="GDW recovery database generation changed",
|
| 1050 |
+
)
|
| 1051 |
+
try:
|
| 1052 |
+
governance = _transient_recovery_governance(
|
| 1053 |
+
principal=principal,
|
| 1054 |
+
recovery_id=idempotency_key,
|
| 1055 |
+
source_revision=str(expected_source_revision),
|
| 1056 |
+
database_generation_id=runtime_generation,
|
| 1057 |
+
limit=limit,
|
| 1058 |
+
)
|
| 1059 |
+
except PermissionError as exc:
|
| 1060 |
+
raise HTTPException(
|
| 1061 |
+
status_code=403,
|
| 1062 |
+
detail="GDW recovery denied by canonical policy",
|
| 1063 |
+
) from exc
|
| 1064 |
+
except Exception as exc:
|
| 1065 |
+
raise HTTPException(
|
| 1066 |
+
status_code=503,
|
| 1067 |
+
detail="GDW recovery canonical policy unavailable",
|
| 1068 |
+
) from exc
|
| 1069 |
+
try:
|
| 1070 |
+
report = workspace.recover_retry_scheduled_effects(
|
| 1071 |
+
recovery_id=idempotency_key,
|
| 1072 |
+
credential_key_id=principal.key_id,
|
| 1073 |
+
expected_source_revision=str(expected_source_revision),
|
| 1074 |
+
expected_database_generation_id=runtime_generation,
|
| 1075 |
+
governance=governance,
|
| 1076 |
+
limit=limit,
|
| 1077 |
+
)
|
| 1078 |
+
except GDWConfigurationError as exc:
|
| 1079 |
+
if str(exc) == "recovery_id was already used with different content":
|
| 1080 |
+
raise HTTPException(
|
| 1081 |
+
status_code=409,
|
| 1082 |
+
detail="Idempotency-Key was already used with different content",
|
| 1083 |
+
) from exc
|
| 1084 |
+
raise HTTPException(
|
| 1085 |
+
status_code=503,
|
| 1086 |
+
detail="GDW recovery refused by integrity gate",
|
| 1087 |
+
) from exc
|
| 1088 |
+
if report.get("database_generation_id") != runtime_generation:
|
| 1089 |
+
raise HTTPException(
|
| 1090 |
+
status_code=503,
|
| 1091 |
+
detail="GDW recovery database generation changed",
|
| 1092 |
+
)
|
| 1093 |
+
return report
|
| 1094 |
+
|
| 1095 |
@app.get(prefix + "/integrity/global")
|
| 1096 |
@app.get("/v1/gdw/integrity/global")
|
| 1097 |
def gdw_global_integrity(
|
|
|
|
| 1542 |
prefix + "/integrity",
|
| 1543 |
prefix + "/integrity/global",
|
| 1544 |
prefix + "/drain",
|
| 1545 |
+
prefix + "/recovery/transient-effects",
|
| 1546 |
prefix + "/sessions/{session_id}",
|
| 1547 |
prefix + "/step",
|
| 1548 |
],
|