betterwithage Claude Opus 4.7 commited on
Commit
f5c395e
·
verified ·
1 Parent(s): adec5ab

deploy(hf): sync szl-holdings/a11oy@900209d5b4a435f836fd6229b13e65c9f4e8c23b derived COPY set

Browse files

Reusable Dockerfile-COPY-derived deploy from szl-holdings/a11oy 900209d5b4a435f836fd6229b13e65c9f4e8c23b.
Files: 1182 Pruned: 0
Derived from Dockerfile COPY sources (NO hand-maintained allowlist).

Signed-off-by: SZL Holdings <noreply@szlholdings.ai>
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Files changed (2) hide show
  1. gdw_workspace.py +1123 -2
  2. routers/gdw_frontier.py +200 -1
gdw_workspace.py CHANGED
@@ -1,5 +1,6 @@
1
  """Tenant-safe SQLite state, idempotency, quota, and outbox storage for GDW."""
2
 
 
3
  import contextlib
4
  import hashlib
5
  import json
@@ -13,8 +14,10 @@ from datetime import datetime, timedelta, timezone
13
  from pathlib import Path
14
  from typing import Any, Dict, Iterator, Optional, Tuple
15
 
 
16
 
17
- SCHEMA_VERSION = 3
 
18
  _SCHEMA_LOCK = threading.RLock()
19
  _PROCESS_WRITE_LOCK = threading.RLock()
20
  _IDENTIFIER = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
@@ -278,6 +281,28 @@ _SCHEMA_STATEMENTS = (
278
  )
279
  """,
280
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281
  CREATE INDEX idx_sessions_lifecycle
282
  ON session_state(namespace, owner_id, lifecycle, expires_at)
283
  """,
@@ -500,6 +525,7 @@ class GDWWorkspace:
500
  "ALTER TABLE effect_outbox RENAME TO effect_outbox_v2"
501
  )
502
  connection.execute(_SCHEMA_STATEMENTS[6])
 
503
  connection.execute(
504
  "ALTER TABLE schema_meta "
505
  "ADD COLUMN database_generation_id TEXT"
@@ -794,6 +820,32 @@ class GDWWorkspace:
794
  finally:
795
  connection.execute("PRAGMA foreign_keys=ON")
796
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
797
  @staticmethod
798
  def _validate_schema(connection: sqlite3.Connection) -> str:
799
  row = connection.execute(
@@ -816,6 +868,7 @@ class GDWWorkspace:
816
  "receipts",
817
  "proof_outbox",
818
  "effect_outbox",
 
819
  }
820
  missing = required - GDWWorkspace._table_names(connection)
821
  if missing:
@@ -834,6 +887,24 @@ class GDWWorkspace:
834
  "intent_sha256",
835
  "claim_generation",
836
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
837
  }
838
  for table, expected in expected_columns.items():
839
  columns = {
@@ -895,6 +966,9 @@ class GDWWorkspace:
895
  ).fetchone()
896
  if version is not None and int(version[0]) == 2:
897
  self._migrate_v2(connection)
 
 
 
898
  self.database_generation_id = self._validate_schema(connection)
899
  finally:
900
  connection.close()
@@ -2051,7 +2125,1033 @@ class GDWWorkspace:
2051
  ),
2052
  )
2053
  requeued += int(updated.rowcount)
2054
- return requeued
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2055
 
2056
  def assert_effect_claim(
2057
  self,
@@ -2393,6 +3493,7 @@ class GDWWorkspace:
2393
  UNION SELECT namespace, owner_id FROM receipts
2394
  UNION SELECT namespace, owner_id FROM proof_outbox
2395
  UNION SELECT namespace, owner_id FROM effect_outbox
 
2396
  ORDER BY namespace, owner_id
2397
  """
2398
  ).fetchall()
@@ -2460,6 +3561,9 @@ class GDWWorkspace:
2460
  LENGTH(CAST(payload_json AS BLOB)) +
2461
  COALESCE(LENGTH(CAST(artifact_json AS BLOB)), 0))
2462
  FROM effect_outbox
 
 
 
2463
  WHERE namespace = ? AND owner_id = ?), 0)
2464
  """,
2465
  (
@@ -2473,6 +3577,8 @@ class GDWWorkspace:
2473
  owner_id,
2474
  namespace,
2475
  owner_id,
 
 
2476
  ),
2477
  ).fetchone()[0]
2478
  )
@@ -2509,6 +3615,7 @@ class GDWWorkspace:
2509
  UNION SELECT namespace, owner_id FROM receipts
2510
  UNION SELECT namespace, owner_id FROM proof_outbox
2511
  UNION SELECT namespace, owner_id FROM effect_outbox
 
2512
  """
2513
  ).fetchall()
2514
  connection.execute("DELETE FROM usage")
@@ -2745,6 +3852,12 @@ class GDWWorkspace:
2745
  )
2746
  for table in _V1_TABLES
2747
  }
 
 
 
 
 
 
2748
  effect_predicate = (
2749
  "status IN ('PENDING', 'CLAIMED')"
2750
  if global_scope
@@ -2813,6 +3926,7 @@ class GDWWorkspace:
2813
  "invalid_request_digests": 0,
2814
  "invalid_receipt_digests": 0,
2815
  "invalid_proof_digests": 0,
 
2816
  }
2817
  scoped_suffix = (
2818
  ""
@@ -2946,6 +4060,13 @@ class GDWWorkspace:
2946
  json.JSONDecodeError,
2947
  ):
2948
  digest_violations["invalid_proof_digests"] += 1
 
 
 
 
 
 
 
2949
  effect_rows = connection.execute(
2950
  """
2951
  SELECT namespace, owner_id, idempotency_key,
 
1
  """Tenant-safe SQLite state, idempotency, quota, and outbox storage for GDW."""
2
 
3
+ import base64
4
  import contextlib
5
  import hashlib
6
  import json
 
14
  from pathlib import Path
15
  from typing import Any, Dict, Iterator, Optional, Tuple
16
 
17
+ import szl_dsse
18
 
19
+
20
+ SCHEMA_VERSION = 4
21
  _SCHEMA_LOCK = threading.RLock()
22
  _PROCESS_WRITE_LOCK = threading.RLock()
23
  _IDENTIFIER = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
 
281
  )
282
  """,
283
  """
284
+ CREATE TABLE effect_recovery_audit (
285
+ namespace TEXT NOT NULL,
286
+ owner_id TEXT NOT NULL,
287
+ recovery_id TEXT NOT NULL,
288
+ sequence INTEGER NOT NULL CHECK(sequence >= 0),
289
+ credential_key_id TEXT NOT NULL,
290
+ database_generation_id TEXT NOT NULL,
291
+ request_sha256 TEXT NOT NULL,
292
+ outcome_sha256 TEXT NOT NULL,
293
+ governance_sha256 TEXT NOT NULL,
294
+ receipt_sha256 TEXT NOT NULL,
295
+ previous_receipt_sha256 TEXT NOT NULL,
296
+ previous_chain_sha256 TEXT NOT NULL,
297
+ chain_sha256 TEXT NOT NULL,
298
+ dsse_envelope_sha256 TEXT NOT NULL,
299
+ report_json TEXT NOT NULL,
300
+ created_at TEXT NOT NULL,
301
+ PRIMARY KEY(namespace, owner_id, recovery_id),
302
+ UNIQUE(namespace, owner_id, sequence)
303
+ )
304
+ """,
305
+ """
306
  CREATE INDEX idx_sessions_lifecycle
307
  ON session_state(namespace, owner_id, lifecycle, expires_at)
308
  """,
 
525
  "ALTER TABLE effect_outbox RENAME TO effect_outbox_v2"
526
  )
527
  connection.execute(_SCHEMA_STATEMENTS[6])
528
+ connection.execute(_SCHEMA_STATEMENTS[7])
529
  connection.execute(
530
  "ALTER TABLE schema_meta "
531
  "ADD COLUMN database_generation_id TEXT"
 
820
  finally:
821
  connection.execute("PRAGMA foreign_keys=ON")
822
 
823
+ @staticmethod
824
+ def _migrate_v3(connection: sqlite3.Connection) -> None:
825
+ """Add atomically bound recovery audit records without rebinding data."""
826
+
827
+ timestamp = _text_time()
828
+ connection.execute("BEGIN IMMEDIATE")
829
+ try:
830
+ version = connection.execute(
831
+ "SELECT schema_version FROM schema_meta WHERE schema_name = 'gdw'"
832
+ ).fetchone()
833
+ if version is None or int(version[0]) != 3:
834
+ raise GDWSchemaError("GDW v3 migration source changed under lock")
835
+ connection.execute(_SCHEMA_STATEMENTS[7])
836
+ connection.execute(
837
+ """
838
+ UPDATE schema_meta
839
+ SET schema_version = ?, upgraded_at = ?
840
+ WHERE schema_name = 'gdw'
841
+ """,
842
+ (SCHEMA_VERSION, timestamp),
843
+ )
844
+ connection.execute("COMMIT")
845
+ except Exception:
846
+ connection.execute("ROLLBACK")
847
+ raise
848
+
849
  @staticmethod
850
  def _validate_schema(connection: sqlite3.Connection) -> str:
851
  row = connection.execute(
 
868
  "receipts",
869
  "proof_outbox",
870
  "effect_outbox",
871
+ "effect_recovery_audit",
872
  }
873
  missing = required - GDWWorkspace._table_names(connection)
874
  if missing:
 
887
  "intent_sha256",
888
  "claim_generation",
889
  },
890
+ "effect_recovery_audit": {
891
+ "namespace",
892
+ "owner_id",
893
+ "recovery_id",
894
+ "sequence",
895
+ "credential_key_id",
896
+ "database_generation_id",
897
+ "request_sha256",
898
+ "outcome_sha256",
899
+ "governance_sha256",
900
+ "receipt_sha256",
901
+ "previous_receipt_sha256",
902
+ "previous_chain_sha256",
903
+ "chain_sha256",
904
+ "dsse_envelope_sha256",
905
+ "report_json",
906
+ "created_at",
907
+ },
908
  }
909
  for table, expected in expected_columns.items():
910
  columns = {
 
966
  ).fetchone()
967
  if version is not None and int(version[0]) == 2:
968
  self._migrate_v2(connection)
969
+ version = (SCHEMA_VERSION,)
970
+ if version is not None and int(version[0]) == 3:
971
+ self._migrate_v3(connection)
972
  self.database_generation_id = self._validate_schema(connection)
973
  finally:
974
  connection.close()
 
2125
  ),
2126
  )
2127
  requeued += int(updated.rowcount)
2128
+ return requeued
2129
+
2130
+ @staticmethod
2131
+ def _recoverable_publication_error(
2132
+ value: Any,
2133
+ *,
2134
+ expected_intent_sha256: str,
2135
+ ) -> bool:
2136
+ match = re.fullmatch(
2137
+ r"OSError: \[Errno 95\] Operation not supported: "
2138
+ r"'(?P<stage>[^']*[\\/]\.gdw-artifact-[^'\\/]+\.tmp)' -> "
2139
+ r"'(?P<final>[^']*[\\/](?P<digest>[0-9a-f]{64})\.json)'",
2140
+ str(value or ""),
2141
+ )
2142
+ if match is None or match.group("digest") != expected_intent_sha256:
2143
+ return False
2144
+ stage = match.group("stage").replace("\\", "/")
2145
+ final = match.group("final").replace("\\", "/")
2146
+ return stage.rsplit("/", 1)[0] == final.rsplit("/", 1)[0]
2147
+
2148
+ @staticmethod
2149
+ def _validated_recovery_governance(
2150
+ governance: Any,
2151
+ *,
2152
+ namespace: str,
2153
+ owner_id: str,
2154
+ credential_key_id: str,
2155
+ recovery_id: str,
2156
+ source_revision: str,
2157
+ database_generation_id: str,
2158
+ limit: int,
2159
+ ) -> Tuple[Dict[str, Any], str, str]:
2160
+ try:
2161
+ canonical = json.loads(_json_text(governance))
2162
+ if type(canonical) is not dict or set(canonical) != {
2163
+ "schema",
2164
+ "decision",
2165
+ "binding",
2166
+ "binding_sha256",
2167
+ "policy_gateway",
2168
+ }:
2169
+ raise ValueError("recovery governance shape mismatch")
2170
+ expected_binding = {
2171
+ "schema": "szl.gdw.transient-effect-recovery-authorization/v1",
2172
+ "action_type": "gdw.transient-effect-recovery",
2173
+ "namespace": namespace,
2174
+ "owner_id": owner_id,
2175
+ "credential_key_id": credential_key_id,
2176
+ "recovery_id": recovery_id,
2177
+ "source_revision": source_revision,
2178
+ "database_generation_id": database_generation_id,
2179
+ "limit": limit,
2180
+ "failure_class": "hf-hard-link-enotsup/v1",
2181
+ }
2182
+ binding_sha256 = hashlib.sha256(
2183
+ _json_text(expected_binding).encode("utf-8")
2184
+ ).hexdigest()
2185
+ expected_witnesses = [
2186
+ {
2187
+ "id": f"principal:{namespace}:{owner_id}:{credential_key_id}",
2188
+ "role": "operator",
2189
+ "attested": True,
2190
+ },
2191
+ {
2192
+ "id": f"workload:szl-holdings/a11oy@{source_revision}",
2193
+ "role": "workload",
2194
+ "attested": True,
2195
+ },
2196
+ ]
2197
+ gateway = canonical["policy_gateway"]
2198
+ if (
2199
+ canonical["schema"]
2200
+ != "szl.gdw.transient-effect-recovery-governance/v1"
2201
+ or canonical["decision"] != "ALLOW"
2202
+ or canonical["binding"] != expected_binding
2203
+ or canonical["binding_sha256"] != binding_sha256
2204
+ or type(gateway) is not dict
2205
+ or set(gateway) != {
2206
+ "decision",
2207
+ "gate",
2208
+ "receipt_hash",
2209
+ "receipt_signed",
2210
+ "receipts_in_eq_out",
2211
+ "action_id",
2212
+ "witnesses",
2213
+ }
2214
+ or gateway["decision"] != "ALLOW"
2215
+ or gateway["gate"] != "ThresholdPolicySeverity"
2216
+ or re.fullmatch(
2217
+ r"[0-9a-f]{64}", str(gateway["receipt_hash"] or "")
2218
+ )
2219
+ is None
2220
+ or gateway["receipt_signed"] is not True
2221
+ or gateway["receipts_in_eq_out"] is not True
2222
+ or gateway["action_id"] != f"gdw-recovery:{binding_sha256}"
2223
+ or gateway["witnesses"] != expected_witnesses
2224
+ ):
2225
+ raise ValueError("recovery governance binding mismatch")
2226
+ governance_sha256 = hashlib.sha256(
2227
+ _json_text(canonical).encode("utf-8")
2228
+ ).hexdigest()
2229
+ return canonical, binding_sha256, governance_sha256
2230
+ except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
2231
+ raise GDWConfigurationError(
2232
+ "transient effect recovery governance is invalid"
2233
+ ) from exc
2234
+
2235
+ @staticmethod
2236
+ def _validated_recovery_audit(row: sqlite3.Row) -> Dict[str, Any]:
2237
+ try:
2238
+ report = json.loads(row["report_json"])
2239
+ if type(report) is not dict:
2240
+ raise ValueError("recovery report must be an object")
2241
+ outcome_fields = {
2242
+ "schema",
2243
+ "status",
2244
+ "recovery_id",
2245
+ "source_revision",
2246
+ "requested_limit",
2247
+ "failure_class",
2248
+ "database_generation_id",
2249
+ "inspected_pending_effects",
2250
+ "eligible_effects",
2251
+ "rescheduled_effects",
2252
+ "attempts_before",
2253
+ "attempts_after",
2254
+ "selection",
2255
+ "selection_sha256",
2256
+ "sqlite_integrity",
2257
+ "claimed_effects",
2258
+ "dead_letter_effects",
2259
+ "invalid_effect_bindings",
2260
+ "invalid_exported_artifacts",
2261
+ "invalid_recovery_audits",
2262
+ "credential_values_recorded",
2263
+ }
2264
+ if set(report) != outcome_fields | {
2265
+ "governance",
2266
+ "audit_receipt",
2267
+ "replayed",
2268
+ }:
2269
+ raise ValueError("recovery report shape mismatch")
2270
+ if (
2271
+ type(report["governance"]) is not dict
2272
+ or type(report["audit_receipt"]) is not dict
2273
+ ):
2274
+ raise ValueError("recovery receipt must be an object")
2275
+ outcome = {field: report[field] for field in outcome_fields}
2276
+ receipt = dict(report["audit_receipt"])
2277
+ receipt_payload_fields = {
2278
+ "schema",
2279
+ "operator",
2280
+ "recovery_id",
2281
+ "source_revision",
2282
+ "database_generation_id",
2283
+ "request_sha256",
2284
+ "outcome_sha256",
2285
+ "governance_sha256",
2286
+ "selection_sha256",
2287
+ "rescheduled_effects",
2288
+ "attempts_before",
2289
+ "attempts_after",
2290
+ "sequence",
2291
+ "previous_receipt_sha256",
2292
+ "previous_chain_sha256",
2293
+ "atomic_with_mutation",
2294
+ "created_at",
2295
+ "credential_values_recorded",
2296
+ }
2297
+ receipt_fields = receipt_payload_fields | {
2298
+ "receipt_status",
2299
+ "receipt_sha256",
2300
+ "dsse_envelope_sha256",
2301
+ "chain_sha256",
2302
+ "dsse_envelope",
2303
+ }
2304
+ if set(receipt) != receipt_fields:
2305
+ raise ValueError("recovery receipt shape mismatch")
2306
+ receipt_payload = {
2307
+ field: receipt[field] for field in receipt_payload_fields
2308
+ }
2309
+ if (
2310
+ type(receipt.get("created_at")) is not str
2311
+ or _text_time(receipt["created_at"]) != receipt["created_at"]
2312
+ ):
2313
+ raise ValueError("recovery receipt timestamp mismatch")
2314
+ recovery_created_at = _normalise_time(receipt["created_at"])
2315
+
2316
+ def is_digest(value: Any, length: int = 64) -> bool:
2317
+ return type(value) is str and re.fullmatch(
2318
+ rf"[0-9a-f]{{{length}}}", value
2319
+ ) is not None
2320
+
2321
+ observed_outcome_sha256 = hashlib.sha256(
2322
+ _json_text(outcome).encode("utf-8")
2323
+ ).hexdigest()
2324
+ observed_receipt_sha256 = hashlib.sha256(
2325
+ _json_text(receipt_payload).encode("utf-8")
2326
+ ).hexdigest()
2327
+ envelope = receipt["dsse_envelope"]
2328
+ if type(envelope) is not dict:
2329
+ raise ValueError("recovery DSSE envelope shape mismatch")
2330
+ observed_envelope_sha256 = hashlib.sha256(
2331
+ _json_text(envelope).encode("utf-8")
2332
+ ).hexdigest()
2333
+ decoded_payload = json.loads(
2334
+ base64.b64decode(
2335
+ str(envelope.get("payload") or ""),
2336
+ validate=True,
2337
+ ).decode("utf-8")
2338
+ )
2339
+ if (
2340
+ envelope.get("payloadType") != szl_dsse.KHIPU_PAYLOAD_TYPE
2341
+ or decoded_payload != receipt_payload
2342
+ ):
2343
+ raise ValueError("recovery DSSE payload binding mismatch")
2344
+ signed = envelope.get("signed") is True
2345
+ if signed:
2346
+ if (
2347
+ receipt["receipt_status"] != "SIGNED_KHIPU_DSSE"
2348
+ or szl_dsse.verify_envelope(envelope).get("verified") is not True
2349
+ ):
2350
+ raise ValueError("recovery DSSE signature is invalid")
2351
+ elif (
2352
+ receipt["receipt_status"] != "UNSIGNED_KHIPU_DSSE"
2353
+ or envelope.get("signed") is not False
2354
+ or envelope.get("signatures") != []
2355
+ or "UNSIGNED" not in str(envelope.get("honesty") or "")
2356
+ ):
2357
+ raise ValueError("recovery unsigned DSSE evidence is dishonest")
2358
+ observed_chain_sha256 = hashlib.sha256(
2359
+ _json_text(
2360
+ {
2361
+ "previous_chain_sha256": receipt["previous_chain_sha256"],
2362
+ "receipt_sha256": observed_receipt_sha256,
2363
+ "receipt_status": receipt["receipt_status"],
2364
+ "dsse_envelope_sha256": observed_envelope_sha256,
2365
+ }
2366
+ ).encode("utf-8")
2367
+ ).hexdigest()
2368
+ expected_operator = {
2369
+ "namespace": row["namespace"],
2370
+ "owner_id": row["owner_id"],
2371
+ "credential_key_id": row["credential_key_id"],
2372
+ }
2373
+ expected_request = {
2374
+ "schema": "szl.gdw.transient-effect-recovery-request/v1",
2375
+ "namespace": row["namespace"],
2376
+ "owner_id": row["owner_id"],
2377
+ "credential_key_id": row["credential_key_id"],
2378
+ "recovery_id": row["recovery_id"],
2379
+ "source_revision": outcome.get("source_revision"),
2380
+ "database_generation_id": row["database_generation_id"],
2381
+ "limit": outcome.get("requested_limit"),
2382
+ "failure_class": outcome.get("failure_class"),
2383
+ "governance_binding_sha256": report["governance"].get(
2384
+ "binding_sha256"
2385
+ ),
2386
+ }
2387
+ observed_request_sha256 = hashlib.sha256(
2388
+ _json_text(expected_request).encode("utf-8")
2389
+ ).hexdigest()
2390
+ (
2391
+ canonical_governance,
2392
+ _,
2393
+ observed_governance_sha256,
2394
+ ) = GDWWorkspace._validated_recovery_governance(
2395
+ report["governance"],
2396
+ namespace=row["namespace"],
2397
+ owner_id=row["owner_id"],
2398
+ credential_key_id=row["credential_key_id"],
2399
+ recovery_id=row["recovery_id"],
2400
+ source_revision=outcome.get("source_revision"),
2401
+ database_generation_id=row["database_generation_id"],
2402
+ limit=outcome.get("requested_limit"),
2403
+ )
2404
+ if canonical_governance != report["governance"]:
2405
+ raise ValueError("recovery governance is not canonical")
2406
+ selection = outcome["selection"]
2407
+ count_fields = (
2408
+ "inspected_pending_effects",
2409
+ "eligible_effects",
2410
+ "rescheduled_effects",
2411
+ "attempts_before",
2412
+ "attempts_after",
2413
+ "claimed_effects",
2414
+ "dead_letter_effects",
2415
+ "invalid_effect_bindings",
2416
+ "invalid_exported_artifacts",
2417
+ "invalid_recovery_audits",
2418
+ )
2419
+ counts = {field: outcome[field] for field in count_fields}
2420
+ if (
2421
+ type(selection) is not list
2422
+ or any(type(value) is not int or value < 0 for value in counts.values())
2423
+ ):
2424
+ raise ValueError("recovery accounting is invalid")
2425
+ selection_fields = {
2426
+ "namespace",
2427
+ "owner_id",
2428
+ "idempotency_key",
2429
+ "database_generation_id",
2430
+ "request_id",
2431
+ "kind",
2432
+ "receipt_hash",
2433
+ "payload_sha256",
2434
+ "intent_sha256",
2435
+ "attempts",
2436
+ "max_attempts",
2437
+ "next_attempt_at",
2438
+ "claim_generation",
2439
+ "last_error_sha256",
2440
+ }
2441
+ for item in selection:
2442
+ if type(item) is not dict or set(item) != selection_fields:
2443
+ raise ValueError("recovery selection shape mismatch")
2444
+ if any(
2445
+ _IDENTIFIER.fullmatch(str(item.get(field) or "")) is None
2446
+ for field in (
2447
+ "namespace",
2448
+ "owner_id",
2449
+ "idempotency_key",
2450
+ "request_id",
2451
+ )
2452
+ ):
2453
+ raise ValueError("recovery selection identity mismatch")
2454
+ if (
2455
+ item.get("database_generation_id")
2456
+ != row["database_generation_id"]
2457
+ or item.get("kind")
2458
+ not in {"receipt_projection", "proof_export"}
2459
+ or (
2460
+ item.get("receipt_hash") is not None
2461
+ and not is_digest(item.get("receipt_hash"))
2462
+ )
2463
+ or any(
2464
+ not is_digest(item.get(field))
2465
+ for field in (
2466
+ "payload_sha256",
2467
+ "intent_sha256",
2468
+ "last_error_sha256",
2469
+ )
2470
+ )
2471
+ or type(item.get("attempts")) is not int
2472
+ or type(item.get("max_attempts")) is not int
2473
+ or not 0 < item["attempts"] < item["max_attempts"]
2474
+ or type(item.get("claim_generation")) is not int
2475
+ or item["claim_generation"] < 0
2476
+ or type(item.get("next_attempt_at")) is not str
2477
+ or _text_time(item["next_attempt_at"])
2478
+ != item["next_attempt_at"]
2479
+ or _normalise_time(item["next_attempt_at"])
2480
+ <= recovery_created_at
2481
+ ):
2482
+ raise ValueError("recovery selection binding mismatch")
2483
+ observed_selection_sha256 = hashlib.sha256(
2484
+ _json_text(selection).encode("utf-8")
2485
+ ).hexdigest()
2486
+ status = outcome["status"]
2487
+ status_contract = (
2488
+ (
2489
+ status == "RESCHEDULED"
2490
+ and counts["rescheduled_effects"] > 0
2491
+ and counts["eligible_effects"]
2492
+ == counts["rescheduled_effects"]
2493
+ and counts["claimed_effects"] == 0
2494
+ )
2495
+ or (
2496
+ status == "NO_ELIGIBLE_EFFECTS"
2497
+ and counts["eligible_effects"] == 0
2498
+ and counts["rescheduled_effects"] == 0
2499
+ and counts["claimed_effects"] == 0
2500
+ )
2501
+ or (
2502
+ status == "DEFERRED_ACTIVE_CLAIM"
2503
+ and counts["eligible_effects"] == 0
2504
+ and counts["rescheduled_effects"] == 0
2505
+ and counts["claimed_effects"] > 0
2506
+ )
2507
+ )
2508
+ if (
2509
+ report["replayed"] is not False
2510
+ or outcome.get("schema")
2511
+ != "szl.gdw.transient-effect-recovery/v2"
2512
+ or receipt.get("schema")
2513
+ != "szl.gdw.transient-effect-recovery-receipt/v2"
2514
+ or receipt.get("operator") != expected_operator
2515
+ or any(
2516
+ _IDENTIFIER.fullmatch(str(value or "")) is None
2517
+ for value in expected_operator.values()
2518
+ )
2519
+ or receipt.get("recovery_id") != row["recovery_id"]
2520
+ or outcome.get("recovery_id") != row["recovery_id"]
2521
+ or re.fullmatch(
2522
+ r"[0-9a-f]{40}",
2523
+ str(outcome.get("source_revision") or ""),
2524
+ )
2525
+ is None
2526
+ or receipt.get("source_revision")
2527
+ != outcome.get("source_revision")
2528
+ or receipt.get("database_generation_id")
2529
+ != row["database_generation_id"]
2530
+ or outcome.get("database_generation_id")
2531
+ != row["database_generation_id"]
2532
+ or outcome.get("failure_class") != "hf-hard-link-enotsup/v1"
2533
+ or type(outcome.get("requested_limit")) is not int
2534
+ or not 1 <= outcome["requested_limit"] <= 1_000
2535
+ or outcome.get("sqlite_integrity") != "ok"
2536
+ or outcome.get("credential_values_recorded") is not False
2537
+ or receipt.get("credential_values_recorded") is not False
2538
+ or receipt.get("atomic_with_mutation") is not True
2539
+ or type(receipt.get("sequence")) is not int
2540
+ or receipt["sequence"] < 0
2541
+ or receipt["sequence"] != row["sequence"]
2542
+ or not is_digest(receipt.get("previous_receipt_sha256"))
2543
+ or receipt.get("previous_receipt_sha256")
2544
+ != row["previous_receipt_sha256"]
2545
+ or not is_digest(receipt.get("previous_chain_sha256"))
2546
+ or receipt.get("previous_chain_sha256")
2547
+ != row["previous_chain_sha256"]
2548
+ or counts["inspected_pending_effects"]
2549
+ < max(counts["eligible_effects"], counts["claimed_effects"])
2550
+ or counts["attempts_before"] != counts["attempts_after"]
2551
+ or counts["attempts_before"]
2552
+ != sum(item["attempts"] for item in selection)
2553
+ or len(selection) != counts["rescheduled_effects"]
2554
+ or any(
2555
+ counts[field] != 0
2556
+ for field in (
2557
+ "dead_letter_effects",
2558
+ "invalid_effect_bindings",
2559
+ "invalid_exported_artifacts",
2560
+ "invalid_recovery_audits",
2561
+ )
2562
+ )
2563
+ or not status_contract
2564
+ or observed_selection_sha256 != outcome.get("selection_sha256")
2565
+ or receipt.get("created_at") != row["created_at"]
2566
+ or receipt.get("request_sha256") != row["request_sha256"]
2567
+ or observed_request_sha256 != row["request_sha256"]
2568
+ or receipt.get("outcome_sha256") != row["outcome_sha256"]
2569
+ or observed_outcome_sha256 != row["outcome_sha256"]
2570
+ or receipt.get("governance_sha256")
2571
+ != row["governance_sha256"]
2572
+ or observed_governance_sha256 != row["governance_sha256"]
2573
+ or receipt.get("selection_sha256")
2574
+ != outcome.get("selection_sha256")
2575
+ or receipt.get("rescheduled_effects")
2576
+ != outcome.get("rescheduled_effects")
2577
+ or receipt.get("attempts_before")
2578
+ != outcome.get("attempts_before")
2579
+ or receipt.get("attempts_after")
2580
+ != outcome.get("attempts_after")
2581
+ or receipt.get("receipt_sha256") != row["receipt_sha256"]
2582
+ or observed_receipt_sha256 != row["receipt_sha256"]
2583
+ or receipt.get("dsse_envelope_sha256")
2584
+ != row["dsse_envelope_sha256"]
2585
+ or observed_envelope_sha256 != row["dsse_envelope_sha256"]
2586
+ or receipt.get("chain_sha256") != row["chain_sha256"]
2587
+ or observed_chain_sha256 != row["chain_sha256"]
2588
+ ):
2589
+ raise ValueError("recovery audit binding mismatch")
2590
+ report["audit_receipt"] = receipt
2591
+ report["replayed"] = False
2592
+ return report
2593
+ except (
2594
+ AttributeError,
2595
+ KeyError,
2596
+ OverflowError,
2597
+ TypeError,
2598
+ ValueError,
2599
+ json.JSONDecodeError,
2600
+ ) as exc:
2601
+ raise GDWConfigurationError(
2602
+ "transient effect recovery audit is invalid"
2603
+ ) from exc
2604
+
2605
+ def _recovery_audit_chain_errors(
2606
+ self,
2607
+ connection: sqlite3.Connection,
2608
+ *,
2609
+ namespace: Optional[str] = None,
2610
+ owner_id: Optional[str] = None,
2611
+ ) -> int:
2612
+ predicate = ""
2613
+ params: Tuple[Any, ...] = ()
2614
+ if namespace is not None or owner_id is not None:
2615
+ if namespace is None or owner_id is None:
2616
+ raise GDWConfigurationError(
2617
+ "recovery audit chain scope must include namespace and owner"
2618
+ )
2619
+ predicate = " WHERE namespace = ? AND owner_id = ?"
2620
+ params = (namespace, owner_id)
2621
+ rows = connection.execute(
2622
+ "SELECT * FROM effect_recovery_audit"
2623
+ + predicate
2624
+ + " ORDER BY namespace, owner_id, sequence",
2625
+ params,
2626
+ ).fetchall()
2627
+ errors = 0
2628
+ identity: Optional[Tuple[str, str]] = None
2629
+ expected_sequence = 0
2630
+ previous_receipt_sha256 = "0" * 64
2631
+ previous_chain_sha256 = "0" * 64
2632
+ for row in rows:
2633
+ row_identity = (row["namespace"], row["owner_id"])
2634
+ if row_identity != identity:
2635
+ identity = row_identity
2636
+ expected_sequence = 0
2637
+ previous_receipt_sha256 = "0" * 64
2638
+ previous_chain_sha256 = "0" * 64
2639
+ row_invalid = False
2640
+ try:
2641
+ self._validated_recovery_audit(row)
2642
+ except GDWConfigurationError:
2643
+ row_invalid = True
2644
+ if (
2645
+ row["sequence"] != expected_sequence
2646
+ or row["previous_receipt_sha256"]
2647
+ != previous_receipt_sha256
2648
+ or row["previous_chain_sha256"] != previous_chain_sha256
2649
+ ):
2650
+ row_invalid = True
2651
+ errors += int(row_invalid)
2652
+ expected_sequence += 1
2653
+ previous_receipt_sha256 = str(row["receipt_sha256"])
2654
+ previous_chain_sha256 = str(row["chain_sha256"])
2655
+ return errors
2656
+
2657
+ def recover_retry_scheduled_effects(
2658
+ self,
2659
+ *,
2660
+ recovery_id: str,
2661
+ credential_key_id: str,
2662
+ expected_source_revision: str,
2663
+ expected_database_generation_id: str,
2664
+ governance: Dict[str, Any],
2665
+ now: Optional[Any] = None,
2666
+ limit: int = 100,
2667
+ ) -> Dict[str, Any]:
2668
+ """Make only integrity-bound legacy HF publication failures due now."""
2669
+
2670
+ canonical_recovery_id = _checked_identity(recovery_id, "recovery_id")
2671
+ canonical_key_id = _checked_identity(
2672
+ credential_key_id,
2673
+ "credential_key_id",
2674
+ )
2675
+ source_revision = str(expected_source_revision or "").strip().lower()
2676
+ generation_id = str(
2677
+ expected_database_generation_id or ""
2678
+ ).strip().lower()
2679
+ if re.fullmatch(r"[0-9a-f]{40}", source_revision) is None:
2680
+ raise GDWConfigurationError(
2681
+ "expected_source_revision must be a full Git SHA"
2682
+ )
2683
+ if (
2684
+ re.fullmatch(r"[0-9a-f]{32}", generation_id) is None
2685
+ or generation_id != self.database_generation_id
2686
+ ):
2687
+ raise GDWConfigurationError(
2688
+ "transient effect recovery database generation mismatch"
2689
+ )
2690
+ if type(limit) is not int or not 1 <= limit <= 1_000:
2691
+ raise GDWConfigurationError(
2692
+ "transient effect recovery limit must be between 1 and 1000"
2693
+ )
2694
+
2695
+ current = _normalise_time(now)
2696
+ now_text = current.isoformat()
2697
+ (
2698
+ canonical_governance,
2699
+ governance_binding_sha256,
2700
+ governance_sha256,
2701
+ ) = self._validated_recovery_governance(
2702
+ governance,
2703
+ namespace=self.namespace,
2704
+ owner_id=self.owner_id,
2705
+ credential_key_id=canonical_key_id,
2706
+ recovery_id=canonical_recovery_id,
2707
+ source_revision=source_revision,
2708
+ database_generation_id=generation_id,
2709
+ limit=limit,
2710
+ )
2711
+ request = {
2712
+ "schema": "szl.gdw.transient-effect-recovery-request/v1",
2713
+ "namespace": self.namespace,
2714
+ "owner_id": self.owner_id,
2715
+ "credential_key_id": canonical_key_id,
2716
+ "recovery_id": canonical_recovery_id,
2717
+ "source_revision": source_revision,
2718
+ "database_generation_id": generation_id,
2719
+ "limit": limit,
2720
+ "failure_class": "hf-hard-link-enotsup/v1",
2721
+ "governance_binding_sha256": governance_binding_sha256,
2722
+ }
2723
+ request_sha256 = hashlib.sha256(
2724
+ _json_text(request).encode("utf-8")
2725
+ ).hexdigest()
2726
+ empty_selection_sha256 = hashlib.sha256(b"[]").hexdigest()
2727
+ transient_error = (
2728
+ "OSError: [Errno 95] Operation not supported: "
2729
+ "'%/.gdw-artifact-%.tmp' -> '%"
2730
+ )
2731
+
2732
+ with self.transaction() as connection:
2733
+ cached = connection.execute(
2734
+ """
2735
+ SELECT * FROM effect_recovery_audit
2736
+ WHERE namespace = ? AND owner_id = ? AND recovery_id = ?
2737
+ """,
2738
+ (self.namespace, self.owner_id, canonical_recovery_id),
2739
+ ).fetchone()
2740
+ if cached is not None:
2741
+ if cached["request_sha256"] != request_sha256:
2742
+ raise GDWConfigurationError(
2743
+ "recovery_id was already used with different content"
2744
+ )
2745
+ report = self._validated_recovery_audit(cached)
2746
+ if self._recovery_audit_chain_errors(
2747
+ connection,
2748
+ namespace=self.namespace,
2749
+ owner_id=self.owner_id,
2750
+ ):
2751
+ raise GDWConfigurationError(
2752
+ "transient effect recovery audit chain is invalid"
2753
+ )
2754
+ report["replayed"] = True
2755
+ return report
2756
+
2757
+ before = self.integrity(
2758
+ global_scope=True,
2759
+ connection=connection,
2760
+ )
2761
+ if (
2762
+ before.get("ok") is not True
2763
+ or before.get("sqlite_integrity") != "ok"
2764
+ or before.get("dead_letter_effects") != 0
2765
+ or before.get("pending_proofs") != 0
2766
+ or before.get("invalid_effect_bindings") != 0
2767
+ or before.get("invalid_exported_artifacts") != 0
2768
+ or before.get("invalid_recovery_audits") != 0
2769
+ ):
2770
+ raise GDWConfigurationError(
2771
+ "transient effect recovery refused by global integrity"
2772
+ )
2773
+
2774
+ def record(outcome: Dict[str, Any]) -> Dict[str, Any]:
2775
+ outcome_sha256 = hashlib.sha256(
2776
+ _json_text(outcome).encode("utf-8")
2777
+ ).hexdigest()
2778
+ previous = connection.execute(
2779
+ """
2780
+ SELECT sequence, receipt_sha256, chain_sha256
2781
+ FROM effect_recovery_audit
2782
+ WHERE namespace = ? AND owner_id = ?
2783
+ ORDER BY sequence DESC
2784
+ LIMIT 1
2785
+ """,
2786
+ (self.namespace, self.owner_id),
2787
+ ).fetchone()
2788
+ sequence = 0 if previous is None else int(previous["sequence"]) + 1
2789
+ previous_receipt_sha256 = (
2790
+ "0" * 64 if previous is None else previous["receipt_sha256"]
2791
+ )
2792
+ previous_chain_sha256 = (
2793
+ "0" * 64 if previous is None else previous["chain_sha256"]
2794
+ )
2795
+ receipt_payload = {
2796
+ "schema": "szl.gdw.transient-effect-recovery-receipt/v2",
2797
+ "operator": {
2798
+ "namespace": self.namespace,
2799
+ "owner_id": self.owner_id,
2800
+ "credential_key_id": canonical_key_id,
2801
+ },
2802
+ "recovery_id": canonical_recovery_id,
2803
+ "source_revision": source_revision,
2804
+ "database_generation_id": generation_id,
2805
+ "request_sha256": request_sha256,
2806
+ "outcome_sha256": outcome_sha256,
2807
+ "governance_sha256": governance_sha256,
2808
+ "selection_sha256": outcome["selection_sha256"],
2809
+ "rescheduled_effects": outcome["rescheduled_effects"],
2810
+ "attempts_before": outcome["attempts_before"],
2811
+ "attempts_after": outcome["attempts_after"],
2812
+ "sequence": sequence,
2813
+ "previous_receipt_sha256": previous_receipt_sha256,
2814
+ "previous_chain_sha256": previous_chain_sha256,
2815
+ "atomic_with_mutation": True,
2816
+ "created_at": now_text,
2817
+ "credential_values_recorded": False,
2818
+ }
2819
+ receipt_sha256 = hashlib.sha256(
2820
+ _json_text(receipt_payload).encode("utf-8")
2821
+ ).hexdigest()
2822
+ dsse_envelope = szl_dsse.sign_payload(
2823
+ receipt_payload,
2824
+ szl_dsse.KHIPU_PAYLOAD_TYPE,
2825
+ )
2826
+ signed = dsse_envelope.get("signed") is True
2827
+ signatures = dsse_envelope.get("signatures")
2828
+ if (
2829
+ type(signatures) is not list
2830
+ or (signed and len(signatures) != 1)
2831
+ or (not signed and signatures != [])
2832
+ ):
2833
+ raise GDWConfigurationError(
2834
+ "transient recovery DSSE signer returned invalid evidence"
2835
+ )
2836
+ receipt_status = (
2837
+ "SIGNED_KHIPU_DSSE" if signed else "UNSIGNED_KHIPU_DSSE"
2838
+ )
2839
+ dsse_envelope_sha256 = hashlib.sha256(
2840
+ _json_text(dsse_envelope).encode("utf-8")
2841
+ ).hexdigest()
2842
+ chain_sha256 = hashlib.sha256(
2843
+ _json_text(
2844
+ {
2845
+ "previous_chain_sha256": previous_chain_sha256,
2846
+ "receipt_sha256": receipt_sha256,
2847
+ "receipt_status": receipt_status,
2848
+ "dsse_envelope_sha256": dsse_envelope_sha256,
2849
+ }
2850
+ ).encode("utf-8")
2851
+ ).hexdigest()
2852
+ receipt = {
2853
+ **receipt_payload,
2854
+ "receipt_status": receipt_status,
2855
+ "receipt_sha256": receipt_sha256,
2856
+ "dsse_envelope_sha256": dsse_envelope_sha256,
2857
+ "chain_sha256": chain_sha256,
2858
+ "dsse_envelope": dsse_envelope,
2859
+ }
2860
+ report = {
2861
+ **outcome,
2862
+ "governance": canonical_governance,
2863
+ "audit_receipt": receipt,
2864
+ "replayed": False,
2865
+ }
2866
+ report_text = _json_text(report)
2867
+ self._reserve_usage(
2868
+ connection,
2869
+ self.namespace,
2870
+ self.owner_id,
2871
+ stored_bytes=_byte_len(report_text),
2872
+ )
2873
+ connection.execute(
2874
+ """
2875
+ INSERT INTO effect_recovery_audit(
2876
+ namespace, owner_id, recovery_id, sequence,
2877
+ credential_key_id, database_generation_id,
2878
+ request_sha256, outcome_sha256, governance_sha256,
2879
+ receipt_sha256, previous_receipt_sha256,
2880
+ previous_chain_sha256, chain_sha256,
2881
+ dsse_envelope_sha256, report_json, created_at
2882
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2883
+ """,
2884
+ (
2885
+ self.namespace,
2886
+ self.owner_id,
2887
+ canonical_recovery_id,
2888
+ sequence,
2889
+ canonical_key_id,
2890
+ generation_id,
2891
+ request_sha256,
2892
+ outcome_sha256,
2893
+ governance_sha256,
2894
+ receipt_sha256,
2895
+ previous_receipt_sha256,
2896
+ previous_chain_sha256,
2897
+ chain_sha256,
2898
+ dsse_envelope_sha256,
2899
+ report_text,
2900
+ now_text,
2901
+ ),
2902
+ )
2903
+ return report
2904
+
2905
+ if before.get("claimed_effects") != 0:
2906
+ return record(
2907
+ {
2908
+ "schema": "szl.gdw.transient-effect-recovery/v2",
2909
+ "status": "DEFERRED_ACTIVE_CLAIM",
2910
+ "recovery_id": canonical_recovery_id,
2911
+ "source_revision": source_revision,
2912
+ "requested_limit": limit,
2913
+ "failure_class": "hf-hard-link-enotsup/v1",
2914
+ "database_generation_id": generation_id,
2915
+ "inspected_pending_effects": before["pending_effects"],
2916
+ "eligible_effects": 0,
2917
+ "rescheduled_effects": 0,
2918
+ "attempts_before": 0,
2919
+ "attempts_after": 0,
2920
+ "selection": [],
2921
+ "selection_sha256": empty_selection_sha256,
2922
+ "sqlite_integrity": before["sqlite_integrity"],
2923
+ "claimed_effects": before["claimed_effects"],
2924
+ "dead_letter_effects": before["dead_letter_effects"],
2925
+ "invalid_effect_bindings": before[
2926
+ "invalid_effect_bindings"
2927
+ ],
2928
+ "invalid_exported_artifacts": before[
2929
+ "invalid_exported_artifacts"
2930
+ ],
2931
+ "invalid_recovery_audits": before[
2932
+ "invalid_recovery_audits"
2933
+ ],
2934
+ "credential_values_recorded": False,
2935
+ }
2936
+ )
2937
+
2938
+ eligibility = """
2939
+ status = 'PENDING'
2940
+ AND attempts > 0
2941
+ AND attempts < max_attempts
2942
+ AND next_attempt_at > ?
2943
+ AND database_generation_id = ?
2944
+ AND kind IN ('receipt_projection', 'proof_export')
2945
+ AND last_error LIKE ?
2946
+ AND lease_owner IS NULL
2947
+ AND lease_until IS NULL
2948
+ AND artifact_json IS NULL
2949
+ AND exported_at IS NULL
2950
+ AND tombstoned_at IS NULL
2951
+ """
2952
+ eligible_total = int(
2953
+ connection.execute(
2954
+ f"SELECT COUNT(*) FROM effect_outbox WHERE {eligibility}",
2955
+ (now_text, generation_id, transient_error),
2956
+ ).fetchone()[0]
2957
+ )
2958
+ if eligible_total > limit:
2959
+ raise GDWConfigurationError(
2960
+ "transient effect recovery exceeds the bounded limit"
2961
+ )
2962
+ rows = connection.execute(
2963
+ f"""
2964
+ SELECT * FROM effect_outbox
2965
+ WHERE {eligibility}
2966
+ ORDER BY next_attempt_at, created_at, idempotency_key
2967
+ LIMIT ?
2968
+ """,
2969
+ (now_text, generation_id, transient_error, limit),
2970
+ ).fetchall()
2971
+ selection = []
2972
+ original_rows = {}
2973
+ for row in rows:
2974
+ try:
2975
+ payload = json.loads(row["payload_json"])
2976
+ except (TypeError, json.JSONDecodeError) as exc:
2977
+ raise GDWConfigurationError(
2978
+ "transient effect recovery found invalid payload"
2979
+ ) from exc
2980
+ candidate = dict(row)
2981
+ candidate["payload"] = payload
2982
+ if not self._recoverable_publication_error(
2983
+ row["last_error"],
2984
+ expected_intent_sha256=row["intent_sha256"],
2985
+ ):
2986
+ raise GDWConfigurationError(
2987
+ "transient effect recovery found unknown error"
2988
+ )
2989
+ if self._connection_effect_binding_errors(connection, candidate):
2990
+ raise GDWConfigurationError(
2991
+ "transient effect recovery found invalid binding"
2992
+ )
2993
+ original_rows[
2994
+ (row["namespace"], row["owner_id"], row["idempotency_key"])
2995
+ ] = dict(row)
2996
+ selection.append(
2997
+ {
2998
+ "namespace": row["namespace"],
2999
+ "owner_id": row["owner_id"],
3000
+ "idempotency_key": row["idempotency_key"],
3001
+ "database_generation_id": row[
3002
+ "database_generation_id"
3003
+ ],
3004
+ "request_id": row["request_id"],
3005
+ "kind": row["kind"],
3006
+ "receipt_hash": row["receipt_hash"],
3007
+ "payload_sha256": row["payload_sha256"],
3008
+ "intent_sha256": row["intent_sha256"],
3009
+ "attempts": int(row["attempts"]),
3010
+ "max_attempts": int(row["max_attempts"]),
3011
+ "next_attempt_at": row["next_attempt_at"],
3012
+ "claim_generation": int(row["claim_generation"]),
3013
+ "last_error_sha256": hashlib.sha256(
3014
+ str(row["last_error"]).encode("utf-8")
3015
+ ).hexdigest(),
3016
+ }
3017
+ )
3018
+
3019
+ selection_sha256 = hashlib.sha256(
3020
+ _json_text(selection).encode("utf-8")
3021
+ ).hexdigest()
3022
+ attempts_before = sum(item["attempts"] for item in selection)
3023
+ for row, item in zip(rows, selection):
3024
+ updated = connection.execute(
3025
+ """
3026
+ UPDATE effect_outbox
3027
+ SET next_attempt_at = ?
3028
+ WHERE namespace = ? AND owner_id = ?
3029
+ AND idempotency_key = ?
3030
+ AND database_generation_id = ?
3031
+ AND request_id = ? AND kind = ?
3032
+ AND receipt_hash IS ?
3033
+ AND payload_json = ? AND payload_sha256 = ?
3034
+ AND intent_sha256 = ?
3035
+ AND status = 'PENDING'
3036
+ AND attempts = ? AND max_attempts = ?
3037
+ AND next_attempt_at = ? AND claim_generation = ?
3038
+ AND last_error = ?
3039
+ AND lease_owner IS NULL AND lease_until IS NULL
3040
+ AND artifact_json IS NULL AND exported_at IS NULL
3041
+ AND tombstoned_at IS NULL
3042
+ """,
3043
+ (
3044
+ now_text,
3045
+ item["namespace"],
3046
+ item["owner_id"],
3047
+ item["idempotency_key"],
3048
+ item["database_generation_id"],
3049
+ item["request_id"],
3050
+ item["kind"],
3051
+ item["receipt_hash"],
3052
+ row["payload_json"],
3053
+ item["payload_sha256"],
3054
+ item["intent_sha256"],
3055
+ item["attempts"],
3056
+ item["max_attempts"],
3057
+ item["next_attempt_at"],
3058
+ item["claim_generation"],
3059
+ row["last_error"],
3060
+ ),
3061
+ )
3062
+ if updated.rowcount != 1:
3063
+ raise GDWConfigurationError(
3064
+ "transient effect changed during recovery"
3065
+ )
3066
+
3067
+ after = self.integrity(global_scope=True, connection=connection)
3068
+ attempts_after = 0
3069
+ for item in selection:
3070
+ persisted = connection.execute(
3071
+ """
3072
+ SELECT * FROM effect_outbox
3073
+ WHERE namespace = ? AND owner_id = ? AND idempotency_key = ?
3074
+ """,
3075
+ (
3076
+ item["namespace"],
3077
+ item["owner_id"],
3078
+ item["idempotency_key"],
3079
+ ),
3080
+ ).fetchone()
3081
+ if persisted is None:
3082
+ raise GDWConfigurationError(
3083
+ "transient effect disappeared during recovery"
3084
+ )
3085
+ original = original_rows[
3086
+ (
3087
+ item["namespace"],
3088
+ item["owner_id"],
3089
+ item["idempotency_key"],
3090
+ )
3091
+ ]
3092
+ changed = {
3093
+ field
3094
+ for field in persisted.keys()
3095
+ if field != "next_attempt_at"
3096
+ and persisted[field] != original[field]
3097
+ }
3098
+ if (
3099
+ changed
3100
+ or persisted["next_attempt_at"] != now_text
3101
+ or persisted["status"] != "PENDING"
3102
+ ):
3103
+ raise GDWConfigurationError(
3104
+ "transient effect accounting changed during recovery"
3105
+ )
3106
+ attempts_after += int(persisted["attempts"])
3107
+ if (
3108
+ after.get("ok") is not True
3109
+ or after.get("sqlite_integrity") != "ok"
3110
+ or after.get("pending_effects") != before.get("pending_effects")
3111
+ or after.get("claimed_effects") != 0
3112
+ or after.get("dead_letter_effects") != 0
3113
+ or after.get("invalid_effect_bindings") != 0
3114
+ or after.get("invalid_exported_artifacts") != 0
3115
+ or after.get("invalid_recovery_audits") != 0
3116
+ or attempts_after != attempts_before
3117
+ ):
3118
+ raise GDWConfigurationError(
3119
+ "transient effect recovery changed protected accounting"
3120
+ )
3121
+
3122
+ return record(
3123
+ {
3124
+ "schema": "szl.gdw.transient-effect-recovery/v2",
3125
+ "status": (
3126
+ "RESCHEDULED" if rows else "NO_ELIGIBLE_EFFECTS"
3127
+ ),
3128
+ "recovery_id": canonical_recovery_id,
3129
+ "source_revision": source_revision,
3130
+ "requested_limit": limit,
3131
+ "failure_class": "hf-hard-link-enotsup/v1",
3132
+ "database_generation_id": generation_id,
3133
+ "inspected_pending_effects": before["pending_effects"],
3134
+ "eligible_effects": eligible_total,
3135
+ "rescheduled_effects": len(rows),
3136
+ "attempts_before": attempts_before,
3137
+ "attempts_after": attempts_after,
3138
+ "selection": selection,
3139
+ "selection_sha256": selection_sha256,
3140
+ "sqlite_integrity": after["sqlite_integrity"],
3141
+ "claimed_effects": after["claimed_effects"],
3142
+ "dead_letter_effects": after["dead_letter_effects"],
3143
+ "invalid_effect_bindings": after[
3144
+ "invalid_effect_bindings"
3145
+ ],
3146
+ "invalid_exported_artifacts": after[
3147
+ "invalid_exported_artifacts"
3148
+ ],
3149
+ "invalid_recovery_audits": after[
3150
+ "invalid_recovery_audits"
3151
+ ],
3152
+ "credential_values_recorded": False,
3153
+ }
3154
+ )
3155
 
3156
  def assert_effect_claim(
3157
  self,
 
3493
  UNION SELECT namespace, owner_id FROM receipts
3494
  UNION SELECT namespace, owner_id FROM proof_outbox
3495
  UNION SELECT namespace, owner_id FROM effect_outbox
3496
+ UNION SELECT namespace, owner_id FROM effect_recovery_audit
3497
  ORDER BY namespace, owner_id
3498
  """
3499
  ).fetchall()
 
3561
  LENGTH(CAST(payload_json AS BLOB)) +
3562
  COALESCE(LENGTH(CAST(artifact_json AS BLOB)), 0))
3563
  FROM effect_outbox
3564
+ WHERE namespace = ? AND owner_id = ?), 0) +
3565
+ COALESCE((SELECT SUM(LENGTH(CAST(report_json AS BLOB)))
3566
+ FROM effect_recovery_audit
3567
  WHERE namespace = ? AND owner_id = ?), 0)
3568
  """,
3569
  (
 
3577
  owner_id,
3578
  namespace,
3579
  owner_id,
3580
+ namespace,
3581
+ owner_id,
3582
  ),
3583
  ).fetchone()[0]
3584
  )
 
3615
  UNION SELECT namespace, owner_id FROM receipts
3616
  UNION SELECT namespace, owner_id FROM proof_outbox
3617
  UNION SELECT namespace, owner_id FROM effect_outbox
3618
+ UNION SELECT namespace, owner_id FROM effect_recovery_audit
3619
  """
3620
  ).fetchall()
3621
  connection.execute("DELETE FROM usage")
 
3852
  )
3853
  for table in _V1_TABLES
3854
  }
3855
+ counts["effect_recovery_audit"] = int(
3856
+ connection.execute(
3857
+ "SELECT COUNT(*) FROM effect_recovery_audit" + predicate,
3858
+ params,
3859
+ ).fetchone()[0]
3860
+ )
3861
  effect_predicate = (
3862
  "status IN ('PENDING', 'CLAIMED')"
3863
  if global_scope
 
3926
  "invalid_request_digests": 0,
3927
  "invalid_receipt_digests": 0,
3928
  "invalid_proof_digests": 0,
3929
+ "invalid_recovery_audits": 0,
3930
  }
3931
  scoped_suffix = (
3932
  ""
 
4060
  json.JSONDecodeError,
4061
  ):
4062
  digest_violations["invalid_proof_digests"] += 1
4063
+ digest_violations["invalid_recovery_audits"] += (
4064
+ self._recovery_audit_chain_errors(
4065
+ connection,
4066
+ namespace=None if global_scope else ns,
4067
+ owner_id=None if global_scope else owner,
4068
+ )
4069
+ )
4070
  effect_rows = connection.execute(
4071
  """
4072
  SELECT namespace, owner_id, idempotency_key,
routers/gdw_frontier.py CHANGED
@@ -12,7 +12,7 @@ from typing import List, Literal, Optional
12
  from urllib.request import Request as UrlRequest
13
  from urllib.request import urlopen
14
 
15
- from fastapi import Header, HTTPException, Request
16
  from fastapi.responses import PlainTextResponse
17
  from pydantic import BaseModel, Field, ValidationError
18
 
@@ -352,6 +352,59 @@ def _require_write_ready(namespace: str) -> None:
352
  )
353
 
354
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
355
  def _public_runtime_health(runtime: dict) -> dict:
356
  storage = runtime.get("storage")
357
  public_storage = None
@@ -567,6 +620,70 @@ def _canonical_policy_evaluate(action: dict) -> dict:
567
  return result
568
 
569
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
570
  def _risk_severity(risk_budget: float) -> str:
571
  if risk_budget < 0.25:
572
  return "low"
@@ -894,6 +1011,87 @@ def register(app, ns: str = "a11oy"):
894
  "database_generation_id": integrity["database_generation_id"],
895
  }
896
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
897
  @app.get(prefix + "/integrity/global")
898
  @app.get("/v1/gdw/integrity/global")
899
  def gdw_global_integrity(
@@ -1344,6 +1542,7 @@ def register(app, ns: str = "a11oy"):
1344
  prefix + "/integrity",
1345
  prefix + "/integrity/global",
1346
  prefix + "/drain",
 
1347
  prefix + "/sessions/{session_id}",
1348
  prefix + "/step",
1349
  ],
 
12
  from urllib.request import Request as UrlRequest
13
  from urllib.request import urlopen
14
 
15
+ from fastapi import Header, HTTPException, Query, Request
16
  from fastapi.responses import PlainTextResponse
17
  from pydantic import BaseModel, Field, ValidationError
18
 
 
352
  )
353
 
354
 
355
+ def _require_transient_recovery_runtime(
356
+ namespace: str,
357
+ expected_source_revision: Optional[str],
358
+ ) -> str:
359
+ expected = str(expected_source_revision or "").strip().lower()
360
+ observed_source = os.environ.get("SZL_GIT_SHA", "").strip().lower()
361
+ if (
362
+ re.fullmatch(r"[0-9a-f]{40}", expected) is None
363
+ or observed_source != expected
364
+ ):
365
+ raise HTTPException(
366
+ status_code=409,
367
+ detail="GDW recovery source revision mismatch",
368
+ )
369
+ runtime = runtime_health()
370
+ storage = runtime.get("storage") or {}
371
+ drain = runtime.get("drain") or {}
372
+ database_generation_id = str(
373
+ storage.get("database_generation_id") or ""
374
+ )
375
+ runtime_ready = (
376
+ runtime.get("startup_state") == "READY"
377
+ and runtime.get("evidence_label") == "VERIFIED"
378
+ and storage.get("persistence_required") is True
379
+ and storage.get("mount_verified") is True
380
+ and storage.get("journal_mode_requested") == "DELETE"
381
+ and storage.get("journal_mode_observed") == "DELETE"
382
+ and storage.get("synchronous_requested") == "FULL"
383
+ and storage.get("synchronous_observed") == 2
384
+ and storage.get("sqlite_integrity") == "ok"
385
+ and storage.get("proof_export_mode") == "outbox"
386
+ and storage.get("schema_version") == GDWWorkspace.schema_version()
387
+ and re.fullmatch(
388
+ r"[0-9a-f]{32}",
389
+ database_generation_id,
390
+ )
391
+ is not None
392
+ and drain.get("enabled") is True
393
+ and drain.get("running") is True
394
+ and _governance_ready()
395
+ )
396
+ try:
397
+ policy_ready = _canonical_policy_ready()
398
+ except Exception:
399
+ policy_ready = False
400
+ if not runtime_ready or not policy_ready:
401
+ raise HTTPException(
402
+ status_code=503,
403
+ detail="GDW recovery runtime contract is unavailable",
404
+ )
405
+ return database_generation_id
406
+
407
+
408
  def _public_runtime_health(runtime: dict) -> dict:
409
  storage = runtime.get("storage")
410
  public_storage = None
 
620
  return result
621
 
622
 
623
+ def _transient_recovery_governance(
624
+ *,
625
+ principal: Principal,
626
+ recovery_id: str,
627
+ source_revision: str,
628
+ database_generation_id: str,
629
+ limit: int,
630
+ ) -> dict:
631
+ """Obtain an explicit signed canonical-policy allow for one recovery call."""
632
+
633
+ binding = {
634
+ "schema": "szl.gdw.transient-effect-recovery-authorization/v1",
635
+ "action_type": "gdw.transient-effect-recovery",
636
+ "namespace": principal.namespace,
637
+ "owner_id": principal.owner_id,
638
+ "credential_key_id": principal.key_id,
639
+ "recovery_id": recovery_id,
640
+ "source_revision": source_revision,
641
+ "database_generation_id": database_generation_id,
642
+ "limit": limit,
643
+ "failure_class": "hf-hard-link-enotsup/v1",
644
+ }
645
+ binding_sha256 = _sha(binding)
646
+ action = {
647
+ "actionId": f"gdw-recovery:{binding_sha256}",
648
+ "severity": "high",
649
+ "decisionClass": "ordinary",
650
+ "confidence": 1.0,
651
+ "witnesses": [
652
+ {
653
+ "id": (
654
+ f"principal:{principal.namespace}:"
655
+ f"{principal.owner_id}:{principal.key_id}"
656
+ ),
657
+ "role": "operator",
658
+ "attested": True,
659
+ },
660
+ {
661
+ "id": f"workload:szl-holdings/a11oy@{source_revision}",
662
+ "role": "workload",
663
+ "attested": True,
664
+ },
665
+ ],
666
+ }
667
+ result = _canonical_policy_evaluate(action)
668
+ if result.get("decision") != "allow":
669
+ raise PermissionError("canonical policy denied transient recovery")
670
+ return {
671
+ "schema": "szl.gdw.transient-effect-recovery-governance/v1",
672
+ "decision": "ALLOW",
673
+ "binding": binding,
674
+ "binding_sha256": binding_sha256,
675
+ "policy_gateway": {
676
+ "decision": "ALLOW",
677
+ "gate": result["gate"],
678
+ "receipt_hash": result["receipt_hash"],
679
+ "receipt_signed": True,
680
+ "receipts_in_eq_out": True,
681
+ "action_id": action["actionId"],
682
+ "witnesses": action["witnesses"],
683
+ },
684
+ }
685
+
686
+
687
  def _risk_severity(risk_budget: float) -> str:
688
  if risk_budget < 0.25:
689
  return "low"
 
1011
  "database_generation_id": integrity["database_generation_id"],
1012
  }
1013
 
1014
+ @app.post(prefix + "/recovery/transient-effects")
1015
+ @app.post("/v1/gdw/recovery/transient-effects")
1016
+ def gdw_recover_transient_effects(
1017
+ limit: int = Query(default=100, ge=1, le=1_000),
1018
+ authorization: Optional[str] = Header(default=None, alias="Authorization"),
1019
+ expected_source_revision: Optional[str] = Header(
1020
+ default=None,
1021
+ alias="X-Expected-Source-Revision",
1022
+ ),
1023
+ idempotency_key: Optional[str] = Header(
1024
+ default=None,
1025
+ alias="Idempotency-Key",
1026
+ ),
1027
+ ):
1028
+ principal = _authorise(
1029
+ authorization,
1030
+ namespace=ns,
1031
+ required_scopes=("effects:recover", "integrity:global"),
1032
+ )
1033
+ runtime_generation = _require_transient_recovery_runtime(
1034
+ ns,
1035
+ expected_source_revision,
1036
+ )
1037
+ if not idempotency_key or not _ID_PATTERN.fullmatch(idempotency_key):
1038
+ raise HTTPException(
1039
+ status_code=422,
1040
+ detail=(
1041
+ "Idempotency-Key must be 1-128 canonical identifier "
1042
+ "characters"
1043
+ ),
1044
+ )
1045
+ workspace = _workspace(principal)
1046
+ if workspace.database_generation_id != runtime_generation:
1047
+ raise HTTPException(
1048
+ status_code=503,
1049
+ detail="GDW recovery database generation changed",
1050
+ )
1051
+ try:
1052
+ governance = _transient_recovery_governance(
1053
+ principal=principal,
1054
+ recovery_id=idempotency_key,
1055
+ source_revision=str(expected_source_revision),
1056
+ database_generation_id=runtime_generation,
1057
+ limit=limit,
1058
+ )
1059
+ except PermissionError as exc:
1060
+ raise HTTPException(
1061
+ status_code=403,
1062
+ detail="GDW recovery denied by canonical policy",
1063
+ ) from exc
1064
+ except Exception as exc:
1065
+ raise HTTPException(
1066
+ status_code=503,
1067
+ detail="GDW recovery canonical policy unavailable",
1068
+ ) from exc
1069
+ try:
1070
+ report = workspace.recover_retry_scheduled_effects(
1071
+ recovery_id=idempotency_key,
1072
+ credential_key_id=principal.key_id,
1073
+ expected_source_revision=str(expected_source_revision),
1074
+ expected_database_generation_id=runtime_generation,
1075
+ governance=governance,
1076
+ limit=limit,
1077
+ )
1078
+ except GDWConfigurationError as exc:
1079
+ if str(exc) == "recovery_id was already used with different content":
1080
+ raise HTTPException(
1081
+ status_code=409,
1082
+ detail="Idempotency-Key was already used with different content",
1083
+ ) from exc
1084
+ raise HTTPException(
1085
+ status_code=503,
1086
+ detail="GDW recovery refused by integrity gate",
1087
+ ) from exc
1088
+ if report.get("database_generation_id") != runtime_generation:
1089
+ raise HTTPException(
1090
+ status_code=503,
1091
+ detail="GDW recovery database generation changed",
1092
+ )
1093
+ return report
1094
+
1095
  @app.get(prefix + "/integrity/global")
1096
  @app.get("/v1/gdw/integrity/global")
1097
  def gdw_global_integrity(
 
1542
  prefix + "/integrity",
1543
  prefix + "/integrity/global",
1544
  prefix + "/drain",
1545
+ prefix + "/recovery/transient-effects",
1546
  prefix + "/sessions/{session_id}",
1547
  prefix + "/step",
1548
  ],