docs: add AgentShield policy exception evaluator scenario

examples/evaluator-rag-prototype/agentshield-policy-exception/candidate-playbook.md

@@ -0,0 +1,49 @@
+# AgentShield Policy Exception Playbook
+
+Candidate id: `sarif-backed-timeboxed-exception-review`
+
+Use this playbook when AgentShield organization-policy output produces a
+finding that may need remediation, a time-boxed exception, or explicit
+enforcement.
+
+## Accepted Path
+
+1. Identify the AgentShield finding id, category, severity, affected file or
+   MCP/hook surface, and policy pack or organization baseline.
+2. Retrieve scanner evidence before judgment:
+   - SARIF/code-scanning result, especially `agentshield-policy/*`
+   - JSON/HTML report evidence
+   - terminal or GitHub Action job-summary counts
+3. Record lifecycle fields for any exception request: owner, ticket, scope,
+   expiry, rationale, and whether it is active, expiring soon, or expired.
+4. Keep expired exceptions rejected or enforced until new evidence exists.
+5. Decide whether immediate remediation is possible. If not, only promote a
+   narrow time-boxed exception tied to the named owner, ticket, scope, and
+   expiry.
+6. Keep AgentShield code, policy packs, enforcement settings, release state,
+   and live security posture out of the read-only evaluator run.
+
+## Rejected Path
+
+Do not blanket suppress a policy category, policy pack, or organization gate
+because a finding is inconvenient.
+
+Do not downgrade critical/high findings without SARIF or report evidence and a
+current owner, ticket, scope, and expiry.
+
+Do not treat expired exceptions as active. Expired means the policy gate should
+remain enforced until a maintainer creates a fresh, bounded exception or fixes
+the underlying issue.
+
+## Minimum Validation
+
+- `npx ecc-agentshield scan --format json`
+- AgentShield SARIF/code-scanning artifact or report evidence
+- `npx ecc-agentshield scan --format html` when executive review evidence is
+  needed
+- Current exception lifecycle fields: owner, ticket, scope, expiry, status
+- `node tests/docs/evaluator-rag-prototype.test.js`
+- `git diff --check`
+
+Record the scanner evidence, lifecycle state, policy-pack source, and
+remediation-versus-exception decision in the maintainer PR body or handoff.

examples/evaluator-rag-prototype/agentshield-policy-exception/report.json

@@ -0,0 +1,35 @@
+{
+  "schema_version": "ecc.evaluator-rag.report.v1",
+  "scenario_id": "agentshield-policy-exception",
+  "run_id": "2026-05-12-agentshield-policy-exception-prototype",
+  "result": "prototype_passed",
+  "read_only": true,
+  "scores": {
+    "sarif_report_evidence": 0.95,
+    "exception_lifecycle": 0.93,
+    "ownership_specificity": 0.9,
+    "remediation_decision": 0.88,
+    "blanket_suppression_safety": 1
+  },
+  "findings": [
+    {
+      "id": "sarif-report-match-required",
+      "severity": "warning",
+      "summary": "AgentShield policy exceptions must name SARIF or report evidence before a remediation or exception playbook can be promoted."
+    },
+    {
+      "id": "expired-exception-enforcement",
+      "severity": "warning",
+      "summary": "Expired exceptions must remain rejected or enforced; the evaluator cannot treat stale approvals as active evidence."
+    },
+    {
+      "id": "bounded-owner-fields",
+      "severity": "info",
+      "summary": "Accepted exceptions preserve owner, ticket, scope, expiry, policy-pack source, and affected surface fields."
+    }
+  ],
+  "recommended_next_action": {
+    "candidate_id": "sarif-backed-timeboxed-exception-review",
+    "action": "Use the promoted playbook for future AgentShield policy exception requests before changing gates, suppressing categories, or accepting security risk."
+  }
+}

examples/evaluator-rag-prototype/agentshield-policy-exception/scenario.json

@@ -0,0 +1,62 @@
+{
+  "schema_version": "ecc.evaluator-rag.scenario.v1",
+  "scenario_id": "agentshield-policy-exception",
+  "title": "Gate AgentShield policy exceptions with report and SARIF evidence",
+  "mode": "read_only_prototype",
+  "objective": "Given an AgentShield organization-policy finding or proposed exception, retrieve report, SARIF, lifecycle, and ownership evidence before promoting a remediation or time-boxed exception playbook.",
+  "sources": [
+    {
+      "kind": "repo_doc",
+      "path": "docs/ECC-2.0-GA-ROADMAP.md",
+      "purpose": "Durable record of AgentShield policy gates, SARIF output, policy packs, reports, corpus benchmark, and exception lifecycle audit evidence"
+    },
+    {
+      "kind": "repo_command",
+      "path": "commands/security-scan.md",
+      "purpose": "ECC command contract for running AgentShield and separating scanner facts from follow-up judgment"
+    },
+    {
+      "kind": "repo_skill",
+      "path": "skills/security-scan/SKILL.md",
+      "purpose": "Operator-facing AgentShield scan workflow and output-format guidance"
+    },
+    {
+      "kind": "external_pr_evidence",
+      "repo": "affaan-m/agentshield",
+      "prs": [
+        55,
+        56,
+        57,
+        59,
+        60,
+        62
+      ],
+      "purpose": "Policy gate, SARIF, policy-pack, HTML report, corpus benchmark, and exception lifecycle implementation evidence"
+    }
+  ],
+  "retrieval_questions": [
+    "Which AgentShield policy finding, category, severity, and affected file or MCP/hook surface triggered the request?",
+    "Is there SARIF/code-scanning evidence for an `agentshield-policy/*` result, and does it match the report finding?",
+    "Is the exception active, expiring soon, or expired?",
+    "Does the exception include owner, ticket, scope, expiry, and rationale fields?",
+    "Which policy pack or organization baseline produced the finding?",
+    "Is remediation possible now, or is a bounded exception safer than a blanket suppression?"
+  ],
+  "forbidden_actions": [
+    "approving policy exceptions without SARIF or report evidence",
+    "treating expired exceptions as active",
+    "blanket-suppressing AgentShield policy packs or organization-policy gates",
+    "downgrading critical/high findings without owner, ticket, scope, and expiry",
+    "editing AgentShield code or policy files from this ECC evaluator run",
+    "publishing or enforcing new security policy from this read-only evaluator run"
+  ],
+  "acceptance_gates": [
+    "SARIF or report evidence is named",
+    "finding id, category, severity, and affected surface are preserved",
+    "policy pack or organization baseline is named",
+    "owner, ticket, scope, and expiry state are recorded",
+    "expired exceptions stay rejected or enforced",
+    "remediation versus time-boxed exception decision is explicit",
+    "at least one blanket suppression candidate is rejected"
+  ]
+}

examples/evaluator-rag-prototype/agentshield-policy-exception/trace.json

@@ -0,0 +1,45 @@
+{
+  "schema_version": "ecc.evaluator-rag.trace.v1",
+  "scenario_id": "agentshield-policy-exception",
+  "run_id": "2026-05-12-agentshield-policy-exception-prototype",
+  "read_only": true,
+  "events": [
+    {
+      "phase": "observation",
+      "summary": "A policy finding or exception request references AgentShield organization-policy output. The evaluator records the affected finding without editing AgentShield code, policy packs, or enforcement settings.",
+      "evidence": [
+        "docs/ECC-2.0-GA-ROADMAP.md",
+        "commands/security-scan.md"
+      ]
+    },
+    {
+      "phase": "retrieval",
+      "summary": "Retrieved SARIF/report evidence, policy-pack source, exception lifecycle state, owner, ticket, scope, expiry, and whether remediation is immediately available.",
+      "evidence": [
+        "agentshield-policy/* SARIF result",
+        "AgentShield report exception counts",
+        "skills/security-scan/SKILL.md"
+      ]
+    },
+    {
+      "phase": "proposal",
+      "summary": "Generated two candidate playbooks: SARIF-backed time-boxed exception review, and blanket policy suppression for the affected category.",
+      "candidate_ids": [
+        "sarif-backed-timeboxed-exception-review",
+        "blanket-policy-suppression"
+      ]
+    },
+    {
+      "phase": "verification",
+      "summary": "Accepted the evidence-backed exception review because it preserves finding details and lifecycle fields. Rejected blanket suppression because it bypasses policy gates and ignores expired exceptions.",
+      "evidence": [
+        "examples/evaluator-rag-prototype/agentshield-policy-exception/verifier-result.json"
+      ]
+    },
+    {
+      "phase": "promotion",
+      "summary": "Promoted only the read-only AgentShield policy exception playbook. The evaluator does not modify AgentShield code, policy packs, enforcement settings, release state, or live security posture.",
+      "promoted_candidate_id": "sarif-backed-timeboxed-exception-review"
+    }
+  ]
+}

examples/evaluator-rag-prototype/agentshield-policy-exception/verifier-result.json

@@ -0,0 +1,35 @@
+{
+  "schema_version": "ecc.evaluator-rag.verifier.v1",
+  "scenario_id": "agentshield-policy-exception",
+  "run_id": "2026-05-12-agentshield-policy-exception-prototype",
+  "read_only": true,
+  "candidates": [
+    {
+      "candidate_id": "sarif-backed-timeboxed-exception-review",
+      "decision": "accepted",
+      "score": 0.93,
+      "reasons": [
+        "names SARIF/code-scanning or report evidence for the AgentShield finding",
+        "preserves finding id, category, severity, affected surface, and policy-pack source",
+        "records owner, ticket, scope, expiry, and active/expiring/expired lifecycle state",
+        "rejects expired exceptions and requires remediation or a time-boxed exception",
+        "keeps AgentShield code, policy packs, enforcement settings, and release actions out of the read-only evaluator run"
+      ],
+      "rollback": "Do not apply the future exception or suppression; re-run AgentShield, restore the prior organization policy, and keep the finding enforced until owner/ticket/scope/expiry evidence is current."
+    },
+    {
+      "candidate_id": "blanket-policy-suppression",
+      "decision": "rejected",
+      "score": 0.11,
+      "reasons": [
+        "has no SARIF or report evidence",
+        "blanket-suppresses AgentShield policy packs and organization-policy gates",
+        "treats expired exceptions as active",
+        "drops owner, ticket, scope, and expiry fields",
+        "would edit AgentShield or policy gate behavior from an ECC evaluator run"
+      ],
+      "rollback": "Do not suppress the policy category; restart from scanner evidence, lifecycle state, and a bounded remediation or exception request."
+    }
+  ],
+  "promoted_candidate_id": "sarif-backed-timeboxed-exception-review"
+}