docs: add evaluator rag prototype (#1824)

examples/evaluator-rag-prototype/candidate-playbook.md

@@ -0,0 +1,41 @@
+# Candidate Playbook: Maintainer-Owned Stale Salvage
+
+Candidate id: `maintainer-salvage-branch`
+
+## Use When
+
+- A stale or conflicted PR was closed to keep the public queue usable.
+- The closed diff contains a useful focused idea, skill, command, doc, test, or
+  bug fix.
+- The contributor may not have time or interest to rebase.
+
+## Steps
+
+1. Record the source PR, author, useful concept, and closure reason in
+   `docs/stale-pr-salvage-ledger.md`.
+2. Re-read the closed PR diff against current `main`.
+3. Decide whether the patch can be cherry-picked safely. Prefer reimplementation
+   when current architecture has moved.
+4. Create a maintainer-owned branch with one focused salvage unit.
+5. Preserve attribution in the PR body and, when useful, in the commit body.
+6. Update the catalog, docs, tests, or release evidence required by the touched
+   surface.
+7. Run the same validation gates a normal change would require.
+8. After merge, update the ledger from pending/salvage-branch to landed,
+   already-present, superseded, skipped, or translator/manual review.
+
+## Reject Conditions
+
+- The patch is bulk generated churn.
+- The patch is stale localization that needs translator/manual review.
+- The patch imports personal paths, secrets, local settings, or private operator context.
+- The patch bypasses current install, catalog, plugin, or release architecture.
+- The branch would mix unrelated salvage units into one PR.
+
+## Minimum Validation
+
+- Targeted test for the touched surface.
+- `git diff --check`.
+- Markdown lint when docs are touched.
+- Catalog/install validation when skills, agents, commands, or plugin surfaces
+  are touched.

examples/evaluator-rag-prototype/report.json

@@ -0,0 +1,35 @@
+{
+  "schema_version": "ecc.evaluator-rag.report.v1",
+  "scenario_id": "stale-pr-salvage-maintainer-branch",
+  "run_id": "2026-05-12-cleanup-salvage-prototype",
+  "result": "prototype_passed",
+  "read_only": true,
+  "scores": {
+    "source_attribution": 1,
+    "blast_radius_control": 1,
+    "manual_review_respected": 1,
+    "validation_specificity": 0.8,
+    "publication_safety": 1
+  },
+  "findings": [
+    {
+      "id": "salvage-policy-usable",
+      "severity": "info",
+      "summary": "The stale-salvage ledger and maintainer PR examples provide enough evidence to promote a reusable maintainer-owned salvage playbook."
+    },
+    {
+      "id": "translation-tail-blocked",
+      "severity": "warning",
+      "summary": "Localization tails remain useful but must stay translator/manual-review only."
+    },
+    {
+      "id": "release-actions-blocked",
+      "severity": "warning",
+      "summary": "Release, npm, plugin, billing, and announcement actions remain outside this evaluator run and require separate approval."
+    }
+  ],
+  "recommended_next_action": {
+    "candidate_id": "maintainer-salvage-branch",
+    "action": "Use the promoted playbook for future stale cleanup batches and add additional evaluator/RAG scenarios for CI failure diagnosis, harness-config drift, billing readiness, and AgentShield policy exceptions."
+  }
+}

examples/evaluator-rag-prototype/scenario.json

@@ -0,0 +1,56 @@
+{
+  "schema_version": "ecc.evaluator-rag.scenario.v1",
+  "scenario_id": "stale-pr-salvage-maintainer-branch",
+  "title": "Recover useful stale PR work through maintainer-owned branches",
+  "mode": "read_only_prototype",
+  "objective": "Given a closed stale PR batch, identify useful work, reject unsafe bulk imports, and promote only a maintainer-owned salvage playbook with attribution and validation.",
+  "sources": [
+    {
+      "kind": "repo_doc",
+      "path": "docs/stale-pr-salvage-ledger.md",
+      "purpose": "Durable source-to-disposition mapping for stale PR cleanup"
+    },
+    {
+      "kind": "repo_doc",
+      "path": "docs/legacy-artifact-inventory.md",
+      "purpose": "Import guardrails for legacy and private-context material"
+    },
+    {
+      "kind": "roadmap",
+      "path": "docs/ECC-2.0-GA-ROADMAP.md",
+      "purpose": "Operating rule and current execution lane"
+    },
+    {
+      "kind": "github_pr",
+      "url": "https://github.com/affaan-m/everything-claude-code/pull/1815",
+      "purpose": "Example maintainer-owned stale salvage PR with attribution"
+    },
+    {
+      "kind": "github_pr",
+      "url": "https://github.com/affaan-m/everything-claude-code/pull/1818",
+      "purpose": "Example gap pass classifying already-present and skipped stale work"
+    }
+  ],
+  "retrieval_questions": [
+    "Which closed PRs contain useful work that is not already present?",
+    "Which files or concepts are unsafe to cherry-pick without manual review?",
+    "Which current docs, skills, commands, or tests are the correct integration points?",
+    "Which validation gates are required before the salvage work can merge?"
+  ],
+  "forbidden_actions": [
+    "closing, reopening, or commenting on PRs",
+    "merging PRs",
+    "creating release tags",
+    "publishing packages or plugins",
+    "copying private paths, secrets, or raw personal context",
+    "blindly cherry-picking bulk localization"
+  ],
+  "acceptance_gates": [
+    "source attribution is preserved",
+    "salvage ledger or equivalent tracker is updated",
+    "translation/manual-review tails remain blocked",
+    "candidate action is reversible and maintainer-owned",
+    "validation commands are named",
+    "at least one unsafe candidate is rejected"
+  ]
+}

examples/evaluator-rag-prototype/trace.json

@@ -0,0 +1,46 @@
+{
+  "schema_version": "ecc.evaluator-rag.trace.v1",
+  "scenario_id": "stale-pr-salvage-maintainer-branch",
+  "run_id": "2026-05-12-cleanup-salvage-prototype",
+  "read_only": true,
+  "events": [
+    {
+      "phase": "observation",
+      "summary": "Public PR, issue, and discussion queues are clear; release publication remains approval-gated; stale-salvage ledger has landed, skipped, superseded, and manual-review states.",
+      "evidence": [
+        "docs/ECC-2.0-GA-ROADMAP.md",
+        "docs/stale-pr-salvage-ledger.md"
+      ]
+    },
+    {
+      "phase": "retrieval",
+      "summary": "Retrieved stale PR source mappings, existing maintainer salvage examples, legacy import rules, and manual-review localization tails.",
+      "evidence": [
+        "docs/stale-pr-salvage-ledger.md",
+        "docs/legacy-artifact-inventory.md",
+        "https://github.com/affaan-m/everything-claude-code/pull/1815",
+        "https://github.com/affaan-m/everything-claude-code/pull/1818"
+      ]
+    },
+    {
+      "phase": "proposal",
+      "summary": "Generated two candidate playbooks: maintainer-owned salvage branch with attribution, and blind cherry-pick of stale translations.",
+      "candidate_ids": [
+        "maintainer-salvage-branch",
+        "blind-cherry-pick-translations"
+      ]
+    },
+    {
+      "phase": "verification",
+      "summary": "Accepted the maintainer-owned salvage branch and rejected blind translation cherry-picking because it violates manual-review and attribution gates.",
+      "evidence": [
+        "examples/evaluator-rag-prototype/verifier-result.json"
+      ]
+    },
+    {
+      "phase": "promotion",
+      "summary": "Promoted only the maintainer-owned salvage branch playbook as a reusable process. No repository, GitHub, release, billing, or plugin publication action is performed by this prototype.",
+      "promoted_candidate_id": "maintainer-salvage-branch"
+    }
+  ]
+}

examples/evaluator-rag-prototype/verifier-result.json

@@ -0,0 +1,35 @@
+{
+  "schema_version": "ecc.evaluator-rag.verifier.v1",
+  "scenario_id": "stale-pr-salvage-maintainer-branch",
+  "run_id": "2026-05-12-cleanup-salvage-prototype",
+  "read_only": true,
+  "candidates": [
+    {
+      "candidate_id": "maintainer-salvage-branch",
+      "decision": "accepted",
+      "score": 0.94,
+      "reasons": [
+        "preserves source PR attribution",
+        "keeps work on a fresh maintainer-owned branch",
+        "updates the salvage ledger",
+        "names validation gates",
+        "does not perform release or publication actions"
+      ],
+      "rollback": "Close the maintainer PR or revert its merge commit; source PR state remains unchanged."
+    },
+    {
+      "candidate_id": "blind-cherry-pick-translations",
+      "decision": "rejected",
+      "score": 0.21,
+      "reasons": [
+        "bulk localization requires translator/manual review",
+        "does not preserve enough source attribution",
+        "could import stale generated docs",
+        "does not name validation gates",
+        "risks bypassing current catalog and install architecture"
+      ],
+      "rollback": "Do not create this branch; keep the localization tail in translator/manual-review state."
+    }
+  ],
+  "promoted_candidate_id": "maintainer-salvage-branch"
+}