mirror of
https://github.com/affaan-m/everything-claude-code.git
synced 2026-05-13 16:13:03 +08:00
docs: add evaluator billing readiness scenario (#1825)
This commit is contained in:
Affaan Mustafa
@@ -0,0 +1,41 @@
|
||||
# Billing Marketplace Readiness Playbook
|
||||
|
||||
Use this playbook when release copy or roadmap text mentions ECC Tools
|
||||
billing, Marketplace availability, account recovery, plans, seats,
|
||||
entitlements, or subscription state.
|
||||
|
||||
## Accepted Path
|
||||
|
||||
1. Start from `docs/releases/2.0.0-rc.1/publication-readiness.md`.
|
||||
2. Check the current repo and public listing surfaces:
|
||||
- `gh api repos/ECC-Tools/ECC-Tools`
|
||||
- `https://github.com/marketplace/ecc-tools`
|
||||
3. Classify every billing or Marketplace claim as:
|
||||
- `verified`
|
||||
- `blocked`
|
||||
- `remove-before-publication`
|
||||
4. Keep roadmap acceptance criteria separate from live product claims.
|
||||
5. Update release copy only after the evidence points to a live URL or command
|
||||
result.
|
||||
6. Leave tag creation, npm publish, plugin submission, marketplace edits,
|
||||
subscription changes, and announcement posting approval-gated.
|
||||
|
||||
## Rejected Path
|
||||
|
||||
Do not say billing is live because a roadmap item exists, a dry run passed, or a
|
||||
Marketplace URL is known. Roadmap intent and dry-run publication evidence are
|
||||
not a billing state.
|
||||
|
||||
Do not edit plan limits, subscriptions, seats, entitlements, or Marketplace
|
||||
metadata from the evaluator run. Those are product/operator actions and require
|
||||
their own approval path.
|
||||
|
||||
## Validation Gates
|
||||
|
||||
- `rg -n "billing|Billing|Marketplace|marketplace|subscription|seat|entitlement|plan" README.md docs/releases/2.0.0-rc.1 docs/ECC-2.0-GA-ROADMAP.md`
|
||||
- `gh api repos/ECC-Tools/ECC-Tools`
|
||||
- Manual live check of `https://github.com/marketplace/ecc-tools`
|
||||
- `npx --yes markdownlint-cli docs/releases/2.0.0-rc.1/*.md docs/ECC-2.0-GA-ROADMAP.md`
|
||||
- `git diff --check`
|
||||
|
||||
Record the evidence in a maintainer-owned PR before release copy is published.
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"schema_version": "ecc.evaluator-rag.report.v1",
|
||||
"scenario_id": "billing-marketplace-readiness",
|
||||
"run_id": "2026-05-12-billing-marketplace-readiness-prototype",
|
||||
"result": "prototype_passed",
|
||||
"read_only": true,
|
||||
"scores": {
|
||||
"claim_evidence": 0.82,
|
||||
"publication_safety": 1,
|
||||
"marketplace_specificity": 0.84,
|
||||
"billing_scope_control": 1,
|
||||
"announcement_safety": 1
|
||||
},
|
||||
"findings": [
|
||||
{
|
||||
"id": "billing-claim-gate-needed",
|
||||
"severity": "warning",
|
||||
"summary": "Release docs require a fresh ECC Tools billing/App/Marketplace check before launch copy can claim live billing readiness."
|
||||
},
|
||||
{
|
||||
"id": "dry-run-not-live-state",
|
||||
"severity": "warning",
|
||||
"summary": "May 12 evidence proves package/plugin dry runs and clean install smoke, but it does not prove a live Marketplace billing state."
|
||||
},
|
||||
{
|
||||
"id": "safe-next-action",
|
||||
"severity": "info",
|
||||
"summary": "The reusable next action is a read-only evidence checklist that classifies each launch-copy billing claim before publication."
|
||||
}
|
||||
],
|
||||
"recommended_next_action": {
|
||||
"candidate_id": "evidence-backed-billing-check",
|
||||
"action": "Run the promoted billing/Marketplace claim-verification checklist before any launch copy, GitHub release text, or social copy says billing is live."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"schema_version": "ecc.evaluator-rag.scenario.v1",
|
||||
"scenario_id": "billing-marketplace-readiness",
|
||||
"title": "Verify billing and Marketplace claims before launch copy",
|
||||
"mode": "read_only_prototype",
|
||||
"objective": "Given rc.1 release docs and ECC Tools billing roadmap evidence, separate verified Marketplace/App/billing state from assumptions before any announcement or publication action.",
|
||||
"sources": [
|
||||
{
|
||||
"kind": "repo_doc",
|
||||
"path": "docs/releases/2.0.0-rc.1/publication-readiness.md",
|
||||
"purpose": "Release gate that blocks billing and Marketplace claims until fresh evidence exists"
|
||||
},
|
||||
{
|
||||
"kind": "repo_doc",
|
||||
"path": "docs/releases/2.0.0-rc.1/publication-evidence-2026-05-12.md",
|
||||
"purpose": "Dry-run publication evidence and explicit remaining blocker list"
|
||||
},
|
||||
{
|
||||
"kind": "roadmap",
|
||||
"path": "docs/ECC-2.0-GA-ROADMAP.md",
|
||||
"purpose": "ECC Tools billing audit acceptance criteria and remaining release blockers"
|
||||
},
|
||||
{
|
||||
"kind": "github_api",
|
||||
"command": "gh api repos/ECC-Tools/ECC-Tools",
|
||||
"purpose": "Fresh repository access and app-surface evidence before launch claims"
|
||||
},
|
||||
{
|
||||
"kind": "public_url",
|
||||
"url": "https://github.com/marketplace/ecc-tools",
|
||||
"purpose": "Marketplace listing that must be checked live before copy says billing is ready"
|
||||
}
|
||||
],
|
||||
"retrieval_questions": [
|
||||
"Which billing or Marketplace claims are already backed by repo evidence?",
|
||||
"Which claims still need a live Marketplace, App, subscription, plan, or entitlement check?",
|
||||
"Which announcement docs mention billing or Marketplace status?",
|
||||
"Which publication actions remain approval-gated and must not run during this evaluator pass?"
|
||||
],
|
||||
"forbidden_actions": [
|
||||
"creating or editing GitHub Marketplace listings",
|
||||
"changing plan limits, subscriptions, seats, or entitlements",
|
||||
"creating release tags",
|
||||
"publishing packages or plugins",
|
||||
"posting announcement copy",
|
||||
"claiming live billing readiness from dry-run evidence alone"
|
||||
],
|
||||
"acceptance_gates": [
|
||||
"launch-copy claims are classified as verified, blocked, or remove-before-publication",
|
||||
"Marketplace and App checks name the exact URL or command needed",
|
||||
"billing claims link to fresh evidence rather than roadmap intent",
|
||||
"publication actions remain approval-gated",
|
||||
"at least one overclaim candidate is rejected"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"schema_version": "ecc.evaluator-rag.trace.v1",
|
||||
"scenario_id": "billing-marketplace-readiness",
|
||||
"run_id": "2026-05-12-billing-marketplace-readiness-prototype",
|
||||
"read_only": true,
|
||||
"events": [
|
||||
{
|
||||
"phase": "observation",
|
||||
"summary": "Publication readiness still marks ECC Tools billing references and announcement copy as pending. Dry-run publication evidence says billing/App/Marketplace claims must be verified before launch copy uses them.",
|
||||
"evidence": [
|
||||
"docs/releases/2.0.0-rc.1/publication-readiness.md",
|
||||
"docs/releases/2.0.0-rc.1/publication-evidence-2026-05-12.md"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "retrieval",
|
||||
"summary": "Retrieved the release gate, dry-run evidence, roadmap billing acceptance criteria, and the public Marketplace URL that requires a live operator check.",
|
||||
"evidence": [
|
||||
"docs/ECC-2.0-GA-ROADMAP.md",
|
||||
"gh api repos/ECC-Tools/ECC-Tools",
|
||||
"https://github.com/marketplace/ecc-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "proposal",
|
||||
"summary": "Generated two candidate playbooks: evidence-backed billing claim verification, and announcement-first billing copy that treats roadmap intent as live billing readiness.",
|
||||
"candidate_ids": [
|
||||
"evidence-backed-billing-check",
|
||||
"announcement-first-billing-copy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "verification",
|
||||
"summary": "Accepted the evidence-backed check and rejected announcement-first copy because billing and Marketplace surfaces remain pending until verified by fresh URLs or API output.",
|
||||
"evidence": [
|
||||
"examples/evaluator-rag-prototype/billing-marketplace-readiness/verifier-result.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "promotion",
|
||||
"summary": "Promoted only the read-only verification playbook. No Marketplace edits, subscription changes, tags, package publishes, plugin submission, or announcement posts are performed.",
|
||||
"promoted_candidate_id": "evidence-backed-billing-check"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"schema_version": "ecc.evaluator-rag.verifier.v1",
|
||||
"scenario_id": "billing-marketplace-readiness",
|
||||
"run_id": "2026-05-12-billing-marketplace-readiness-prototype",
|
||||
"read_only": true,
|
||||
"candidates": [
|
||||
{
|
||||
"candidate_id": "evidence-backed-billing-check",
|
||||
"decision": "accepted",
|
||||
"score": 0.91,
|
||||
"reasons": [
|
||||
"keeps the run read-only",
|
||||
"requires fresh Marketplace or GitHub API evidence",
|
||||
"classifies launch-copy claims before publication",
|
||||
"separates roadmap intent from live billing state",
|
||||
"keeps release, package, plugin, billing, and announcement actions approval-gated"
|
||||
],
|
||||
"rollback": "Remove or revert any release-copy edits that cite unverified billing claims; no live billing state is changed by this playbook."
|
||||
},
|
||||
{
|
||||
"candidate_id": "announcement-first-billing-copy",
|
||||
"decision": "rejected",
|
||||
"score": 0.18,
|
||||
"reasons": [
|
||||
"treats roadmap acceptance criteria as live billing evidence",
|
||||
"does not require a fresh Marketplace listing check",
|
||||
"could publish announcement copy before release URLs exist",
|
||||
"does not classify unsupported claims for removal",
|
||||
"risks implying subscription or entitlement readiness without proof"
|
||||
],
|
||||
"rollback": "Do not publish this copy; keep billing and Marketplace language blocked until the evidence checklist passes."
|
||||
}
|
||||
],
|
||||
"promoted_candidate_id": "evidence-backed-billing-check"
|
||||
}
|
||||
Reference in New Issue
Block a user