From af9b2c1c4ce6acec0ab341cb7174f15a348d9b8e Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Tue, 19 May 2026 06:20:54 -0400 Subject: [PATCH] feat: extend harness audit integration scoring (#1990) Salvages the useful harness-audit scoring work from #1989 while preserving the current hook registry and newer plugin install detection. Adds GitHub integration checks, conditional deploy-provider categories, dynamic applicable category metadata, and CODEOWNERS coverage. --- .github/CODEOWNERS | 1 + .opencode/commands/harness-audit.md | 27 ++-- commands/harness-audit.md | 27 ++-- scripts/harness-audit.js | 215 +++++++++++++++++++++++++++- tests/scripts/harness-audit.test.js | 194 ++++++++++++++++++++++++- 5 files changed, 441 insertions(+), 23 deletions(-) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..5b92bc5f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @affaan-m diff --git a/.opencode/commands/harness-audit.md b/.opencode/commands/harness-audit.md index 108042ce..fc36601a 100644 --- a/.opencode/commands/harness-audit.md +++ b/.opencode/commands/harness-audit.md @@ -24,9 +24,9 @@ node scripts/harness-audit.js --format [--root ] This script is the source of truth for scoring and checks. Do not invent additional dimensions or ad-hoc points. -Rubric version: `2026-03-30`. +Rubric version: `2026-05-19`. -The script computes 7 fixed categories (`0-10` normalized each): +The script computes up to 12 fixed categories (`0-10` normalized each). The first seven are always applicable; GitHub Integration is always applicable; deploy-target categories are applicable only when a matching marker is detected. 1. Tool Coverage 2. Context Efficiency @@ -35,6 +35,11 @@ The script computes 7 fixed categories (`0-10` normalized each): 5. Eval Coverage 6. Security Guardrails 7. Cost Efficiency +8. GitHub Integration +9. Vercel Integration *(when `vercel.json` or `.vercel/` is present)* +10. Netlify Integration *(when `netlify.toml` or `.netlify/` is present)* +11. Cloudflare Integration *(when `wrangler.toml` or `wrangler.jsonc` is present)* +12. Fly Integration *(when `fly.toml` is present)* Scores are derived from explicit file/rule checks and are reproducible for the same commit. The script audits the current working directory by default and auto-detects whether the target is the ECC repo itself or a consumer project using ECC. @@ -43,11 +48,12 @@ The script audits the current working directory by default and auto-detects whet Return: -1. `overall_score` out of `max_score` (70 for `repo`; smaller for scoped audits) -2. Category scores and concrete findings -3. Failed checks with exact file paths -4. Top 3 actions from the deterministic output (`top_actions`) -5. Suggested ECC skills to apply next +1. `overall_score` out of `max_score`. `max_score` depends on which categories are applicable to the target; never assume a fixed total. +2. `applicable_categories[]` and `category_count` describing which categories contributed. +3. Category scores and concrete findings. +4. Failed checks with exact file paths. +5. Top 3 actions from the deterministic output (`top_actions`). +6. Suggested ECC skills to apply next. ## Checklist @@ -59,14 +65,15 @@ Return: ## Example Result ```text -Harness Audit (repo): 66/70 +Harness Audit (repo, repo): 71/80 - Tool Coverage: 10/10 (10/10 pts) - Context Efficiency: 9/10 (9/10 pts) - Quality Gates: 10/10 (10/10 pts) +- GitHub Integration: 2/10 (2/10 pts) Top 3 Actions: -1) [Security Guardrails] Add prompt/tool preflight security guards in hooks/hooks.json. (hooks/hooks.json) -2) [Tool Coverage] Sync commands/harness-audit.md and .opencode/commands/harness-audit.md. (.opencode/commands/harness-audit.md) +1) [GitHub Integration] Add at least one workflow under .github/workflows/. (.github/workflows/) +2) [Security Guardrails] Add prompt/tool preflight security guards in hooks/hooks.json. (hooks/hooks.json) 3) [Eval Coverage] Increase automated test coverage across scripts/hooks/lib. (tests/) ``` diff --git a/commands/harness-audit.md b/commands/harness-audit.md index 108042ce..fc36601a 100644 --- a/commands/harness-audit.md +++ b/commands/harness-audit.md @@ -24,9 +24,9 @@ node scripts/harness-audit.js --format [--root ] This script is the source of truth for scoring and checks. Do not invent additional dimensions or ad-hoc points. -Rubric version: `2026-03-30`. +Rubric version: `2026-05-19`. -The script computes 7 fixed categories (`0-10` normalized each): +The script computes up to 12 fixed categories (`0-10` normalized each). The first seven are always applicable; GitHub Integration is always applicable; deploy-target categories are applicable only when a matching marker is detected. 1. Tool Coverage 2. Context Efficiency @@ -35,6 +35,11 @@ The script computes 7 fixed categories (`0-10` normalized each): 5. Eval Coverage 6. Security Guardrails 7. Cost Efficiency +8. GitHub Integration +9. Vercel Integration *(when `vercel.json` or `.vercel/` is present)* +10. Netlify Integration *(when `netlify.toml` or `.netlify/` is present)* +11. Cloudflare Integration *(when `wrangler.toml` or `wrangler.jsonc` is present)* +12. Fly Integration *(when `fly.toml` is present)* Scores are derived from explicit file/rule checks and are reproducible for the same commit. The script audits the current working directory by default and auto-detects whether the target is the ECC repo itself or a consumer project using ECC. @@ -43,11 +48,12 @@ The script audits the current working directory by default and auto-detects whet Return: -1. `overall_score` out of `max_score` (70 for `repo`; smaller for scoped audits) -2. Category scores and concrete findings -3. Failed checks with exact file paths -4. Top 3 actions from the deterministic output (`top_actions`) -5. Suggested ECC skills to apply next +1. `overall_score` out of `max_score`. `max_score` depends on which categories are applicable to the target; never assume a fixed total. +2. `applicable_categories[]` and `category_count` describing which categories contributed. +3. Category scores and concrete findings. +4. Failed checks with exact file paths. +5. Top 3 actions from the deterministic output (`top_actions`). +6. Suggested ECC skills to apply next. ## Checklist @@ -59,14 +65,15 @@ Return: ## Example Result ```text -Harness Audit (repo): 66/70 +Harness Audit (repo, repo): 71/80 - Tool Coverage: 10/10 (10/10 pts) - Context Efficiency: 9/10 (9/10 pts) - Quality Gates: 10/10 (10/10 pts) +- GitHub Integration: 2/10 (2/10 pts) Top 3 Actions: -1) [Security Guardrails] Add prompt/tool preflight security guards in hooks/hooks.json. (hooks/hooks.json) -2) [Tool Coverage] Sync commands/harness-audit.md and .opencode/commands/harness-audit.md. (.opencode/commands/harness-audit.md) +1) [GitHub Integration] Add at least one workflow under .github/workflows/. (.github/workflows/) +2) [Security Guardrails] Add prompt/tool preflight security guards in hooks/hooks.json. (hooks/hooks.json) 3) [Eval Coverage] Increase automated test coverage across scripts/hooks/lib. (tests/) ``` diff --git a/scripts/harness-audit.js b/scripts/harness-audit.js index 79bc57c8..3e525ca7 100644 --- a/scripts/harness-audit.js +++ b/scripts/harness-audit.js @@ -12,8 +12,53 @@ const CATEGORIES = [ 'Eval Coverage', 'Security Guardrails', 'Cost Efficiency', + 'GitHub Integration', + 'Vercel Integration', + 'Netlify Integration', + 'Cloudflare Integration', + 'Fly Integration', ]; +const RUBRIC_VERSION = '2026-05-19'; + +const PROVIDERS = { + Vercel: { + detect: (rootDir) => + fileExists(rootDir, 'vercel.json') || + fileExists(rootDir, '.vercel/project.json') || + fileExists(rootDir, '.vercel'), + keyPattern: /vercel/i, + buildPattern: /vercel/i, + workflowPattern: /(vercel-action|vercel\s+(deploy|--prod))/i, + }, + Netlify: { + detect: (rootDir) => + fileExists(rootDir, 'netlify.toml') || fileExists(rootDir, '.netlify'), + keyPattern: /netlify/i, + buildPattern: /netlify/i, + workflowPattern: /(netlify\/actions|netlify\s+deploy)/i, + }, + Cloudflare: { + detect: (rootDir) => + fileExists(rootDir, 'wrangler.toml') || fileExists(rootDir, 'wrangler.jsonc'), + keyPattern: /\b(cloudflare|wrangler)\b/i, + buildPattern: /(wrangler|cloudflare)/i, + workflowPattern: /(cloudflare\/wrangler-action|wrangler\s+(deploy|publish))/i, + }, + Fly: { + detect: (rootDir) => fileExists(rootDir, 'fly.toml'), + keyPattern: /fly[_-]?(api|io)/i, + buildPattern: /fly\s+(deploy|launch)/i, + workflowPattern: /(superfly\/flyctl-actions|flyctl\s+deploy|fly\s+deploy)/i, + }, +}; + +function getApplicableProviders(rootDir) { + return Object.entries(PROVIDERS) + .filter(([_, spec]) => spec.detect(rootDir)) + .map(([name]) => name); +} + function normalizeScope(scope) { const value = (scope || 'repo').toLowerCase(); if (!['repo', 'hooks', 'skills', 'commands', 'agents'].includes(value)) { @@ -607,9 +652,172 @@ function getRepoChecks(rootDir) { pass: fileExists(rootDir, 'commands/model-route.md'), fix: 'Add commands/model-route.md and route policies for cheap-default execution.', }, + ...buildGithubChecks(rootDir), ]; } +// GitHub Integration is intentionally repo-scoped. Scoped audits such as hooks, +// skills, commands, and agents should keep reporting only that surface. +function buildGithubChecks(rootDir) { + return [ + { + id: 'github-workflows', + category: 'GitHub Integration', + points: 3, + scopes: ['repo'], + path: '.github/workflows/', + description: 'GitHub Actions workflows are checked in', + pass: hasFileWithExtension(rootDir, '.github/workflows', ['.yml', '.yaml']), + fix: 'Add at least one workflow under .github/workflows/ so CI runs on every PR.', + }, + { + id: 'github-pr-template', + category: 'GitHub Integration', + points: 2, + scopes: ['repo'], + path: '.github/PULL_REQUEST_TEMPLATE.md', + description: 'A pull request template is configured', + pass: + fileExists(rootDir, '.github/PULL_REQUEST_TEMPLATE.md') || + fileExists(rootDir, '.github/pull_request_template.md'), + fix: 'Add .github/PULL_REQUEST_TEMPLATE.md so PR descriptions follow a consistent shape.', + }, + { + id: 'github-issue-templates', + category: 'GitHub Integration', + points: 2, + scopes: ['repo'], + path: '.github/ISSUE_TEMPLATE/', + description: 'Issue templates are configured', + pass: hasFileWithExtension(rootDir, '.github/ISSUE_TEMPLATE', ['.md', '.yml', '.yaml']), + fix: 'Add at least one issue template under .github/ISSUE_TEMPLATE/.', + }, + { + id: 'github-codeowners', + category: 'GitHub Integration', + points: 1, + scopes: ['repo'], + path: '.github/CODEOWNERS', + description: 'A CODEOWNERS file routes reviews', + pass: + fileExists(rootDir, 'CODEOWNERS') || + fileExists(rootDir, '.github/CODEOWNERS') || + fileExists(rootDir, 'docs/CODEOWNERS'), + fix: 'Add a CODEOWNERS file so PRs auto-request the right reviewers.', + }, + { + id: 'github-dep-updates', + category: 'GitHub Integration', + points: 2, + scopes: ['repo'], + path: '.github/dependabot.yml', + description: 'Automated dependency updates are configured', + pass: + fileExists(rootDir, '.github/dependabot.yml') || + fileExists(rootDir, '.github/dependabot.yaml') || + fileExists(rootDir, 'renovate.json') || + fileExists(rootDir, '.github/renovate.json') || + fileExists(rootDir, '.renovaterc'), + fix: 'Add a Dependabot or Renovate config so dependency updates land automatically.', + }, + ]; +} + +function readAllWorkflowsText(rootDir) { + const dir = path.join(rootDir, '.github/workflows'); + if (!fs.existsSync(dir)) { + return ''; + } + + const stack = [dir]; + let combined = ''; + + while (stack.length > 0) { + const current = stack.pop(); + const entries = fs.readdirSync(current, { withFileTypes: true }); + + for (const entry of entries) { + const nextPath = path.join(current, entry.name); + if (entry.isDirectory()) { + stack.push(nextPath); + } else if (entry.name.endsWith('.yml') || entry.name.endsWith('.yaml')) { + try { + combined += `${fs.readFileSync(nextPath, 'utf8')}\n`; + } catch (_error) { + // Ignore unreadable workflow files; the finding should stay deterministic. + } + } + } + } + + return combined; +} + +function buildProviderChecks(rootDir, provider, sharedContext) { + const spec = PROVIDERS[provider]; + const packageJson = sharedContext.packageJson || {}; + const scriptsText = Object.values(packageJson.scripts || {}).join('\n'); + const category = `${provider} Integration`; + + return [ + { + id: `${provider.toLowerCase()}-config`, + category, + points: 3, + scopes: ['repo'], + path: `${provider} config`, + description: `${provider} deployment config is checked in`, + pass: spec.detect(rootDir), + fix: `Commit ${provider} configuration so deploys are reproducible from source.`, + }, + { + id: `${provider.toLowerCase()}-build-script`, + category, + points: 2, + scopes: ['repo'], + path: 'package.json scripts', + description: `package.json scripts reference ${provider}`, + pass: spec.buildPattern.test(scriptsText), + fix: `Add a build or deploy script in package.json that runs ${provider}.`, + }, + { + id: `${provider.toLowerCase()}-env-doc`, + category, + points: 2, + scopes: ['repo'], + path: '.env.example', + description: `${provider} env keys are documented in .env.example`, + pass: spec.keyPattern.test(sharedContext.envExample), + fix: `Document ${provider} environment variables in .env.example.`, + }, + { + id: `${provider.toLowerCase()}-workflow-uses`, + category, + points: 3, + scopes: ['repo'], + path: '.github/workflows/', + description: `A GitHub workflow uses the ${provider} action or CLI`, + pass: spec.workflowPattern.test(sharedContext.workflowsText), + fix: `Reference the ${provider} action or CLI from a workflow under .github/workflows/.`, + }, + ]; +} + +function collectProviderChecks(rootDir, packageJson) { + const providers = getApplicableProviders(rootDir); + if (providers.length === 0) { + return []; + } + + const sharedContext = { + packageJson: packageJson || {}, + envExample: `${safeRead(rootDir, '.env.example')}\n${safeRead(rootDir, '.env.sample')}`, + workflowsText: readAllWorkflowsText(rootDir), + }; + + return providers.flatMap(provider => buildProviderChecks(rootDir, provider, sharedContext)); +} + function getConsumerChecks(rootDir) { const packageJson = safeParseJson(safeRead(rootDir, 'package.json')); const gitignore = safeRead(rootDir, '.gitignore'); @@ -731,6 +939,8 @@ function getConsumerChecks(rootDir) { pass: projectHooks.includes('PreToolUse') || projectHooks.includes('beforeSubmitPrompt') || fileExists(rootDir, '.claude/hooks.json'), fix: 'Add project-local hook settings or hook definitions for prompt/tool guardrails.', }, + ...buildGithubChecks(rootDir), + ...collectProviderChecks(rootDir, packageJson), ]; } @@ -764,6 +974,7 @@ function buildReport(scope, options = {}) { const overallScore = checks .filter(check => check.pass) .reduce((sum, check) => sum + check.points, 0); + const applicableCategories = CATEGORIES.filter(name => categoryScores[name]?.max > 0); const failedChecks = checks.filter(check => !check.pass); const topActions = failedChecks @@ -781,10 +992,12 @@ function buildReport(scope, options = {}) { root_dir: rootDir, target_mode: targetMode, deterministic: true, - rubric_version: '2026-03-30', + rubric_version: RUBRIC_VERSION, overall_score: overallScore, max_score: maxScore, categories: categoryScores, + applicable_categories: applicableCategories, + category_count: applicableCategories.length, checks: checks.map(check => ({ id: check.id, category: check.category, diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js index 685200e3..c0087475 100644 --- a/tests/scripts/harness-audit.test.js +++ b/tests/scripts/harness-audit.test.js @@ -126,7 +126,7 @@ function runTests() { const parsed = JSON.parse(run(['repo', '--format', 'json'])); assert.strictEqual(parsed.deterministic, true); - assert.strictEqual(parsed.rubric_version, '2026-03-30'); + assert.strictEqual(parsed.rubric_version, '2026-05-19'); assert.strictEqual(parsed.target_mode, 'repo'); assert.ok(parsed.overall_score >= 0); assert.ok(parsed.max_score > 0); @@ -140,6 +140,192 @@ function runTests() { assert.ok(categoryNames.includes('Eval Coverage')); assert.ok(categoryNames.includes('Security Guardrails')); assert.ok(categoryNames.includes('Cost Efficiency')); + assert.ok(categoryNames.includes('GitHub Integration')); + })) passed++; else failed++; + + if (test('report exposes applicable_categories and category_count', () => { + const parsed = JSON.parse(run(['repo', '--format', 'json'])); + + assert.ok(Array.isArray(parsed.applicable_categories), 'applicable_categories must be an array'); + assert.ok(parsed.applicable_categories.length > 0); + assert.strictEqual(parsed.category_count, parsed.applicable_categories.length); + for (const name of parsed.applicable_categories) { + assert.ok(parsed.categories[name].max > 0, `${name} must have max > 0 to be applicable`); + } + })) passed++; else failed++; + + if (test('GitHub Integration category scores against a fully-wired consumer fixture', () => { + const homeDir = createTempDir('harness-audit-home-gh-'); + const projectRoot = createTempDir('harness-audit-project-gh-'); + + try { + fs.mkdirSync(path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + + fs.mkdirSync(path.join(projectRoot, '.github', 'workflows'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.github', 'ISSUE_TEMPLATE'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, '.github', 'workflows', 'ci.yml'), 'name: ci\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'PULL_REQUEST_TEMPLATE.md'), '# PR\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'ISSUE_TEMPLATE', 'bug.md'), '# Bug\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'CODEOWNERS'), '* @owner\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'dependabot.yml'), 'version: 2\n'); + fs.writeFileSync(path.join(projectRoot, 'package.json'), JSON.stringify({ name: 'gh-test' })); + + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + const github = parsed.categories['GitHub Integration']; + + assert.ok(github, 'GitHub Integration category must exist'); + assert.strictEqual(github.score, 10, `GitHub Integration should score 10/10, got ${github.score}`); + assert.strictEqual(github.earned, github.max); + assert.ok(parsed.applicable_categories.includes('GitHub Integration')); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('provider categories are omitted unless a marker is present', () => { + const homeDir = createTempDir('harness-audit-home-no-provider-'); + const projectRoot = createTempDir('harness-audit-project-no-provider-'); + + try { + fs.mkdirSync(path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + fs.writeFileSync(path.join(projectRoot, 'package.json'), JSON.stringify({ name: 'p' })); + + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + + assert.ok(!parsed.applicable_categories.includes('Vercel Integration')); + const vercel = parsed.categories['Vercel Integration']; + assert.ok(!vercel || vercel.max === 0, 'Vercel Integration should not contribute when no marker'); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('Vercel Integration category scores when vercel.json present', () => { + const homeDir = createTempDir('harness-audit-home-vercel-'); + const projectRoot = createTempDir('harness-audit-project-vercel-'); + + try { + fs.mkdirSync(path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + + fs.mkdirSync(path.join(projectRoot, '.github', 'workflows'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, 'vercel.json'), '{}\n'); + fs.writeFileSync(path.join(projectRoot, '.env.example'), 'VERCEL_TOKEN=\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'workflows', 'deploy.yml'), 'uses: amondnet/vercel-action@v25\n'); + fs.writeFileSync( + path.join(projectRoot, 'package.json'), + JSON.stringify({ name: 'p', scripts: { build: 'next build', deploy: 'vercel deploy' } }) + ); + + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + const vercel = parsed.categories['Vercel Integration']; + + assert.ok(vercel, 'Vercel Integration category must exist when vercel.json present'); + assert.ok(vercel.max > 0); + assert.ok(parsed.applicable_categories.includes('Vercel Integration')); + assert.strictEqual(vercel.score, 10, `Vercel should score 10/10 with full wiring, got ${vercel.score}`); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('detector map: Netlify, Cloudflare, Fly each trigger their category', () => { + const homeDir = createTempDir('harness-audit-home-multi-'); + + function probe(markerFile, markerContents, expectedCategory) { + const root = createTempDir('harness-audit-project-multi-'); + try { + fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ name: 'p' })); + fs.writeFileSync(path.join(root, markerFile), markerContents); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: root, homeDir })); + assert.ok( + parsed.applicable_categories.includes(expectedCategory), + `${markerFile} should activate ${expectedCategory}` + ); + } finally { + cleanup(root); + } + } + + try { + fs.mkdirSync(path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + + probe('netlify.toml', '[build]\n', 'Netlify Integration'); + probe('wrangler.toml', 'name = "p"\n', 'Cloudflare Integration'); + probe('fly.toml', 'app = "p"\n', 'Fly Integration'); + } finally { + cleanup(homeDir); + } + })) passed++; else failed++; + + if (test('max_score reflects only applicable categories', () => { + const homeDir = createTempDir('harness-audit-home-max-'); + const noVercel = createTempDir('harness-audit-project-max-novercel-'); + const withVercel = createTempDir('harness-audit-project-max-vercel-'); + + try { + fs.mkdirSync(path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + + fs.writeFileSync(path.join(noVercel, 'package.json'), JSON.stringify({ name: 'p' })); + fs.writeFileSync(path.join(withVercel, 'package.json'), JSON.stringify({ name: 'p' })); + fs.writeFileSync(path.join(withVercel, 'vercel.json'), '{}\n'); + + const noVercelParsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: noVercel, homeDir })); + const withVercelParsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: withVercel, homeDir })); + + assert.ok( + withVercelParsed.max_score > noVercelParsed.max_score, + `with-vercel max_score (${withVercelParsed.max_score}) should exceed no-vercel (${noVercelParsed.max_score})` + ); + } finally { + cleanup(homeDir); + cleanup(noVercel); + cleanup(withVercel); + } + })) passed++; else failed++; + + if (test('non-git directory does not crash the script', () => { + const homeDir = createTempDir('harness-audit-home-bare-'); + const bare = createTempDir('harness-audit-project-bare-'); + + try { + fs.mkdirSync(path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin'), { recursive: true }); + fs.writeFileSync( + path.join(homeDir, '.claude', 'plugins', 'ecc', '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'ecc' }, null, 2) + ); + fs.writeFileSync(path.join(bare, 'package.json'), JSON.stringify({ name: 'p' })); + + const output = run(['repo', '--format', 'json'], { cwd: bare, homeDir }); + const parsed = JSON.parse(output); + assert.ok(parsed.overall_score >= 0); + assert.ok(parsed.max_score > 0); + } finally { + cleanup(homeDir); + cleanup(bare); + } })) passed++; else failed++; if (test('scope filtering changes max score and check list', () => { @@ -251,6 +437,7 @@ function runTests() { ); fs.mkdirSync(path.join(projectRoot, '.claude'), { recursive: true }); fs.mkdirSync(path.join(projectRoot, '.github', 'workflows', 'nested'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.github', 'ISSUE_TEMPLATE'), { recursive: true }); fs.mkdirSync(path.join(projectRoot, 'docs', 'adr'), { recursive: true }); fs.mkdirSync(path.join(projectRoot, 'evals'), { recursive: true }); fs.mkdirSync(path.join(projectRoot, 'src'), { recursive: true }); @@ -259,6 +446,9 @@ function runTests() { fs.writeFileSync(path.join(projectRoot, 'CLAUDE.md'), '# Consumer instructions\n'); fs.writeFileSync(path.join(projectRoot, 'src', 'app.spec.ts'), 'test placeholder\n'); fs.writeFileSync(path.join(projectRoot, '.github', 'workflows', 'nested', 'ci.yaml'), 'name: ci\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'PULL_REQUEST_TEMPLATE.md'), '# PR\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'ISSUE_TEMPLATE', 'bug.md'), '# Bug\n'); + fs.writeFileSync(path.join(projectRoot, '.github', 'CODEOWNERS'), '* @owner\n'); fs.writeFileSync(path.join(projectRoot, 'docs', 'adr', '001.md'), '# Record\n'); fs.writeFileSync(path.join(projectRoot, 'evals', 'smoke.json'), '{}\n'); fs.writeFileSync(path.join(projectRoot, '.github', 'dependabot.yml'), 'version: 2\n'); @@ -274,7 +464,7 @@ function runTests() { const text = run(['repo'], { cwd: projectRoot, homeDir }); assert.ok(text.includes(`Harness Audit (repo, consumer): ${parsed.max_score}/${parsed.max_score}`)); - assert.ok(text.includes('Checks: 11 total, 0 failing')); + assert.ok(text.includes('Checks: 16 total, 0 failing')); assert.ok(!text.includes('Top 3 Actions:')); const scopedText = run(['agents'], { cwd: projectRoot, homeDir });