docs: add data-backed harness adapter scorecard (#1785)

* docs: add data-backed harness adapter scorecard

* fix: normalize adapter matrix line endings

* test: avoid doubled CRLF simulation
This commit is contained in:
Affaan Mustafa
2026-05-12 02:59:52 -04:00
committed by GitHub
parent 969acd9078
commit cdf1b03779
7 changed files with 688 additions and 25 deletions

View File

@@ -1,10 +1,18 @@
'use strict';
const assert = require('assert');
const { execFileSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const {
ADAPTER_RECORDS,
extractMatrixBlock,
renderMarkdownTable,
validateAdapterRecords,
} = require('../../scripts/lib/harness-adapter-compliance');
const repoRoot = path.resolve(__dirname, '..', '..');
const scriptPath = path.join(repoRoot, 'scripts', 'harness-adapter-compliance.js');
let passed = 0;
let failed = 0;
@@ -46,6 +54,29 @@ test('adapter compliance matrix covers the required harness surfaces', () => {
}
});
test('adapter compliance source data validates required evidence fields', () => {
assert.deepStrictEqual(validateAdapterRecords(), []);
for (const record of ADAPTER_RECORDS) {
assert.ok(record.install_or_onramp.length > 0, `${record.id} needs an install or onramp`);
assert.ok(record.verification_commands.length > 0, `${record.id} needs verification commands`);
assert.ok(record.risk_notes.length > 0, `${record.id} needs risk notes`);
assert.ok(record.source_docs.length > 0, `${record.id} needs source docs`);
}
});
test('adapter compliance matrix is generated from source data', () => {
const source = read('docs/architecture/harness-adapter-compliance.md');
assert.strictEqual(extractMatrixBlock(source), renderMarkdownTable());
});
test('adapter compliance matrix extraction tolerates Windows line endings', () => {
const source = read('docs/architecture/harness-adapter-compliance.md')
.replace(/\r\n/g, '\n')
.replace(/\n/g, '\r\n');
assert.strictEqual(extractMatrixBlock(source), renderMarkdownTable());
});
test('adapter compliance matrix includes the required evidence columns', () => {
const source = read('docs/architecture/harness-adapter-compliance.md');
for (const heading of [
@@ -62,6 +93,7 @@ test('adapter compliance matrix includes the required evidence columns', () => {
test('scorecard onramp names the local verification commands', () => {
const source = read('docs/architecture/harness-adapter-compliance.md');
for (const command of [
'npm run harness:adapters -- --check',
'npm run harness:audit -- --format json',
'npm run observability:ready',
'node scripts/session-inspect.js --list-adapters',
@@ -71,15 +103,39 @@ test('scorecard onramp names the local verification commands', () => {
}
});
test('adapter compliance CLI check passes against the committed doc', () => {
const output = execFileSync('node', [scriptPath, '--check'], {
cwd: repoRoot,
encoding: 'utf8',
});
assert.ok(output.includes('Harness Adapter Compliance: PASS'));
assert.ok(output.includes(`Adapters: ${ADAPTER_RECORDS.length}`));
});
test('adapter compliance CLI emits machine-readable scorecard data', () => {
const output = execFileSync('node', [scriptPath, '--format=json'], {
cwd: repoRoot,
encoding: 'utf8',
});
const parsed = JSON.parse(output);
assert.strictEqual(parsed.schema_version, 'ecc.harness-adapter-compliance.v1');
assert.strictEqual(parsed.valid, true);
assert.strictEqual(parsed.adapter_count, ADAPTER_RECORDS.length);
assert.ok(parsed.adapters.some(record => record.id === 'terminal-only'));
});
test('cross-harness architecture links to the adapter compliance matrix', () => {
const source = read('docs/architecture/cross-harness.md');
assert.ok(source.includes('harness-adapter-compliance.md'));
});
test('GA roadmap records the matrix as current evidence and points to data-backed validation next', () => {
test('GA roadmap records the matrix and validator as current evidence', () => {
const source = read('docs/ECC-2.0-GA-ROADMAP.md');
assert.ok(source.includes('docs/architecture/harness-adapter-compliance.md'));
assert.ok(source.includes('data-backed'));
assert.ok(source.includes('npm run harness:adapters -- --check'));
assert.ok(source.includes('scripts/lib/harness-adapter-compliance.js'));
});
if (failed > 0) {