#!/usr/bin/env bash # End-to-end check of the ingest contract against a real `wrangler dev` + local D1. # # Boots the worker, POSTs a spread of good and bad batches, then shuts the worker # down and inspects the rows that actually landed. Every request uses a fresh # machine_id, so the script is re-runnable against a dirty local database and never # trips the per-machine rate limit. # # npm run db:migrate:local # once # npm run smoke # or: INGEST_PORT=8791 ./scripts/smoke-ingest.sh set -euo pipefail cd "$(dirname "$0")/.." PORT="${INGEST_PORT:-8787}" BASE="http://127.0.0.1:$PORT" DB=codegraph-telemetry pass=0; fail=0 ok() { pass=$((pass + 1)); printf ' ok %s\n' "$1"; } bad() { fail=$((fail + 1)); printf ' FAIL %s — expected %s, got %s\n' "$1" "$2" "$3"; } is() { [ "$2" = "$3" ] && ok "$1" || bad "$1" "$2" "$3"; } uuid() { node -e 'console.log(crypto.randomUUID())'; } # HTTP status of a POST /v1/events with the given body. post() { curl -s -o /dev/null -w '%{http_code}' -X POST "$BASE/v1/events" \ -H 'content-type: application/json' --data-binary "$1"; } # First column of the first row of a query against the LOCAL D1 state. q() { npx wrangler d1 execute "$DB" --local --json --command "$1" 2>/dev/null | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{ const r=JSON.parse(s.slice(s.indexOf("[")))[0]?.results?.[0]; console.log(r===undefined?"":String(Object.values(r)[0]));})' } # --------------------------------------------------------------------------- # Boot # --------------------------------------------------------------------------- echo "booting wrangler dev on :$PORT" npx wrangler dev --port "$PORT" >/tmp/cg-smoke-ingest.log 2>&1 & DEV_PID=$! cleanup() { kill "$DEV_PID" 2>/dev/null || true; wait "$DEV_PID" 2>/dev/null || true; } trap cleanup EXIT for _ in $(seq 1 60); do curl -sf -o /dev/null "$BASE/" && break kill -0 "$DEV_PID" 2>/dev/null || { echo "wrangler dev died:"; cat /tmp/cg-smoke-ingest.log; exit 1; } sleep 1 done curl -sf -o /dev/null "$BASE/" || { echo "worker never came up:"; cat /tmp/cg-smoke-ingest.log; exit 1; } # --------------------------------------------------------------------------- # Request contract # --------------------------------------------------------------------------- echo echo "request contract" INFO=$(curl -s "$BASE/") case "$INFO" in *"codegraph anonymous-telemetry ingest"*) ok "GET / serves the info text";; *) bad "GET / serves the info text" "info text" "$INFO";; esac case "$INFO" in *"never forwarded to any third-party analytics"*) ok "info text states the storage guarantee";; *) bad "info text states the storage guarantee" "the no-third-party sentence" "missing";; esac # The guarantee above holds only while the worker makes no outbound request at all, # so the only `fetch(` anywhere in the source may be the handler's own declaration. is "worker source makes no outbound fetch" 0 \ "$(grep -E 'fetch\(' src/*.ts | grep -vc 'async fetch(request' || true)" is "unknown path → 404" 404 "$(curl -s -o /dev/null -w '%{http_code}' "$BASE/nope")" is "GET /v1/events → 405" 405 "$(curl -s -o /dev/null -w '%{http_code}' "$BASE/v1/events")" is "non-JSON body → 400" 400 "$(post 'not json')" is "JSON array body → 400" 400 "$(post '[]')" is "missing machine_id → 400" 400 "$(post '{"events":[]}')" is "malformed machine_id → 400" 400 "$(post '{"machine_id":"nope","events":[]}')" is "chunked (no length) → 411" 411 "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$BASE/v1/events" \ -H 'content-type: application/json' -H 'transfer-encoding: chunked' --data-binary '{"machine_id":"x"}')" BIG=$(node -e 'process.stdout.write(JSON.stringify({machine_id:"00000000-0000-4000-8000-000000000000",pad:"x".repeat(70000),events:[]}))') is "oversized body → 413" 413 "$(post "$BIG")" # --------------------------------------------------------------------------- # Accepted batches # --------------------------------------------------------------------------- echo echo "ingest" M_OK=$(uuid); M_DROP=$(uuid); M_CI=$(uuid); M_BACK=$(uuid) TODAY=$(date -u +%F) # Three valid events + one unknown event + unknown/malformed props that must be stripped. is "valid batch → 204" 204 "$(post "$(node -e ' const [m] = process.argv.slice(1); process.stdout.write(JSON.stringify({ machine_id: m, codegraph_version: "1.5.0", os: "darwin", arch: "arm64", node_major: 22, ci: false, schema_version: 1, secret_field: "must not be stored", events: [ { event: "install", ts: "2026-07-27T10:00:00Z", props: { scope: "local", kind: "fresh", targets: ["claude", "cursor"], nope: "strip me" } }, { event: "index", ts: "2026-07-27T10:01:00Z", props: { languages: ["typescript"], file_count_bucket: "100-1k", duration_bucket: "bogus-bucket", repo_path: "/Users/someone/secret" } }, { event: "usage_rollup", props: { kind: "mcp_tool", name: "codegraph_explore", count: 12, client_name: "Claude Code" } }, { event: "not_an_event", props: { count: 1 } }, ], }));' "$M_OK")")" # Nothing survives the allowlist: unknown event + usage_rollup missing required props. is "all-dropped batch → 204" 204 "$(post "$(node -e ' const [m] = process.argv.slice(1); process.stdout.write(JSON.stringify({ machine_id: m, os: "linux", events: [ { event: "made_up" }, { event: "usage_rollup", props: { kind: "mcp_tool" } }, { event: "install", props: { scope: "local" } }, ]}));' "$M_DROP")")" # NOTE: build every body into a variable first. Escaped quotes nested inside # "$(post "…\"…\"…")" break out of the quoting context and get brace-expanded. index_batch() { # [ci] [ts] node -e 'const [m, ci, ts] = process.argv.slice(1); const e = { event: "index", props: {} }; if (ts) e.ts = ts; const b = { machine_id: m, os: "linux", events: [e] }; if (ci) b.ci = ci === "true"; process.stdout.write(JSON.stringify(b));' "$@" } # ci = true, then a non-CI batch for the same machine/day: prod must flip 0 → 1. CI_ON=$(index_batch "$M_CI" true); CI_OFF=$(index_batch "$M_CI" false) is "ci batch → 204" 204 "$(post "$CI_ON")" is "same machine, non-ci → 204" 204 "$(post "$CI_OFF")" # A late offline buffer arriving second must move first_day EARLIER, never later. RECENT=$(index_batch "$M_BACK" "" 2026-07-27T09:00:00Z) BACKDATED=$(index_batch "$M_BACK" "" 2026-07-20T09:00:00Z) is "recent batch → 204" 204 "$(post "$RECENT")" is "backdated batch → 204" 204 "$(post "$BACKDATED")" sleep 2 # let the ctx.waitUntil writes drain cleanup; trap - EXIT sleep 1 # and let miniflare release the local sqlite file # --------------------------------------------------------------------------- # What actually got stored # --------------------------------------------------------------------------- echo echo "stored rows" is "3 of 4 events stored (unknown dropped)" 3 "$(q "select count(*) from events where machine_id='$M_OK'")" is "all-dropped batch stored nothing" 0 "$(q "select count(*) from events where machine_id='$M_DROP'")" is "…and no machine_days row for it" 0 "$(q "select count(*) from machine_days where machine_id='$M_DROP'")" is "envelope columns land in their own columns" "darwin|arm64|22|0|1.5.0" \ "$(q "select os||'|'||arch||'|'||node_major||'|'||ci||'|'||codegraph_version from events where machine_id='$M_OK' limit 1")" is "day derived from the client ts" "2026-07-27" \ "$(q "select day from events where machine_id='$M_OK' and event='install'")" is "day falls back to received_at when ts is absent" "$TODAY" \ "$(q "select day from events where machine_id='$M_OK' and event='usage_rollup'")" is "ts is NULL when the client sent none" 1 \ "$(q "select ts is null from events where machine_id='$M_OK' and event='usage_rollup'")" is "allowlisted props stored" "local|fresh|2" \ "$(q "select json_extract(props,'\$.scope')||'|'||json_extract(props,'\$.kind')||'|'||json_array_length(props,'\$.targets') from events where machine_id='$M_OK' and event='install'")" is "unknown prop stripped" 0 \ "$(q "select count(*) from events where machine_id='$M_OK' and props like '%strip me%'")" is "malformed enum prop stripped" 0 \ "$(q "select count(*) from events where machine_id='$M_OK' and props like '%bogus-bucket%'")" is "path-shaped prop stripped" 0 \ "$(q "select count(*) from events where machine_id='$M_OK' and props like '%/Users/%'")" is "unknown envelope field stored nowhere" 0 \ "$(q "select count(*) from events where props like '%must not be stored%'")" # The valid batch mixes ts-dated events (2026-07-27) with an undated rollup (today), # so it legitimately spans two days and must produce a machine_days row for each. is "machine_days: one row per distinct day in the batch" 2 \ "$(q "select count(*) from machine_days where machine_id='$M_OK'")" is "machine_days: non-ci machine is production" 1 \ "$(q "select min(prod) from machine_days where machine_id='$M_OK'")" is "machine_days: a later non-ci batch flips the day to production" 1 \ "$(q "select prod from machine_days where machine_id='$M_CI'")" is "machine_days: each backdated batch gets its own day" "2026-07-20,2026-07-27" \ "$(q "select group_concat(day) from (select day from machine_days where machine_id='$M_BACK' order by day)")" is "machine_first_seen recorded" "2026-07-27" "$(q "select first_day from machine_first_seen where machine_id='$M_OK'")" is "machine_first_seen only moves earlier" "2026-07-20" \ "$(q "select first_day from machine_first_seen where machine_id='$M_BACK'")" echo echo "$pass passed, $fail failed" [ "$fail" -eq 0 ]