hai/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-05-21 20:28:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: report job handles malformed eval JSON gracefully

Browse Source 
Large eval transcripts (350k+ tokens) can produce JSON that jq chokes on.
Skip malformed files instead of crashing the entire report job.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-23 21:21:20 -07:00
parent 69ef0527c7
commit 702f2547bb
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/evals.yml vendored
View File

@@ -182,6 +182,10 @@ jobs:
TOTAL=0; PASSED=0; FAILED=0; COST="0" TOTAL=0; PASSED=0; FAILED=0; COST="0"
SUITE_LINES="" SUITE_LINES=""
for f in $RESULTS; do for f in $RESULTS; do
if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
echo "Skipping malformed JSON: $f"
continue
fi
T=$(jq -r '.total_tests // 0' "$f") T=$(jq -r '.total_tests // 0' "$f")
P=$(jq -r '.passed // 0' "$f") P=$(jq -r '.passed // 0' "$f")
F=$(jq -r '.failed // 0' "$f") F=$(jq -r '.failed // 0' "$f")
@@ -214,9 +218,10 @@ jobs:
if [ "$FAILED" -gt 0 ]; then if [ "$FAILED" -gt 0 ]; then
FAILURES="" FAILURES=""
for f in $RESULTS; do for f in $RESULTS; do
if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
F=$(jq -r '.failed // 0' "$f") F=$(jq -r '.failed // 0' "$f")
[ "$F" -eq 0 ] && continue [ "$F" -eq 0 ] && continue
FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f") FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error")
FAILURES="${FAILURES}${FAILS}\n" FAILURES="${FAILURES}${FAILS}\n"
done done
BODY="${BODY} BODY="${BODY}