P6/benchmark_report.json
epoch 1
{
"run_id": "run-9f9f50da0c",
"epoch": 1,
"passed": 70,
"total": 70,
"score": 1.0,
"threshold": 0.9,
"gate": "pass",
"regenerate_hint": [],
"results": [
{
"id": "case-39dde9b273",
"axis": 1,
"tier": "aggregation",
"nl": "FILES per is_folder",
"dsl": "COUNT FILES BY IS_FOLDER",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-21cf645089",
"axis": 1,
"tier": "aggregation",
"nl": "Total depth",
"dsl": "SUM FILES DEPTH BY IS_FOLDER",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-e6452eba0f",
"axis": 1,
"tier": "aggregation",
"nl": "PERMISSIONS per principal",
"dsl": "COUNT PERMISSIONS BY PRINCIPAL",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-bc98187b5d",
"axis": 1,
"tier": "aggregation",
"nl": "PRINCIPALS per domain",
"dsl": "COUNT PRINCIPALS BY DOMAIN",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-8f624ffec2",
"axis": 1,
"tier": "aggregation",
"nl": "REVISIONS per modified_by",
"dsl": "COUNT REVISIONS BY MODIFIED_BY",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-fd530fbe35",
"axis": 1,
"tier": "aggregation",
"nl": "Total size_bytes",
"dsl": "SUM REVISIONS SIZE_BYTES BY MODIFIED_BY",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-711f86900a",
"axis": 1,
"tier": "aggregation",
"nl": "Total section_index",
"dsl": "SUM SECTIONS SECTION_INDEX",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-d0c986bd18",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST COMMENTS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-ae809e2d1f",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST DRIVES WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-08663c8a7a",
"axis": 1,
"tier": "error",
"nl": "Invalid enum value must coach, not silently drop",
"dsl": "LIST FILES WHERE IS_FOLDER IS \"NOT_A_REAL_VALUE\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-2594d8701b",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST FILES WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-adc888cf55",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST LABELS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-db594966b4",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST LINKS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-6b6d6eedcd",
"axis": 1,
"tier": "error",
"nl": "Invalid enum value must coach, not silently drop",
"dsl": "LIST PERMISSIONS WHERE PRINCIPAL IS \"NOT_A_REAL_VALUE\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-ae50793ccd",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST PERMISSIONS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-88dbeb34d9",
"axis": 1,
"tier": "error",
"nl": "Invalid enum value must coach, not silently drop",
"dsl": "LIST PRINCIPALS WHERE DOMAIN IS \"NOT_A_REAL_VALUE\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-0dcd10f726",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST PRINCIPALS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-e9f87ea40e",
"axis": 1,
"tier": "error",
"nl": "Invalid enum value must coach, not silently drop",
"dsl": "LIST REVISIONS WHERE MODIFIED_BY IS \"NOT_A_REAL_VALUE\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-b50bfc2f60",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST REVISIONS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-3f3ad5ac0e",
"axis": 1,
"tier": "error",
"nl": "Unknown field must coach, not silently drop",
"dsl": "LIST SECTIONS WHERE MADE_UP_FIELD IS \"x\"",
"passed": true,
"why": "structured error with all coaching fields",
"origin": "generated"
},
{
"id": "case-d03ce58f1e",
"axis": 1,
"tier": "filters",
"nl": "Show me cmt_3",
"dsl": "GET COMMENTS \"cmt_3\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-efe60b1084",
"axis": 1,
"tier": "filters",
"nl": "Show me Product",
"dsl": "GET DRIVES \"Product\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-062ff36747",
"axis": 1,
"tier": "filters",
"nl": "Show me Product",
"dsl": "GET FILES \"Product\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-f1c7bfb91e",
"axis": 1,
"tier": "filters",
"nl": "FILES where is_folder is no",
"dsl": "LIST FILES WHERE IS_FOLDER IS \"no\" LIMIT 10",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-e3fbcfb307",
"axis": 1,
"tier": "filters",
"nl": "Show me doc_roadmap26",
"dsl": "GET LABELS \"doc_roadmap26\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-d4d581b17d",
"axis": 1,
"tier": "filters",
"nl": "Show me doc_roadmap26",
"dsl": "GET LINKS \"doc_roadmap26\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-a1b3216d8a",
"axis": 1,
"tier": "filters",
"nl": "Show me fld_product",
"dsl": "GET PERMISSIONS \"fld_product\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-7f00e0eae3",
"axis": 1,
"tier": "filters",
"nl": "PERMISSIONS where principal is acme.com",
"dsl": "LIST PERMISSIONS WHERE PRINCIPAL IS \"acme.com\" LIMIT 10",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-f0a99ab7b9",
"axis": 1,
"tier": "filters",
"nl": "Show me acme.com",
"dsl": "GET PRINCIPALS \"acme.com\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-fa3001e50b",
"axis": 1,
"tier": "filters",
"nl": "PRINCIPALS where domain is *",
"dsl": "LIST PRINCIPALS WHERE DOMAIN IS \"*\" LIMIT 10",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-d5d595fdf4",
"axis": 1,
"tier": "filters",
"nl": "Show me doc_roadmap26@r1",
"dsl": "GET REVISIONS \"doc_roadmap26@r1\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-c0c7b0a725",
"axis": 1,
"tier": "filters",
"nl": "REVISIONS where modified_by is alice@acme.com",
"dsl": "LIST REVISIONS WHERE MODIFIED_BY IS \"alice@acme.com\" LIMIT 10",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-832faf5147",
"axis": 1,
"tier": "filters",
"nl": "Show me doc_roadmap26#s0",
"dsl": "GET SECTIONS \"doc_roadmap26#s0\"",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-5141343ed4",
"axis": 1,
"tier": "help",
"nl": "Read the COMMENTS lesson",
"dsl": "HELP COMMENTS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-82f72e68ec",
"axis": 1,
"tier": "help",
"nl": "Read the DRIVES lesson",
"dsl": "HELP DRIVES",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-a07b5dd5d8",
"axis": 1,
"tier": "help",
"nl": "Read the FILES lesson",
"dsl": "HELP FILES",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-a32ae2f866",
"axis": 1,
"tier": "help",
"nl": "Read the LABELS lesson",
"dsl": "HELP LABELS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-46edf05783",
"axis": 1,
"tier": "help",
"nl": "Read the LINKS lesson",
"dsl": "HELP LINKS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-2956b2b7b2",
"axis": 1,
"tier": "help",
"nl": "Read the PERMISSIONS lesson",
"dsl": "HELP PERMISSIONS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-4f71927c84",
"axis": 1,
"tier": "help",
"nl": "Read the PRINCIPALS lesson",
"dsl": "HELP PRINCIPALS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-334d17df04",
"axis": 1,
"tier": "help",
"nl": "Read the REVISIONS lesson",
"dsl": "HELP REVISIONS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-8e89d40057",
"axis": 1,
"tier": "help",
"nl": "Read the SECTIONS lesson",
"dsl": "HELP SECTIONS",
"passed": true,
"why": "lesson served in-band",
"origin": "generated"
},
{
"id": "case-3664f6d0c4",
"axis": 1,
"tier": "structure",
"nl": "List a few comments",
"dsl": "LIST COMMENTS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-c2834c2aef",
"axis": 1,
"tier": "structure",
"nl": "How many comments are there?",
"dsl": "COUNT COMMENTS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-67fc810820",
"axis": 1,
"tier": "structure",
"nl": "List a few drives",
"dsl": "LIST DRIVES LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-f14773a2f7",
"axis": 1,
"tier": "structure",
"nl": "How many drives are there?",
"dsl": "COUNT DRIVES",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-dcf6e2ff4b",
"axis": 1,
"tier": "structure",
"nl": "List a few files",
"dsl": "LIST FILES LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-07e4f38169",
"axis": 1,
"tier": "structure",
"nl": "How many files are there?",
"dsl": "COUNT FILES",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-7f7b4f63b1",
"axis": 1,
"tier": "structure",
"nl": "List a few labels",
"dsl": "LIST LABELS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-c441cbb5b4",
"axis": 1,
"tier": "structure",
"nl": "How many labels are there?",
"dsl": "COUNT LABELS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-122af1e5e3",
"axis": 1,
"tier": "structure",
"nl": "List a few links",
"dsl": "LIST LINKS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-a80e0ec7fa",
"axis": 1,
"tier": "structure",
"nl": "How many links are there?",
"dsl": "COUNT LINKS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-ad6e716c40",
"axis": 1,
"tier": "structure",
"nl": "List a few permissions",
"dsl": "LIST PERMISSIONS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-4a00fcac5a",
"axis": 1,
"tier": "structure",
"nl": "How many permissions are there?",
"dsl": "COUNT PERMISSIONS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-0ab6984fcf",
"axis": 1,
"tier": "structure",
"nl": "List a few principals",
"dsl": "LIST PRINCIPALS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-b609a7e3e6",
"axis": 1,
"tier": "structure",
"nl": "How many principals are there?",
"dsl": "COUNT PRINCIPALS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-182dd21f5d",
"axis": 1,
"tier": "structure",
"nl": "List a few revisions",
"dsl": "LIST REVISIONS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-9d7cbfc3c7",
"axis": 1,
"tier": "structure",
"nl": "How many revisions are there?",
"dsl": "COUNT REVISIONS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-cd3601df52",
"axis": 1,
"tier": "structure",
"nl": "List a few sections",
"dsl": "LIST SECTIONS LIMIT 5",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-2315d5ebd1",
"axis": 1,
"tier": "structure",
"nl": "How many sections are there?",
"dsl": "COUNT SECTIONS",
"passed": true,
"why": "deterministic match",
"origin": "generated"
},
{
"id": "case-2a200cbdab",
"axis": 2,
"tier": "intent",
"nl": "How many files by kind?",
"dsl": "COUNT FILES BY IS_FOLDER",
"passed": true,
"why": "deterministic match",
"origin": "intent"
},
{
"id": "case-c82a3067b9",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"cmt_3\"",
"dsl": "RESOLVE COMMENTS \"cmt_3\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-ccbcd73c9b",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"produ\"",
"dsl": "RESOLVE DRIVES \"produ\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-ccdeebe8a6",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"produ\"",
"dsl": "RESOLVE FILES \"produ\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-c2e3a27835",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"doc_roadmap\"",
"dsl": "RESOLVE LABELS \"doc_roadmap\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-4965422c60",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"doc_roadmap\"",
"dsl": "RESOLVE LINKS \"doc_roadmap\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-2c5c641721",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"fld_produ\"",
"dsl": "RESOLVE PERMISSIONS \"fld_produ\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-4e7f4f85af",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"acme.c\"",
"dsl": "RESOLVE PRINCIPALS \"acme.c\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-b976afe552",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"doc_roadmap26@\"",
"dsl": "RESOLVE REVISIONS \"doc_roadmap26@\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
},
{
"id": "case-29049059bc",
"axis": 2,
"tier": "robustness",
"nl": "Fuzzy lookup: \"doc_roadmap26#\"",
"dsl": "RESOLVE SECTIONS \"doc_roadmap26#\"",
"passed": true,
"why": "fuzzy match found",
"origin": "generated"
}
],
"generated_at": "2026-09-14T21:44:12.320925Z"
}
Back to instance