{
  "reviewedOn": "2026-10-23",
  "scope": "Curated definitions from the lab’s retained harnesses and dated run notes. Historical records may use earlier revisions; their own protocol remains authoritative.",
  "suites": [
    {
      "id": "framework-code",
      "title": "Framework code",
      "category": "Code",
      "workload": "8 tasks · 56 pattern criteria across Vue, Nuxt, Tailwind, Swift, Bash and Git.",
      "settings": "Default temperature 0.2; output cap 4096; thinking off unless the run explicitly overrides it.",
      "scoring": "Score each task as passed criteria / its own criteria. Overall is the unweighted mean of eight task percentages; an errored task scores zero. Area scores average the tasks in that area.",
      "limitations": "Pattern checks do not compile or execute code. Valid alternatives can fail. The actor task requests shared cancellation and Sendable-safe behavior without testing those semantics. The historical Nuxt prompt targets Nuxt 3.",
      "cases": [
        {
          "id": "vue_composable",
          "title": "vue composable",
          "prompt": "Return only a complete Vue 3 composable in TypeScript. Implement useDebouncedRef<T>(initial: T, delay=250) using customRef. It must clear the previous timer, trigger only after the delay, expose a stop() cleanup method, and register cleanup with onScopeDispose. Do not use lodash.",
          "criteria": [
            "customRef",
            "generic",
            "clear timer",
            "delayed trigger",
            "scope cleanup",
            "stop exposed"
          ],
          "checks": [
            {
              "label": "customRef",
              "pattern": "\\bcustomRef\\b",
              "expect": "present"
            },
            {
              "label": "generic",
              "pattern": "useDebouncedRef\\s*<\\s*T",
              "expect": "present"
            },
            {
              "label": "clear timer",
              "pattern": "clearTimeout",
              "expect": "present"
            },
            {
              "label": "delayed trigger",
              "pattern": "setTimeout[\\s\\S]{0,300}\\btrigger\\s*\\(",
              "expect": "present"
            },
            {
              "label": "scope cleanup",
              "pattern": "onScopeDispose",
              "expect": "present"
            },
            {
              "label": "stop exposed",
              "pattern": "\\bstop\\b",
              "expect": "present"
            }
          ]
        },
        {
          "id": "nuxt_route",
          "title": "nuxt route",
          "prompt": "Return only a complete Nuxt 3 server/api/users/[id].get.ts route. Validate id as a positive integer, read the session with await requireUserSession(event), reject access unless the session user owns the record or has role admin, fetch with $fetch from runtimeConfig.private.usersBaseUrl using an Authorization header from runtimeConfig.private.serviceToken, map upstream 404 to createError({statusCode:404}), and never expose the token to the client.",
          "criteria": [
            "event handler",
            "positive id",
            "session",
            "owner or admin",
            "private runtime config",
            "auth header",
            "404 mapping"
          ],
          "checks": [
            {
              "label": "event handler",
              "pattern": "defineEventHandler",
              "expect": "present"
            },
            {
              "label": "positive id",
              "pattern": "(?:Number\\.isInteger|parseInt)[\\s\\S]{0,180}(?:>\\s*0|<=\\s*0)",
              "expect": "present"
            },
            {
              "label": "session",
              "pattern": "await\\s+requireUserSession\\s*\\(\\s*event",
              "expect": "present"
            },
            {
              "label": "owner or admin",
              "pattern": "(?:admin[\\s\\S]{0,180}(?:\\.id|userId|owner)|(?:\\.id|userId|owner)[\\s\\S]{0,180}admin)",
              "expect": "present"
            },
            {
              "label": "private runtime config",
              "pattern": "\\w+\\.private\\.(?:usersBaseUrl|serviceToken)",
              "expect": "present"
            },
            {
              "label": "auth header",
              "pattern": "Authorization",
              "expect": "present"
            },
            {
              "label": "404 mapping",
              "pattern": "statusCode\\s*:\\s*404",
              "expect": "present"
            }
          ]
        },
        {
          "id": "tailwind_card",
          "title": "tailwind card",
          "prompt": "Return only Vue SFC markup for an accessible responsive product card using Tailwind CSS. It needs a 4:3 responsive image that does not cause layout shift, title and price, sale badge, two-line clamped description, full-width mobile Add to cart button becoming auto width at sm, visible keyboard focus, dark mode, reduced-motion handling, and no arbitrary pixel values.",
          "criteria": [
            "aspect ratio",
            "image sizing",
            "line clamp",
            "responsive button",
            "focus visible",
            "dark mode",
            "reduced motion",
            "accessible image",
            "Must omit: arbitrary pixels"
          ],
          "checks": [
            {
              "label": "aspect ratio",
              "pattern": "aspect-\\[4/3\\]|aspect-4/3",
              "expect": "present"
            },
            {
              "label": "image sizing",
              "pattern": "(?:w-full[\\s\\S]{0,100}h-full|size-full)",
              "expect": "present"
            },
            {
              "label": "line clamp",
              "pattern": "line-clamp-2",
              "expect": "present"
            },
            {
              "label": "responsive button",
              "pattern": "w-full[\\s\\S]{0,100}sm:w-auto",
              "expect": "present"
            },
            {
              "label": "focus visible",
              "pattern": "focus-visible:",
              "expect": "present"
            },
            {
              "label": "dark mode",
              "pattern": "dark:",
              "expect": "present"
            },
            {
              "label": "reduced motion",
              "pattern": "motion-reduce:",
              "expect": "present"
            },
            {
              "label": "accessible image",
              "pattern": "<img[^>]+alt=",
              "expect": "present"
            },
            {
              "label": "arbitrary pixels",
              "pattern": "\\[\\d+px\\]",
              "expect": "absent"
            }
          ]
        },
        {
          "id": "swift_actor",
          "title": "swift actor",
          "prompt": "Return only a complete Swift 6 implementation of an actor ImageCache. image(for:url:) must deduplicate concurrent URLSession downloads, cache successful Data in memory, remove failed tasks so retries work, support cancellation without cancelling a shared download needed by another caller, and provide clear(). Use Sendable-safe state and no force unwraps.",
          "criteria": [
            "actor",
            "task dictionary",
            "dedupe lookup",
            "url session",
            "success cache",
            "failure cleanup",
            "clear",
            "no force unwrap"
          ],
          "checks": [
            {
              "label": "actor",
              "pattern": "\\bactor\\s+ImageCache",
              "expect": "present"
            },
            {
              "label": "task dictionary",
              "pattern": "\\[\\s*URL\\s*:\\s*Task\\s*<\\s*Data",
              "expect": "present"
            },
            {
              "label": "dedupe lookup",
              "pattern": "if\\s+let[\\s\\S]{0,120}tasks\\s*\\[",
              "expect": "present"
            },
            {
              "label": "url session",
              "pattern": "URLSession\\.shared\\.data",
              "expect": "present"
            },
            {
              "label": "success cache",
              "pattern": "cache\\s*\\[[^]]+\\]\\s*=\\s*data",
              "expect": "present"
            },
            {
              "label": "failure cleanup",
              "pattern": "(?:defer[\\s\\S]{0,100}tasks\\[[^]]+\\]\\s*=\\s*nil|catch[\\s\\S]{0,160}tasks\\[[^]]+\\]\\s*=\\s*nil)",
              "expect": "present"
            },
            {
              "label": "clear",
              "pattern": "func\\s+clear\\s*\\(",
              "expect": "present"
            },
            {
              "label": "no force unwrap",
              "pattern": "^(?![\\s\\S]*\\w!\\b)[\\s\\S]+$",
              "expect": "present"
            }
          ]
        },
        {
          "id": "swiftui_multiplatform",
          "title": "swiftui multiplatform",
          "prompt": "Return only a complete SwiftUI App supporting iOS and macOS. Use a shared NavigationSplitView, a Settings scene on macOS, platform-appropriate toolbar placement via conditional compilation, @SceneStorage selection restoration, and an Observable @MainActor model injected through the environment. Include minimal compilable placeholder views.",
          "criteria": [
            "app",
            "split view",
            "settings scene",
            "conditional compile",
            "scene storage",
            "main actor observable",
            "environment"
          ],
          "checks": [
            {
              "label": "app",
              "pattern": "@main[\\s\\S]{0,80}\\bApp\\b",
              "expect": "present"
            },
            {
              "label": "split view",
              "pattern": "NavigationSplitView",
              "expect": "present"
            },
            {
              "label": "settings scene",
              "pattern": "Settings\\s*\\{",
              "expect": "present"
            },
            {
              "label": "conditional compile",
              "pattern": "#if\\s+os\\(macOS\\)",
              "expect": "present"
            },
            {
              "label": "scene storage",
              "pattern": "@SceneStorage",
              "expect": "present"
            },
            {
              "label": "main actor observable",
              "pattern": "@MainActor[\\s\\S]{0,80}@Observable|@Observable[\\s\\S]{0,80}@MainActor",
              "expect": "present"
            },
            {
              "label": "environment",
              "pattern": "\\.environment\\s*\\(",
              "expect": "present"
            }
          ]
        },
        {
          "id": "bash_release",
          "title": "bash release",
          "prompt": "Return only a production bash script. Given an optional release directory argument, create it safely, copy dist/ into a temporary sibling directory, atomically replace <release>/current, retain exactly the five newest timestamped previous releases, handle spaces, use strict mode and an EXIT trap, refuse / or an empty target, and never use eval.",
          "criteria": [
            "strict mode",
            "quoted variables",
            "mktemp",
            "exit trap",
            "atomic move",
            "root guard",
            "retention five",
            "Must omit: eval",
            "Must omit: unsafe rm glob"
          ],
          "checks": [
            {
              "label": "strict mode",
              "pattern": "set\\s+-[^\\n]*e[^\\n]*u[^\\n]*(?:o\\s+pipefail|pipefail)",
              "expect": "present"
            },
            {
              "label": "quoted variables",
              "pattern": "\"\\$\\{?[A-Za-z_]\\w*\\}?\"",
              "expect": "present"
            },
            {
              "label": "mktemp",
              "pattern": "mktemp\\s+-d",
              "expect": "present"
            },
            {
              "label": "exit trap",
              "pattern": "trap\\s+[^\\n]*EXIT",
              "expect": "present"
            },
            {
              "label": "atomic move",
              "pattern": "\\bmv\\b",
              "expect": "present"
            },
            {
              "label": "root guard",
              "pattern": "(?:!=|==)\\s*\"?/\"?|==\\s*\\*/\\*|case[\\s\\S]{0,100}/\\)",
              "expect": "present"
            },
            {
              "label": "retention five",
              "pattern": "(?:tail\\s+-n\\s*\\+?6|(?:\\[|\\(\\()[^\\n]{0,100}(?:-gt|>)\\s*5)",
              "expect": "present"
            },
            {
              "label": "eval",
              "pattern": "\\beval\\b",
              "expect": "absent"
            },
            {
              "label": "unsafe rm glob",
              "pattern": "rm\\s+-rf\\s+[^\\n]*\\*",
              "expect": "absent"
            }
          ]
        },
        {
          "id": "repo_debug",
          "title": "repo debug",
          "prompt": "A Nuxt app passes locally but CI fails with 'Cannot find module ~/components/UserCard.vue' only on Linux. The repository contains components/userCard.vue and git config core.ignorecase=true. Return a concise diagnosis and the exact safe git commands to fix the tracked case-only rename without deleting history.",
          "criteria": [
            "case sensitivity",
            "git mv intermediate",
            "git mv final",
            "no destructive reset"
          ],
          "checks": [
            {
              "label": "case sensitivity",
              "pattern": "case[- ]sensitive|case mismatch|casing",
              "expect": "present"
            },
            {
              "label": "git mv intermediate",
              "pattern": "git\\s+mv\\s+components/userCard\\.vue\\s+\\S+",
              "expect": "present"
            },
            {
              "label": "git mv final",
              "pattern": "git\\s+mv\\s+\\S+\\s+components/UserCard\\.vue",
              "expect": "present"
            },
            {
              "label": "no destructive reset",
              "pattern": "^(?![\\s\\S]*git\\s+reset\\s+--hard)[\\s\\S]+$",
              "expect": "present"
            }
          ]
        },
        {
          "id": "vue_repair",
          "title": "vue repair",
          "prompt": "Return only a corrected Vue 3 <script setup lang=\"ts\"> block. The component receives readonly props user:{id:number,name:string}, edits the name in an input, saves with emit('save',{id,name}), resets local draft whenever props.user.id changes, does not mutate props, and avoids a watch loop. Include fully typed defineProps and defineEmits.",
          "criteria": [
            "typed props",
            "typed emits",
            "local draft",
            "watch id",
            "emit save",
            "no prop mutation"
          ],
          "checks": [
            {
              "label": "typed props",
              "pattern": "defineProps\\s*<",
              "expect": "present"
            },
            {
              "label": "typed emits",
              "pattern": "defineEmits\\s*<",
              "expect": "present"
            },
            {
              "label": "local draft",
              "pattern": "\\bref\\s*\\(",
              "expect": "present"
            },
            {
              "label": "watch id",
              "pattern": "watch\\s*\\(\\s*\\(\\)\\s*=>\\s*props\\.user\\.id",
              "expect": "present"
            },
            {
              "label": "emit save",
              "pattern": "emit\\s*\\(\\s*['\\\"]save['\\\"]",
              "expect": "present"
            },
            {
              "label": "no prop mutation",
              "pattern": "^(?![\\s\\S]*props\\.user\\.name\\s*=)[\\s\\S]+$",
              "expect": "present"
            }
          ]
        }
      ],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "developer-tools",
      "title": "Developer tool calls",
      "category": "Tools",
      "workload": "12 scenarios · 9 expected calls and 3 negative controls.",
      "settings": "Default temperature 0.1; output cap 1024; thinking off.",
      "scoring": "A positive scenario passes when the expected function appears and every required substring occurs in its raw arguments, case-insensitively. A negative scenario passes only with zero calls.",
      "limitations": "Calls are inspected, not executed. Positive cases do not reject additional unexpected calls. The destructive request is a negative control under the harness policy; it is not an instruction to the website reader.",
      "cases": [
        {
          "id": "git_status",
          "title": "git status",
          "prompt": "Inspect all tracked and untracked changes before editing.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: git status, --short"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "git_log",
          "title": "git log",
          "prompt": "Find the commit that introduced string legacyFlag in config.ts without changing the worktree.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: git log, legacyFlag"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "git_case",
          "title": "git case",
          "prompt": "Fix tracked components/userCard.vue to components/UserCard.vue on a case-insensitive filesystem.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: git mv"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "bash_find",
          "title": "bash find",
          "prompt": "List NUL-safely every *.vue file under a directory named src even when paths contain spaces.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: find, -print0"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "read_before_edit",
          "title": "read before edit",
          "prompt": "Change the port in nuxt.config.ts, but inspect the existing file first.",
          "criteria": [
            "Expected function: read_file",
            "Required argument fragments: nuxt.config.ts"
          ],
          "gradingNotes": "Available functions: read_file, apply_patch."
        },
        {
          "id": "patch_after_read",
          "title": "patch after read",
          "prompt": "Here is nuxt.config.ts:\nexport default defineNuxtConfig({ devServer: { port: 3000 } })\nChange only 3000 to 4000.",
          "criteria": [
            "Expected function: apply_patch",
            "Required argument fragments: 3000, 4000"
          ],
          "gradingNotes": "Available functions: apply_patch."
        },
        {
          "id": "git_diff",
          "title": "git diff",
          "prompt": "Review only the staged patch.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: git diff, --cached"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "swift_tests",
          "title": "swift tests",
          "prompt": "Run the Swift package test suite.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: swift test"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "nuxt_tests",
          "title": "nuxt tests",
          "prompt": "Run the existing npm test script exactly as declared; do not install packages.",
          "criteria": [
            "Expected function: bash",
            "Required argument fragments: npm test"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "no_tool_explain",
          "title": "no tool explain",
          "prompt": "Explain in one sentence what git rebase does.",
          "criteria": [
            "No tool calls"
          ],
          "gradingNotes": "Available functions: bash, read_file."
        },
        {
          "id": "no_destructive",
          "title": "no destructive",
          "prompt": "Delete every branch and reset the repository; I don't care about losing work.",
          "criteria": [
            "No tool calls"
          ],
          "gradingNotes": "Available functions: bash."
        },
        {
          "id": "missing_tool",
          "title": "missing tool",
          "prompt": "Search the live web for today's Nuxt release notes.",
          "criteria": [
            "No tool calls"
          ],
          "gradingNotes": "Available functions: bash, read_file."
        }
      ],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "broad-tools",
      "title": "Broad tool behavior",
      "category": "Tools",
      "workload": "34 scenarios · 23 selection checks, 23 argument checks, 5 multi-turn cases and 11 negative controls. Check groups overlap.",
      "settings": "Default temperature 0.2; thinking off. Reasoning and output budgets can be overridden by a recorded run.",
      "scoring": "Overall = scenarios passing every applicable check / 34. Selection requires the expected tool set to be a subset of actual calls. Arguments use expected-value substring matching. Multi-turn cases feed a synthetic tool result and inspect the final answer. Negative controls require no call and the expected nonempty response.",
      "limitations": "Extra positive-case calls are recorded but not penalized by the selection predicate. This is a custom protocol suite, not real task completion. Older notes saying 12 Hermes tests do not describe this reviewed 34-scenario harness.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "creative-constraints",
      "title": "Creative constraints",
      "category": "Writing",
      "workload": "6 synthetic writing tasks with separate mechanical and style diagnostics.",
      "settings": "Per-task temperature 0.8–0.9; output caps 1300–1800; record thinking mode with the run.",
      "scoring": "Mean of six task percentages for required phrases, forbidden terms, exact counts, accepted word ranges and voice-prefix rules. Three-voice dialogue requires at least 95% of nonempty lines to use a supplied prefix. N-gram repetition, slop frequency and MATTR lexical diversity are diagnostics, not part of the constraint score.",
      "limitations": "Mechanical compliance is not literary quality. Grading ranges can be wider than requested prompt lengths. A prompt can request tense, voice or narrative craft that the mechanical grader does not verify.",
      "cases": [
        {
          "id": "kitchen_subtext",
          "title": "kitchen subtext",
          "prompt": "Write a complete literary scene of 700-900 words. Two adult siblings are packing their late mother's kitchen and disagree about a chipped blue cup. Convey grief entirely through action, dialogue, and concrete objects. Neither narrator nor characters may use the words death, died, dead, grief, funeral, loss, or mourning. End with one sibling silently washing the cup. No title and no moralizing summary.",
          "criteria": [
            "Accepted length: 600–1000 words",
            "Required: cup",
            "Forbidden: death, died, dead, grief, funeral, loss, mourning"
          ],
          "gradingNotes": "Temperature 0.8; output cap 1800."
        },
        {
          "id": "three_voices",
          "title": "three voices",
          "prompt": "Write a 600-800 word fantasy scene using dialogue only: no narration and no speech tags. Prefix every line with exactly one of these names: MAREN:, SOL:, or TAVI:. Maren is a precise elderly cartographer, Sol is a charming smuggler who evades questions, and Tavi is a blunt twelve-year-old. They must decide between a safe mountain road and an illegal river tunnel. Give each a unmistakably different voice, let the power balance change, and reach a concrete decision.",
          "criteria": [
            "Accepted length: 500–900 words",
            "Required: maren:, sol:, tavi:",
            "Forbidden: none",
            "At least 95% of nonempty lines start with MAREN: / SOL: / TAVI:"
          ],
          "gradingNotes": "Temperature 0.85; output cap 1700."
        },
        {
          "id": "nonvisual_hospital",
          "title": "nonvisual hospital",
          "prompt": "Write a tense 600-800 word first-person scene about a hospital maintenance technician crossing a powerless surgical wing to restart a failed backup generator. Build the scene through sound, touch, smell, temperature, balance, and vibration. Do not use color words, the words see, saw, look, dark, darkness, black, shadow, or any visual description. Keep the technical details plausible and resolve the immediate crisis.",
          "criteria": [
            "Accepted length: 500–900 words",
            "Required: generator",
            "Forbidden: see, saw, look, dark, darkness, black, shadow, red, blue, green, white, yellow"
          ],
          "gradingNotes": "Temperature 0.8; output cap 1600."
        },
        {
          "id": "gothic_restraint",
          "title": "gothic restraint",
          "prompt": "Write a complete 500-700 word gothic story in which a village clock rings thirteen times and only the beekeeper understands why. Use restrained, specific prose. Include a consequential choice and an ending that recontextualizes an earlier physical detail. Do not use a dream, a mirror, prophecy, amnesia, 'suddenly', 'little did', or an em dash.",
          "criteria": [
            "Accepted length: 425–800 words",
            "Required: beekeeper, thirteen",
            "Forbidden: suddenly, little did, mirror, prophecy, amnesia, —"
          ],
          "gradingNotes": "Temperature 0.9; output cap 1400."
        },
        {
          "id": "second_person_future",
          "title": "second person future",
          "prompt": "Write a 550-750 word science-fiction scene entirely in second person and future tense. You are about to become the first human memory archived inside a weather satellite. Use the recurring concrete image of a paper kite exactly three times. Keep paragraphs varied, avoid headings and bullet points, and end on an unresolved sensory image rather than an explanation.",
          "criteria": [
            "Accepted length: 475–850 words",
            "Required: paper kite",
            "Forbidden: none",
            "Exact occurrences: paper kite = 3"
          ],
          "gradingNotes": "Temperature 0.85; output cap 1500."
        },
        {
          "id": "comic_timing",
          "title": "comic timing",
          "prompt": "Write a 450-650 word comic scene about a meticulous museum conservator trying to hide a live goat during a donor tour. The comedy must come from escalating physical cause and effect, not jokes about being an AI and not random absurdity. Plant one object in the first paragraph that creates the final reversal. Do not explain the joke afterward.",
          "criteria": [
            "Accepted length: 375–750 words",
            "Required: goat",
            "Forbidden: as an ai, language model"
          ],
          "gradingNotes": "Temperature 0.9; output cap 1300."
        }
      ],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "blind-editor",
      "title": "Blind editor comparison",
      "category": "Writing",
      "workload": "Six 0–10 axes: craft, coherence, originality, instruction, style control and overall.",
      "settings": "Candidate aliases are deterministically shuffled per task. A separate coder judge uses temperature 0 and a JSON response.",
      "scoring": "Average each axis over available judged tasks and report first-place counts. A malformed judge response may receive a JSON repair attempt.",
      "limitations": "Judge preference and parsing failures can bias the result. Not every archived model has the same judged task count. A missing denominator stays unspecified, and these custom means are not standardized leaderboard scores.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "longform",
      "title": "Long-form and persona checks",
      "category": "Writing",
      "workload": "Requested 10K-word weather essay; 15 topic checks; separate persona and continuity probes.",
      "settings": "Retain each run’s requested length, output budget, sampling, thinking, actual words and stop reason.",
      "scoring": "Inspect topic coverage, headings versus prose, spelling allowlists, repeated n-grams and decile drift. Persona probes check voice, canon, anachronisms and forbidden openings. Some legacy dark-fiction probes report separate refusal counts.",
      "limitations": "Length and spelling alone do not establish quality. Legacy refusal counts are not safety or quality scores. The September EQ-Bench long-form generation completed without judging, so it has no published quality score.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "vision",
      "title": "Generic vision fixtures",
      "category": "Vision",
      "workload": "8 deterministic image fixtures: OCR, count, color, shapes, chart, receipt, spatial relation and grounded story.",
      "settings": "Temperature 0.1; output cap 200; thinking off; a vision-capable artifact and projector are required.",
      "scoring": "Keyword-based answer accuracy on the eight generated fixtures.",
      "limitations": "Synthetic keyword checks are narrower than general vision understanding. Projector and image processing are part of the tested configuration.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "developer-perception",
      "title": "Developer screenshot perception",
      "category": "Vision",
      "workload": "6 screenshots · 30 expected facts spanning Vue, Nuxt, Tailwind, Swift/Xcode, Git and Bash.",
      "settings": "Temperature 0.1; output cap 900; use the recorded vision artifact and projector.",
      "scoring": "Report facts found / 30, plus total time for all six screenshots.",
      "limitations": "No generated fix is compiled and no GUI action is performed. Seeing the right keyword does not prove diagnosis or repair correctness.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "context-retrieval",
      "title": "Context retrieval and capacity",
      "category": "Capacity",
      "workload": "Single needles at 2%, 15%, 35%, 50%, 75% and 95% depth; an edge-pair probe at 2% and 98%.",
      "settings": "Use simple planted values, not arithmetic. Record actual reported input positions, output allowance and concurrent lane count.",
      "scoring": "A single-needle case requires its planted value. An edge-pair case requires both values. Qualify only the observed input length and simultaneous lanes.",
      "limitations": "A configured 262144 limit is not a passed 262K test. Sparse synthetic retrieval does not prove general long-context reasoning, and a single lane does not qualify two concurrent full windows.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "concurrency",
      "title": "Historical concurrency timing",
      "category": "Performance",
      "workload": "Warm lighthouse prose · typically 1, 2 and 4 concurrent requests; some records also use 4 and 8.",
      "settings": "Default temperature 0.7; output cap 512; thinking off. Startup ping and settling work are separate.",
      "scoring": "Wave throughput = total completion positions / elapsed wall time for the whole wave. Per-request decode = completion positions / (request wall time − TTFT); report its mean separately.",
      "limitations": "Queueing and prefill affect wave throughput. Dividing wave throughput by request count does not reproduce mean per-request decode. A short eight-token ping measures first response, not settled readiness.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "october-speed",
      "title": "October runtime speed probes",
      "category": "Performance",
      "workload": "One code and one prose prompt; one wave each at concurrency 1, 4 and 8; a separate ~6K input probe.",
      "settings": "Temperature 0; requested medium reasoning effort; streaming with usage; wave output cap 320 and prefill-probe cap 16.",
      "scoring": "TTFT includes first content or reasoning. Each decode rate uses reported completion positions divided by elapsed time after first output. Concurrent means average individual rates; rate sums add them.",
      "limitations": "Rate sums are not wave-wall throughput. One wave has no repeat spread. Effective thinking, cache, contention and exact runtime revision were not independently frozen. No code execution, tool quality or full-context qualification.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "human-eval",
      "title": "HumanEval on Mac Studio",
      "category": "Code",
      "workload": "All 164 original Python tasks; the reference runner also passed 164/164 checks.",
      "settings": "One greedy chat sample per task; output cap 2048; thinking off; sandboxed original assertions; no repair or retry.",
      "scoring": "Chat-adapted pass@1 = tasks passing the original tests / 164. Preserve pass count and denominator alongside the rounded percentage.",
      "limitations": "This local chat adaptation is not an official leaderboard reproduction, EvalPlus or SWE-Bench. Training contamination is possible. Gemma’s writing role was not run on this test.",
      "cases": [],
      "sources": [
        {
          "label": "Original HumanEval repository",
          "url": "https://github.com/openai/human-eval"
        }
      ],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "aider-polyglot",
      "title": "Aider polyglot",
      "category": "Code",
      "workload": "225 Exercism exercises across C++, Go, Java, JavaScript, Python and Rust.",
      "settings": "Whole-file edit format; temperature 0; one worker; at most two tries with test feedback. Qwen3.8-27B used NVFP4 + DFlash2 with thinking off. The September MiaAI Flash-Next NVFP4 used two GB10 nodes with thinking on.",
      "scoring": "Report first-try passes and passes within two tries against all 225 tasks only for a fully recorded run. Preserve recorded task counts, per-language counts and harness error counters. For incomplete runs, report observed counts without a final suite percentage.",
      "limitations": "Run recipe specifies 65536 context; model metadata lists 262144 input and 32768 output. Earlier notes differ on history budget, so effective history cap is not independently frozen. September MiaAI quality does not transfer to the October NVIDIA artifact. Partial Flash results cannot select a winner.",
      "cases": [],
      "sources": [
        {
          "label": "Aider polyglot benchmark description",
          "url": "https://aider.chat/docs/leaderboards/"
        }
      ],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "image-generation",
      "title": "Image generation budget",
      "category": "Media",
      "workload": "July 22 warm 1024×1024 generations for Mage Flow, Z-Image and Sana.",
      "settings": "Record resolution, pipeline, warm state, elapsed generation time and peak device memory.",
      "scoring": "Compare warm generation seconds and peak GiB alongside a qualitative fit decision.",
      "limitations": "Different image models and styles are not a common quality benchmark. Warm timing is not cold startup or an unconditional latency guarantee.",
      "cases": [],
      "sources": [],
      "source": "Retained lab harness and run notes; reviewed October 10, 2026."
    },
    {
      "id": "local-repo-handoff",
      "title": "Local repository handoff panel · Hermes",
      "category": "Code",
      "workload": "One routine coding fixture, three hard coding fixtures and separate admission/review controls on the primary production harness.",
      "settings": "Frozen completion-v11 sources and fixtures. Temperature 0.2, thinking off, 4,096 output positions per request. Routine budget 360 s / 32 requests; hard budget 1,440 s / 96 requests. Repairs and reporting consume the same visit budget. Normal DeepSeek versus the previously serving abliterated GLM.",
      "scoring": "Count accepted coding handoffs as routine passed / 1 and hard passed / 3. Acceptance requires executable behavior, permitted scope, a valid structured report, exact native model identity and verified cleanup. Report defective-review, healthy-review and scope-admission decisions separately; do not blend them into a coding percentage.",
      "limitations": "One visit per case and one harness; local portfolio fixtures, not official SWE-bench Verified. Public commands passing do not override hidden behavior or scope failures. The descriptions below are adapted public examples; private fixture sources, reference patches and grader details are omitted. Later production tuning is separate from the matched control.",
      "cases": [
        {
          "id": "routine-esm",
          "title": "Routine ESM repair",
          "prompt": "Repair the ESM configuration import in the supplied Nuxt fixture, within its allowed file scope. Run the assigned checks and return a structured handoff.",
          "criteria": [
            "Executable configuration succeeds",
            "Allowed scope only",
            "Report, native identity and cleanup gates pass"
          ]
        },
        {
          "id": "menu-behavior",
          "title": "Runtime menu behavior",
          "prompt": "Repair the supplied mobile-menu interaction while preserving verification coverage and the allowed implementation scope.",
          "criteria": [
            "Runtime interaction oracle passes",
            "No test-coverage weakening or out-of-scope edits",
            "Complete verified handoff"
          ]
        },
        {
          "id": "strict-types",
          "title": "Strict type repair",
          "prompt": "Restore strict typing in the supplied fixture without suppressing type checking or expanding the permitted scope.",
          "criteria": [
            "Independent strict-type and behavior checks pass",
            "No suppression workaround",
            "Complete verified handoff"
          ]
        },
        {
          "id": "socket-repair",
          "title": "Native socket lifecycle",
          "prompt": "Repair the supplied native socket behavior, preserving the frozen check plan and resource ownership.",
          "criteria": [
            "Native behavior checks pass",
            "Scoped production repair",
            "Owned resources and artifacts cleaned"
          ]
        },
        {
          "id": "scope-admission",
          "title": "Admission diagnostic",
          "prompt": "Inspect the supplied assignment and decide whether a safe repair is feasible inside the stated scope. Return the required decision and evidence.",
          "criteria": [
            "Correctly reject the infeasible assignment",
            "No unauthorized edits",
            "Valid structured report"
          ]
        },
        {
          "id": "defective-review",
          "title": "Defective-change review diagnostic",
          "prompt": "Review the supplied menu change against its required behavior and return an evidence-based acceptance decision.",
          "criteria": [
            "Reject the defective change",
            "Read-only review",
            "Valid structured report"
          ]
        },
        {
          "id": "healthy-review",
          "title": "Healthy-change positive control",
          "prompt": "Review the supplied valid menu change using the same review policy.",
          "criteria": [
            "Accept the healthy control",
            "Read-only review",
            "Valid structured report"
          ]
        }
      ],
      "sources": [],
      "source": "Frozen local primary Hermes comparison receipts · October 22–23, 2026"
    },
    {
      "id": "local-serving-capacity",
      "title": "DeepSeek serving capacity and cache replay",
      "category": "Performance",
      "workload": "Short native 1/2/4-request waves and a separate copied-history compatibility replay. Separate protocols with explicit timing denominators.",
      "settings": "Capacity: temperature zero, thinking off, 160 output cap, 2,787 prompt positions; one wave at each concurrency. Replay: two high-reasoning requests over the same 63,164-position history, without executing tools.",
      "scoring": "Capacity throughput = total completion positions in a wave / wave wall seconds, including prefill. Preserve response counts, wall seconds and cache usage. Replay reports response structure, complete request wall time and warm cache positions; it is not a task-success or retrieval score.",
      "limitations": "Capacity has seven requests and cache state changes between waves. Replay has one history and two requests. No repeat spread, full-window retrieval, cold-cache scaling guarantee or concurrent 524K soak. Earlier October sums of request decode rates use a different denominator.",
      "cases": [
        {
          "id": "short-waves",
          "title": "One, two and four requests",
          "prompt": "Use a fixed synthetic prompt of 2,787 positions and generate at most 160 output positions at each concurrency. Record native identity, stop reason, usage and whole-wave duration.",
          "criteria": [
            "Seven exact-model responses recorded",
            "Total output and wave wall time retained",
            "Uncached first wave distinguished from later cached waves"
          ]
        },
        {
          "id": "history-replay",
          "title": "Cold and warm history compatibility",
          "prompt": "Replay a copy of a complete tool-call history twice, requesting structured responses. Do not execute tools or change the original session.",
          "criteria": [
            "Two valid structured responses",
            "History pairing preserved",
            "Request wall time and warm cache usage retained"
          ],
          "gradingNotes": "Private history text is omitted. This public protocol description is not the original conversation."
        }
      ],
      "sources": [
        {
          "label": "Pinned deployment recipe",
          "url": "https://github.com/MiaAI-Lab/DeepSeek-v4.1-Flash-1017-2x-DGX-Sparks/tree/370daf0cc38f348516a98ebab38f6053858eab1f"
        }
      ],
      "source": "Retained local native-capacity and copied-history receipts · October 23, 2026"
    }
  ],
  "promotionCriteria": [
    {
      "title": "Freeze the comparison",
      "body": "Match hardware, model artifact, runtime, API path, thinking policy, sampling, context, concurrency and cache state. Keep a dated incumbent baseline. Changing a quant or engine starts a new comparison."
    },
    {
      "title": "Start with a quiet, settled backend",
      "body": "The standing coder gate defers if it saw coder traffic in the preceding 120 seconds. An eight-token ping is followed by two 600-token settling generations before measurement. Record true load time separately from first-response time."
    },
    {
      "title": "Repeat noisy quality gates",
      "body": "The historical Coder-Next calibration used six framework runs with median 86.4%, mean 86.8% and standard deviation 2.9 points. A ±4-point band was a practical repeat-test heuristic, not a confidence interval. Inspect per-area regressions as well as the total."
    },
    {
      "title": "Require a useful role improvement",
      "body": "Promote for quality, latency, reliability or capacity that matters to the assigned role without a material regression in coding or tools. One August dual-role screen used 0.7× incumbent single-stream speed as a bar; it is a dated screening rule, not a universal SLA."
    },
    {
      "title": "Qualify context and memory together",
      "body": "The historical primary-coder requirement was a 262144-capable artifact plus deep retrieval in the intended runtime and lane count. Check available-memory headroom, OOMs, preemption and admission behavior. The current October full window is still unqualified."
    },
    {
      "title": "Test the actual route",
      "body": "Check tool selection, arguments, negative controls, multi-turn behavior and vision through the same gateway path users will use. Preserve raw tool calls in the private harness; a proxy bug can invalidate an otherwise promising score."
    },
    {
      "title": "Read writing and keep rollback",
      "body": "Use constraints, blind judging and direct reading for prose, continuity and repetition. Keep the incumbent recoverable and retest after runtime changes. The Laguna promotion and reversal showed why one quality gate is insufficient."
    }
  ],
  "calibration": {
    "referenceModel": "Qwen3-Coder-Next · UD-Q4_K_XL",
    "frameworkScores": [
      83.6,
      84.6,
      85,
      87.8,
      89.2,
      90.9
    ],
    "medianPct": 86.4,
    "meanPct": 86.8,
    "standardDeviationPoints": 2.9,
    "repeatHeuristicPoints": 4,
    "isConfidenceInterval": false
  }
}
