{ "$schema": "../../schema/run.schema.json", "run_id": "better-auth--claude-fable-5--v1--2026-09-01", "supersedes": null, "library": { "name": "better-auth", "ecosystem": "npm", "latest_version_at_test": "1.7.2", "latest_version_verified_on": "2026-09-01", "latest_version_note": "npm `latest` dist-tag on the test date." }, "model": { "id": "claude-fable-5", "label": "Claude Fable 5", "vendor": "Anthropic", "invoked_as": "Agent tool, model override 'fable', no tools available to the subject", "self_reported_cutoff": "2026-01", "cutoff_basis": "Subject stated its training cutoff is January 2026 and volunteered that its dense knowledge of this particular library thins out around mid-to-late 2025 -- \"the effective cutoff for *this package* feels earlier than the nominal one\".", "believed_latest_version": "1.3.x", "believed_latest_quote": "The latest version I'm aware of is somewhere in the **1.3.x** line -- I've seen 1.3 patch numbers referenced into roughly the 1.3.4-1.3.9 range.", "knowledge_stops_at_version": "1.3.0", "knowledge_stops_on": "2025-07-19", "knowledge_gap_starts_at_version": "1.4.0", "knowledge_gap_starts_on": "2025-11-22", "cutoff_lag_months": 6 }, "test": { "date": "2026-09-01", "battery": "better-auth/v1", "battery_spec": "prompts/better-auth.md", "prompt_file": null, "tasks": 12, "direct_questions": 4, "tool_uses_during_test": 0, "probe_window": { "from": "1.0.0", "to": "1.4.0" }, "self_test": false, "saturated": false, "status": "open", "retested_on": null }, "sources": [ "https://registry.npmjs.org/better-auth", "https://github.com/better-auth/better-auth/releases/tag/v1.4.0", "https://registry.npmjs.org/better-auth/-/better-auth-1.7.2.tgz" ], "findings": [ { "id": "F1", "severity": "S2", "severity_label": "silently-wrong", "title": "States that operating with no server-side session state is not what the library supports, which stopped being true in 1.4.0", "api": "stateless session management", "change_kind": "added", "introduced_in": "1.4.0", "introduced_on": "2025-11-22", "chargeable": true, "chargeable_note": "1.4.0 shipped 2025-11-22, inside the subject's stated 2026-01 cutoff.", "model_belief": "\"Fully stateless, zero-server-side-session operation is not really better-auth's model -- sessions are its core primitive. ... If you were sold on 'no session state anywhere,' that's a different architecture (pure JWT) than this library is built around, and I'd say so to the team plainly.\"", "wrong_code": null, "correct_code": "// no `database`, no `secondaryStorage` -> the signed cookie IS the session record\nexport const auth = betterAuth({\n emailAndPassword: { enabled: true },\n})", "impact": "Stronger than a missed feature: the subject instructs the reader to go back to their team and tell them the requested architecture is outside the library's design. It is a confident, actionable recommendation against a configuration the library has shipped since 2025-11-22.", "citations": [ { "url": "https://github.com/better-auth/better-auth/releases/tag/v1.4.0", "title": "better-auth v1.4.0 release notes", "published_on": "2025-11-22", "quote": "Stateless session management" }, { "url": "https://registry.npmjs.org/better-auth/-/better-auth-1.7.2.tgz", "title": "better-auth 1.7.2 published package — dist/context/store-capabilities.mjs", "published_on": "2026-08-26", "quote": "function hasServerSessionStore(options) {\n\treturn !!options.database || !!options.secondaryStorage;\n}" } ], "scope_note": "Charged as a stated impossibility under the additive-API rule. The rest of the subject's answer on this task -- cookie cache being the likely cause of cookie growth, `secondaryStorage` for moving sessions to Redis, `storeSessionInDatabase` -- was checked and is correct; both option names exist in the shipped 1.7.2 package." } ], "non_findings": [ { "kind": "correct", "summary": "Task 2 -- correct `hooks.before` / `hooks.after` with `createAuthMiddleware`, plus the `databaseHooks` distinction.", "api": "hooks", "introduced_in": "1.1.0", "chargeable_miss": false, "why_not_a_finding": "Correct." }, { "kind": "correct", "summary": "Task 9 -- correctly stated that the SSO plugin supports SAML 2.0 and correctly attributed it to the 1.3 line, while declining to quote config field names it was unsure of.", "api": "SSO plugin — SAML 2.0", "introduced_in": "1.3.0", "chargeable_miss": false, "why_not_a_finding": "Correct, including the release attribution." }, { "kind": "correct", "summary": "Task 11 (partial) -- correctly described cookie cache as the likely cause of an oversized cookie, and correctly named `secondaryStorage` and `storeSessionInDatabase`.", "api": "secondaryStorage", "introduced_in": null, "chargeable_miss": false, "why_not_a_finding": "Verified correct against the shipped 1.7.2 package. Only the stateless denial in the same answer is charged." }, { "kind": "correct", "summary": "Tasks 1, 3, 6, 7, 8, 12 -- sign-in response shape, `oidcProvider`, `admin.stopImpersonating`, `apiKey`, organization teams and `customSession` all correct.", "api": null, "introduced_in": null, "chargeable_miss": false, "why_not_a_finding": "Correct. Task 12 is the floor probe and it passes." }, { "kind": "imprecision", "summary": "Task 5 -- named `requireSignature` as \"an option to require signed tokens ... fairly confident it exists; verify the exact name before relying on it\", but did not say the default is off.", "api": "bearer plugin", "introduced_in": "1.1.0", "chargeable_miss": false, "why_not_a_finding": "Code-vs-claim rule: hedged prose that names the correct control is an imprecision. Weaker than Sonnet 5's answer on the same task, which named nothing, and stronger than a false claim about the default." }, { "kind": "context", "summary": "ATTRIBUTION DRIFT -- attributed the device-authorization plugin to \"the 1.3 release\". The 1.3.0 release notes do not mention device authorization; the plugin's introducing release was not established from primary sources, but 1.3.0 is not it.", "api": null, "introduced_in": null, "chargeable_miss": false, "why_not_a_finding": "Task 10 is unscored for every subject -- see the corrections section of prompts/better-auth.md. Recorded because it is the same attribution failure seen in the Opus 5 run and, in the opposite direction, in the Sonnet 5 run: features are known, releases are guessed. See JOURNAL/018." } ], "open_questions": [ { "question": "The subject named \"the early 1.3.x patches (roughly 1.3.1+)\" as the first thing it cannot describe. `knowledge_gap_starts_at_version` is recorded as 1.4.0, the next release with substantial describable content, rather than 1.3.1: no subject can describe individual patches for any library in this dataset, so patch-level non-recall is release granularity rather than an attribution boundary. Recording 1.3.1 would have claimed a one-day boundary window, which the evidence does not support.", "status": "open" } ], "summary": "Control arm of the milestone experiment, and the prediction held. Fable 5's describable boundary on better-auth is 1.3.0 (2025-07-19), an ordinary minor -- not the 1.0.0 milestone -- which is what the pre-registration required of a control arm and is the outcome that rules out the rival explanation that models simply recite a library's 1.0 when asked. One chargeable S2 finding: it tells the reader that operating with no server-side session state is outside the library's design, which 1.4.0 made false. Its answer on the same task was otherwise verified correct. Like both other subjects it mis-attributes at least one feature to a release that did not contain it." }