{"schema_version":4,"title":"When Ben Guo picks a frontier model: aggregate evidence","canonical_url":"https://hraness.com/prompting","analysis_cutoff_utc":"2026-09-24T12:00:00Z","previous_snapshot_cutoff_utc":"2026-09-22T00:00:00Z","input_date_timezone":"America/Puerto_Rico","sample":{"codex_threads_indexed":22020,"codex_human_threads":293,"codex_rollout_files_surviving":56,"claude_transcript_files_all_depths":1536,"claude_top_level_transcripts":26,"devin_sessions":49,"session_records_with_prompt":366,"machine_authored_prompts_excluded":16,"human_sessions":350,"human_labeled_sessions":337,"unlabeled_model_unknown":13,"jev_scored_sessions":327,"frontier_picks":46,"choice_window_start":"2026-09-01","choice_window_labeled_sessions":91,"choice_window_frontier_picks":46,"label_rule":"model serving the first turn: claude-fable-* and gpt-6-astra* = frontier; swe-2*, gpt-5-6-sol*, gpt-5.6-*, gpt-5.x, claude-opus*, claude-sonnet*, and remaining tier = standard; Devin adaptive and synthetic-only transcripts = unknown","previous_window_reconciliation":{"window":"2026-07-30..2026-09-21","labeled_now":125,"labeled_in_previous_snapshot":321,"codex_now":91,"codex_previous":195,"claude_now":14,"claude_previous":37,"devin_now":20,"devin_previous":89},"new_since_previous_snapshot":32},"by_provider":{"codex":{"human_sessions":282,"frontier":34,"models":{"gpt-5.6-sol":218,"gpt-6-astra":34,"gpt-5.5":13,"gpt-5.4":7,"gpt-5.6-terra":6,"gpt-5.6-luna":3,"gpt-5.1-codex-mini":1}},"claude":{"human_sessions":20,"frontier":12,"models":{"claude-fable-5-1":11,"claude-opus-5":6,"claude-opus-5-5":2,"claude-fable-5":1}},"devin":{"human_sessions":35,"frontier":0,"models":{"swe-2-max":25,"gpt-5-6-sol-xhigh":7,"gpt-5-6-sol-max":2,"gpt-5-6-sol-medium":1}}},"by_provider_choice_window":{"codex":{"human_sessions":36,"frontier":34,"models":{"gpt-6-astra":34,"gpt-5.6-sol":1,"gpt-5.1-codex-mini":1}},"claude":{"human_sessions":20,"frontier":12,"models":{"claude-fable-5-1":11,"claude-opus-5":6,"claude-opus-5-5":2,"claude-fable-5":1}},"devin":{"human_sessions":35,"frontier":0,"models":{"swe-2-max":25,"gpt-5-6-sol-xhigh":7,"gpt-5-6-sol-max":2,"gpt-5-6-sol-medium":1}}},"first_prompt_words":{"frontier":{"median":69.5,"mean":115.2,"p10":16,"p90":347,"n":46},"standard":{"median":36,"mean":67.4,"p10":7,"p90":134,"n":291}},"first_prompt_words_choice_window":{"frontier":{"median":69.5,"mean":115.2,"p10":16,"p90":347,"n":46},"standard":{"median":27,"mean":59,"p10":15,"p90":138,"n":45}},"jev_score_means_choice_window":{"difficulty":{"frontier":2.79,"standard":2.14},"scope":{"frontier":2.83,"standard":2.16},"ambiguity":{"frontier":3.09,"standard":2.96},"stakes":{"frontier":1.95,"standard":1.87},"frontier_worthiness":{"frontier":0.52,"standard":0.4}},"single_feature_auc_choice_window":{"words_c":0.709,"words_c_dir":"frontier","imperative":0.583,"imperative_dir":"frontier","resume":0.5,"resume_dir":"frontier","verbs_c":0.634,"verbs_c_dir":"frontier","difficulty":0.698,"difficulty_dir":"frontier","scope":0.685,"scope_dir":"frontier","ambiguity":0.562,"ambiguity_dir":"frontier","stakes":0.524,"stakes_dir":"frontier","frontier_noul":0.669,"frontier_noul_dir":"frontier"},"task_kind_by_jev_choice_window":{"resume":{"n":44,"frontier_share":0.5},"feature":{"n":15,"frontier_share":0.6},"research":{"n":7,"frontier_share":0.71},"ops":{"n":5,"frontier_share":0.2},"chore":{"n":4,"frontier_share":0.5},"probe":{"n":4,"frontier_share":0.5},"bugfix":{"n":3,"frontier_share":0},"writing":{"n":3,"frontier_share":0.67},"question":{"n":2,"frontier_share":0.5},"refactor":{"n":1,"frontier_share":0}},"task_kind_by_jev_all":{"feature":{"n":69,"frontier_share":0.13},"resume":{"n":47,"frontier_share":0.47},"chore":{"n":45,"frontier_share":0.04},"ops":{"n":43,"frontier_share":0.02},"research":{"n":40,"frontier_share":0.12},"question":{"n":32,"frontier_share":0.03},"bugfix":{"n":21,"frontier_share":0},"writing":{"n":16,"frontier_share":0.12},"refactor":{"n":8,"frontier_share":0},"probe":{"n":6,"frontier_share":0.33}},"codex_reasoning_effort":{"frontier":{"ultra":34},"standard":{"ultra":72,"high":62,"max":54,"medium":34,"xhigh":23,"low":3}},"frontier_share_by_month":{"2026-05":{"n":16,"frontier_share":0},"2026-06":{"n":1,"frontier_share":0},"2026-07":{"n":165,"frontier_share":0},"2026-08":{"n":64,"frontier_share":0},"2026-09":{"n":91,"frontier_share":0.51}},"frontier_share_by_iso_week":{"2026-30":{"n":54,"frontier_share":0},"2026-31":{"n":14,"frontier_share":0},"2026-32":{"n":29,"frontier_share":0},"2026-33":{"n":10,"frontier_share":0},"2026-34":{"n":11,"frontier_share":0},"2026-35":{"n":11,"frontier_share":0},"2026-36":{"n":2,"frontier_share":0.5},"2026-37":{"n":5,"frontier_share":1},"2026-38":{"n":40,"frontier_share":0.55},"2026-39":{"n":44,"frontier_share":0.41}},"sessions_with_model_switching":20,"long_prompts":{"threshold_words":150,"n":35,"frontier":8,"median_words":293,"max_words":752,"signature":{"first_person_want_share":0.714,"ends_with_question":0,"like_per_prompt":3.37,"we_lets_per_prompt":3.57,"paragraphs_mean":3.54,"bullets_mean":2.69,"contrast_under_40_words":{"first_person_want_share":0.14,"like_per_prompt":0.27,"we_lets_per_prompt":0.19}},"jev_scored":25,"kinds_by_jev":{"feature":{"n":14,"frontier_share":0.21},"research":{"n":4,"frontier_share":0},"resume":{"n":4,"frontier_share":0.75},"question":{"n":1,"frontier_share":0},"writing":{"n":1,"frontier_share":0},"ops":{"n":1,"frontier_share":0}}},"links":{"link_bearing_messages":619,"unique_urls":342,"sessions_with_links":94,"labeled_sessions_with_links":91,"link_session_frontier_share":0.165,"first_prompt_urls":95,"later_turn_urls":524,"github_link_messages":169,"github_hraness_repo_messages":111,"top_hraness_repos":{"direct":31,"hra":15,"bigdatadepot":12,"wrench":10,"slopcamera":8,"ui":5},"unique_destination_counts":{"external_articles":118,"github_external":40,"x_posts":12,"own_infrastructure":159,"arxiv":3,"docs_packages_video":9,"devin_session_links":1},"note":"Codex later-turn messages come from the thread item store for 56 threads and surviving rollouts for 51; older Codex threads contribute first prompts only.","link_router_eval":{"session_context":{"n":60,"accuracy":0.9,"evaluated_at":"2026-09-24","scoring":"live TypeSafe Jev, titles only"},"session_context_previous":{"n":55,"accuracy":0.909,"evaluated_at":"2026-09-22"},"reading_legacy_titles_previous":{"n":161,"accuracy":0.118}},"new_session_context_mappings_added":0},"classifier":{"artifact":"https://github.com/hraness/algal/blob/main/examples/model-router.algal.json","generation_0":{"training_sample":306,"cv_auc":0.739,"intercept":-1.8027,"weights":[0.453,1.5943,0.6881,0.4405,-0.0755,0.8716,-0.1508,-0.4787,0.5851],"on_choice_window":{"n":84,"auc":0.628,"accuracy_at_p035":0.583,"predicted_frontier_share":0.56}},"generation_1":{"method":"MAP logistic update anchored on generation 0 (xcb_core::reflex::fit rule, prior strength 24, 600 iterations, learning rate 0.5)","training_sample":84,"observed_frontier_share":0.5,"cv_auc":0.639,"in_sample_auc":0.652,"accuracy_at_p035":0.583,"predicted_frontier_share":0.583,"intercept":-1.7683,"weights":[0.5271,1.4028,0.4491,0.4438,0.0045,0.9353,-0.1459,-0.5129,0.6107],"threshold_z":-0.619,"kind_gates_to_standard":["question","probe"]},"september_only_fit_for_comparison":{"method":"L2 logistic fit from zero on the choice window","training_sample":84,"cv_auc":0.652,"in_sample_auc":0.736,"intercept":-1.1436,"weights":[0.7992,0.6323,-0.3219,0.5555,0.717,0.6245,0.0136,-0.7835,0.524],"predicted_frontier_share_at_p035":0.81},"full_corpus_fit_not_shipped":{"training_sample":321,"cv_auc":0.82,"why_not":"every frontier pick is a September prompt, so a head fit across May to September learns the calendar through prompt-style drift, not routing"},"features":["words_c","imperative","resume","verbs_c","difficulty","scope","ambiguity","stakes","frontier_noul"],"xcb_port_test_constant":{"task":"migrate the session store to the new envelope format and update every caller","score_milli":488},"jev_tokens_per_decision":{"in":965,"out":163}},"notes":["One operator, one machine. The label is the model picked, not a measured outcome.","Local stores rotate: Codex keeps a thread index for every session but only recent rollout files survive, Claude Code keeps only recent transcripts, and the Devin CLI store now starts on 2026-09-10. The 2026-09-22 snapshot's 321-session corpus is not reconstructible from today's stores; its aggregates are preserved in this file's history.","The frontier choice existed only from September 2026, so classifier evaluation uses the September choice window; the May to August sessions are all standard picks.","Devin adaptive sessions and transcripts with only synthetic assistant turns are unlabeled."]}