{
 "about": "Jeff v1.3 measured numbers, built by `pnpm results:v13` from results/sources/v1.3/ (see SOURCES.md there). Accuracy and ECE (15 bins, after each model's own fitted temperature) are fractions from 0 to 1. Every adapter test is scored three ways on the same rows: the untrained Qwen3.5-0.8B, the Jeff v1.3 base alone, and the v1.3 base with the adapter. v1_2 holds the published v1.2 numbers only where v1.2 was measured on exactly the same rows. pending lists what is not measured yet.",
 "release": "v1.3",
 "generated": "2026-10-05T18:31:28+00:00",
 "base": {
  "name": "Jeff-Qwen3.5-0.8B v1.3",
  "checkpoint": "ll-v12-final",
  "prompt_layout": "live-last",
  "trained_on": "the v1.2 base training data, unchanged"
 },
 "general_panel": {
  "questions": 4599,
  "ece_bins": 15,
  "v1_3": {
   "accuracy": 0.7864753207218961,
   "ece": 0.027964125342822006,
   "checkpoint": "/raid/work/experiments/jeff/checkpoints/ll-v12/step-01113",
   "source": "results/sources/v1.3/panel-v1.3-base.json"
  },
  "v1_2": {
   "accuracy": 0.7879973907371167,
   "ece": 0.0244639164058482,
   "checkpoint": "/raid/work/experiments/jeff/checkpoints/jeff-0.8b-v1.2",
   "source": "results/sources/v1.3/panel-v1.2-base.json"
  }
 },
 "adapters": {
  "legal-clauses": {
   "run": "0.8b-legal-clauses-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 9895,
     "options": {
      "min": 100,
      "max": 100
     },
     "untrained": {
      "accuracy": 0.12511369378473977,
      "ece": 0.09376162797831193
     },
     "base": {
      "accuracy": 0.07367357251136938,
      "ece": 0.03924839370770678
     },
     "adapter": {
      "accuracy": 0.8363820111167256,
      "ece": 0.010929339202633614
     },
     "v1_2": {
      "rows": 9895,
      "base": {
       "accuracy": 0.66,
       "ece": 0.032
      },
      "adapter": {
       "accuracy": 0.857,
       "ece": 0.011
      }
     },
     "groups": []
    },
    {
     "test": "test-novel",
     "rows": 7357,
     "options": {
      "min": 100,
      "max": 100
     },
     "untrained": {
      "accuracy": 0.12124507271985864,
      "ece": 0.08988471233970585
     },
     "base": {
      "accuracy": 0.07081690906619546,
      "ece": 0.04378476991973563
     },
     "adapter": {
      "accuracy": 0.8143264917765394,
      "ece": 0.014832325678416778
     },
     "v1_2": null,
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "support-intents": {
   "run": "0.8b-support-intents-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 5577,
     "options": {
      "min": 7,
      "max": 64
     },
     "untrained": {
      "accuracy": 0.3385332616101847,
      "ece": 0.16409902511309077
     },
     "base": {
      "accuracy": 0.2415277030661646,
      "ece": 0.09523090230713163
     },
     "adapter": {
      "accuracy": 0.9634211941904249,
      "ece": 0.0028617700344859593
     },
     "v1_2": {
      "rows": 5577,
      "base": {
       "accuracy": 0.851,
       "ece": 0.08
      },
      "adapter": {
       "accuracy": 0.968,
       "ece": 0.006
      }
     },
     "groups": [
      {
       "group": "bitext",
       "rows": 2361,
       "untrained": {
        "accuracy": 0.4341380770859805,
        "ece": 0.2586495471663792
       },
       "base": {
        "accuracy": 0.09995764506565015,
        "ece": 0.20073853436697564
       },
       "adapter": {
        "accuracy": 0.9966116052520119,
        "ece": 0.0020051202562811556
       }
      },
      {
       "group": "hwu64 seen by the base",
       "rows": 897,
       "untrained": {
        "accuracy": 0.19732441471571907,
        "ece": 0.08396503663334628
       },
       "base": {
        "accuracy": 0.26755852842809363,
        "ece": 0.13710387789950684
       },
       "adapter": {
        "accuracy": 0.9163879598662207,
        "ece": 0.01579237329238503
       }
      },
      {
       "group": "hwu64 unseen",
       "rows": 1624,
       "untrained": {
        "accuracy": 0.17610837438423646,
        "ece": 0.08421739124631937
       },
       "base": {
        "accuracy": 0.26908866995073893,
        "ece": 0.13859036447039005
       },
       "adapter": {
        "accuracy": 0.9316502463054187,
        "ece": 0.008265849610985473
       }
      },
      {
       "group": "snips",
       "rows": 695,
       "untrained": {
        "accuracy": 0.5755395683453237,
        "ece": 0.22275825373394015
       },
       "base": {
        "accuracy": 0.6244604316546762,
        "ece": 0.15824651088735403
       },
       "adapter": {
        "accuracy": 0.9856115107913669,
        "ece": 0.009967234149765565
       }
      }
     ]
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "emotion": {
   "run": "0.8b-emotion-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 5408,
     "options": {
      "min": 28,
      "max": 28
     },
     "untrained": {
      "accuracy": 0.1264792899408284,
      "ece": 0.045261605597098076
     },
     "base": {
      "accuracy": 0.2525887573964497,
      "ece": 0.07951819879422782
     },
     "adapter": {
      "accuracy": 0.6046597633136095,
      "ece": 0.017823078475914834
     },
     "v1_2": {
      "rows": 5408,
      "base": {
       "accuracy": 0.322,
       "ece": 0.038
      },
      "adapter": {
       "accuracy": 0.606,
       "ece": 0.02
      }
     },
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "spam": {
   "run": "0.8b-spam-20261003-0235",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 3897,
     "options": {
      "min": 2,
      "max": 3
     },
     "untrained": {
      "accuracy": 0.5814729278932512,
      "ece": 0.04740405432876967
     },
     "base": {
      "accuracy": 0.6982294072363356,
      "ece": 0.04552562317166255
     },
     "adapter": {
      "accuracy": 0.9812676417757249,
      "ece": 0.005544432685541584
     },
     "v1_2": null,
     "groups": [
      {
       "group": "choice: legitimate / phishing / spam (email)",
       "rows": 172,
       "untrained": {
        "accuracy": 0.5697674418604651,
        "ece": 0.15014094925666002
       },
       "base": {
        "accuracy": 0.5813953488372093,
        "ece": 0.08454537571807327
       },
       "adapter": {
        "accuracy": 0.9767441860465116,
        "ece": 0.025464741193604368
       }
      },
      {
       "group": "choice: legitimate / phishing / spam (sms)",
       "rows": 82,
       "untrained": {
        "accuracy": 0.8292682926829268,
        "ece": 0.3926844709618862
       },
       "base": {
        "accuracy": 0.6219512195121951,
        "ece": 0.15309013862446927
       },
       "adapter": {
        "accuracy": 0.975609756097561,
        "ece": 0.039318191680061766
       }
      },
      {
       "group": "choice: legitimate / spam / phishing (email)",
       "rows": 163,
       "untrained": {
        "accuracy": 0.3312883435582822,
        "ece": 0.20264036322079523
       },
       "base": {
        "accuracy": 0.6809815950920245,
        "ece": 0.09733329879807893
       },
       "adapter": {
        "accuracy": 0.9938650306748467,
        "ece": 0.009330106839080213
       }
      },
      {
       "group": "choice: legitimate / spam / phishing (sms)",
       "rows": 108,
       "untrained": {
        "accuracy": 0.25,
        "ece": 0.16751336315908558
       },
       "base": {
        "accuracy": 0.7037037037037037,
        "ece": 0.18680059113638403
       },
       "adapter": {
        "accuracy": 0.9351851851851852,
        "ece": 0.041826755055680856
       }
      },
      {
       "group": "choice: phishing / legitimate / spam (email)",
       "rows": 163,
       "untrained": {
        "accuracy": 0.5766871165644172,
        "ece": 0.2010449643642581
       },
       "base": {
        "accuracy": 0.6441717791411042,
        "ece": 0.10586856102106379
       },
       "adapter": {
        "accuracy": 0.9815950920245399,
        "ece": 0.01700048074749769
       }
      },
      {
       "group": "choice: phishing / legitimate / spam (sms)",
       "rows": 102,
       "untrained": {
        "accuracy": 0.8725490196078431,
        "ece": 0.329489544794706
       },
       "base": {
        "accuracy": 0.5588235294117647,
        "ece": 0.1403434097013877
       },
       "adapter": {
        "accuracy": 0.9803921568627451,
        "ece": 0.022799044038097096
       }
      },
      {
       "group": "choice: phishing / spam / legitimate (email)",
       "rows": 149,
       "untrained": {
        "accuracy": 0.5302013422818792,
        "ece": 0.13553037697558282
       },
       "base": {
        "accuracy": 0.5637583892617449,
        "ece": 0.09529908361307017
       },
       "adapter": {
        "accuracy": 0.9932885906040269,
        "ece": 0.015045849069962746
       }
      },
      {
       "group": "choice: phishing / spam / legitimate (sms)",
       "rows": 93,
       "untrained": {
        "accuracy": 0.7956989247311828,
        "ece": 0.28801219325113603
       },
       "base": {
        "accuracy": 0.7849462365591398,
        "ece": 0.2752834554976313
       },
       "adapter": {
        "accuracy": 0.989247311827957,
        "ece": 0.02535828795027831
       }
      },
      {
       "group": "choice: spam / legitimate / phishing (email)",
       "rows": 178,
       "untrained": {
        "accuracy": 0.6179775280898876,
        "ece": 0.08810158890417302
       },
       "base": {
        "accuracy": 0.5955056179775281,
        "ece": 0.09902892239955755
       },
       "adapter": {
        "accuracy": 0.9719101123595506,
        "ece": 0.025516618370369448
       }
      },
      {
       "group": "choice: spam / legitimate / phishing (sms)",
       "rows": 110,
       "untrained": {
        "accuracy": 0.8363636363636363,
        "ece": 0.2722490771647766
       },
       "base": {
        "accuracy": 0.5545454545454546,
        "ece": 0.1674359434479534
       },
       "adapter": {
        "accuracy": 0.9818181818181818,
        "ece": 0.032625162092659636
       }
      },
      {
       "group": "choice: spam / phishing / legitimate (email)",
       "rows": 168,
       "untrained": {
        "accuracy": 0.6190476190476191,
        "ece": 0.07120433579150925
       },
       "base": {
        "accuracy": 0.5952380952380952,
        "ece": 0.0918259194870443
       },
       "adapter": {
        "accuracy": 0.9821428571428571,
        "ece": 0.011812564243620892
       }
      },
      {
       "group": "choice: spam / phishing / legitimate (sms)",
       "rows": 78,
       "untrained": {
        "accuracy": 0.8461538461538461,
        "ece": 0.29091811539336415
       },
       "base": {
        "accuracy": 0.6025641025641025,
        "ece": 0.10963379882159478
       },
       "adapter": {
        "accuracy": 0.9615384615384616,
        "ece": 0.038795689438696164
       }
      },
      {
       "group": "yes/no question (email)",
       "rows": 1739,
       "untrained": {
        "accuracy": 0.6457734330074756,
        "ece": 0.09649605864805254
       },
       "base": {
        "accuracy": 0.7199539965497412,
        "ece": 0.0389466495821607
       },
       "adapter": {
        "accuracy": 0.983323749281196,
        "ece": 0.008208593366855654
       }
      },
      {
       "group": "yes/no question (sms)",
       "rows": 592,
       "untrained": {
        "accuracy": 0.31756756756756754,
        "ece": 0.23962534928655335
       },
       "base": {
        "accuracy": 0.8412162162162162,
        "ece": 0.061537688394617734
       },
       "adapter": {
        "accuracy": 0.9831081081081081,
        "ece": 0.0098992119350799
       }
      }
     ]
    },
    {
     "test": "test-v12",
     "rows": 3603,
     "options": {
      "min": 2,
      "max": 3
     },
     "untrained": {
      "accuracy": 0.5933943935609215,
      "ece": 0.05956710283207485
     },
     "base": {
      "accuracy": 0.7216208714959755,
      "ece": 0.060164741780753324
     },
     "adapter": {
      "accuracy": 0.980571745767416,
      "ece": 0.006531537137028663
     },
     "v1_2": {
      "rows": 3603,
      "base": {
       "accuracy": 0.723,
       "ece": 0.109
      },
      "adapter": {
       "accuracy": 0.9840000000000001,
       "ece": 0.008
      }
     },
     "groups": [
      {
       "group": "choice: legitimate / phishing / spam (email)",
       "rows": 144,
       "untrained": {
        "accuracy": 0.5833333333333334,
        "ece": 0.165621844909093
       },
       "base": {
        "accuracy": 0.6597222222222222,
        "ece": 0.09721108273963187
       },
       "adapter": {
        "accuracy": 0.9722222222222222,
        "ece": 0.03142699955822313
       }
      },
      {
       "group": "choice: legitimate / phishing / spam (sms)",
       "rows": 82,
       "untrained": {
        "accuracy": 0.8292682926829268,
        "ece": 0.39393439899656374
       },
       "base": {
        "accuracy": 0.6219512195121951,
        "ece": 0.15273676804251318
       },
       "adapter": {
        "accuracy": 0.975609756097561,
        "ece": 0.03936416616347464
       }
      },
      {
       "group": "choice: legitimate / spam / phishing (email)",
       "rows": 149,
       "untrained": {
        "accuracy": 0.28859060402684567,
        "ece": 0.23206036646864425
       },
       "base": {
        "accuracy": 0.7315436241610739,
        "ece": 0.13825723490967756
       },
       "adapter": {
        "accuracy": 0.9932885906040269,
        "ece": 0.008992098540966761
       }
      },
      {
       "group": "choice: legitimate / spam / phishing (sms)",
       "rows": 108,
       "untrained": {
        "accuracy": 0.25,
        "ece": 0.16894498400136077
       },
       "base": {
        "accuracy": 0.6944444444444444,
        "ece": 0.18522472884973945
       },
       "adapter": {
        "accuracy": 0.9351851851851852,
        "ece": 0.041922931196070046
       }
      },
      {
       "group": "choice: phishing / legitimate / spam (email)",
       "rows": 134,
       "untrained": {
        "accuracy": 0.7014925373134329,
        "ece": 0.1451918874575916
       },
       "base": {
        "accuracy": 0.7238805970149254,
        "ece": 0.15088149465616688
       },
       "adapter": {
        "accuracy": 0.9850746268656716,
        "ece": 0.012484833410380673
       }
      },
      {
       "group": "choice: phishing / legitimate / spam (sms)",
       "rows": 102,
       "untrained": {
        "accuracy": 0.8725490196078431,
        "ece": 0.3292715499473559
       },
       "base": {
        "accuracy": 0.5784313725490197,
        "ece": 0.10893962769343006
       },
       "adapter": {
        "accuracy": 0.9803921568627451,
        "ece": 0.022712232257374492
       }
      },
      {
       "group": "choice: phishing / spam / legitimate (email)",
       "rows": 123,
       "untrained": {
        "accuracy": 0.6422764227642277,
        "ece": 0.20715009946883892
       },
       "base": {
        "accuracy": 0.6585365853658537,
        "ece": 0.1284491689095486
       },
       "adapter": {
        "accuracy": 0.991869918699187,
        "ece": 0.021619396465186683
       }
      },
      {
       "group": "choice: phishing / spam / legitimate (sms)",
       "rows": 93,
       "untrained": {
        "accuracy": 0.7956989247311828,
        "ece": 0.3084242275882242
       },
       "base": {
        "accuracy": 0.7849462365591398,
        "ece": 0.26406956740272075
       },
       "adapter": {
        "accuracy": 0.989247311827957,
        "ece": 0.025786619616812043
       }
      },
      {
       "group": "choice: spam / legitimate / phishing (email)",
       "rows": 157,
       "untrained": {
        "accuracy": 0.6815286624203821,
        "ece": 0.14845036389488356
       },
       "base": {
        "accuracy": 0.6178343949044586,
        "ece": 0.08108638889314995
       },
       "adapter": {
        "accuracy": 0.9681528662420382,
        "ece": 0.02830461587600281
       }
      },
      {
       "group": "choice: spam / legitimate / phishing (sms)",
       "rows": 110,
       "untrained": {
        "accuracy": 0.8363636363636363,
        "ece": 0.2727434287828745
       },
       "base": {
        "accuracy": 0.5454545454545454,
        "ece": 0.18478900551257688
       },
       "adapter": {
        "accuracy": 0.9818181818181818,
        "ece": 0.03246842557490827
       }
      },
      {
       "group": "choice: spam / phishing / legitimate (email)",
       "rows": 139,
       "untrained": {
        "accuracy": 0.7266187050359713,
        "ece": 0.10371828007844502
       },
       "base": {
        "accuracy": 0.6474820143884892,
        "ece": 0.15289807408769968
       },
       "adapter": {
        "accuracy": 0.9784172661870504,
        "ece": 0.015518225104374705
       }
      },
      {
       "group": "choice: spam / phishing / legitimate (sms)",
       "rows": 78,
       "untrained": {
        "accuracy": 0.8461538461538461,
        "ece": 0.2900609402803006
       },
       "base": {
        "accuracy": 0.5897435897435898,
        "ece": 0.11638445784134023
       },
       "adapter": {
        "accuracy": 0.9615384615384616,
        "ece": 0.03703002346284038
       }
      },
      {
       "group": "yes/no question (email)",
       "rows": 1592,
       "untrained": {
        "accuracy": 0.6425879396984925,
        "ece": 0.09332178029700504
       },
       "base": {
        "accuracy": 0.7342964824120602,
        "ece": 0.04883094227432691
       },
       "adapter": {
        "accuracy": 0.9830402010050251,
        "ece": 0.008986363596460344
       }
      },
      {
       "group": "yes/no question (sms)",
       "rows": 592,
       "untrained": {
        "accuracy": 0.32263513513513514,
        "ece": 0.24149783705349867
       },
       "base": {
        "accuracy": 0.8412162162162162,
        "ece": 0.061576962491078316
       },
       "adapter": {
        "accuracy": 0.9831081081081081,
        "ece": 0.00984988460173447
       }
      }
     ]
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "nav": {
   "run": "0.8b-nav-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "heldout_synthetic",
     "rows": 3300,
     "options": {
      "min": 6,
      "max": 200
     },
     "untrained": {
      "accuracy": 0.12636363636363637,
      "ece": 0.0381989507141525
     },
     "base": {
      "accuracy": 0.13575757575757577,
      "ece": 0.15623840179780832
     },
     "adapter": {
      "accuracy": 0.9733333333333334,
      "ece": 0.006490042433820258
     },
     "v1_2": null,
     "groups": []
    },
    {
     "test": "heldout_synthetic-v12",
     "rows": 3300,
     "options": {
      "min": 6,
      "max": 201
     },
     "untrained": {
      "accuracy": 0.12303030303030303,
      "ece": 0.03707899694989147
     },
     "base": {
      "accuracy": 0.1390909090909091,
      "ece": 0.1516542479871309
     },
     "adapter": {
      "accuracy": 0.9733333333333334,
      "ece": 0.0060272813996041594
     },
     "v1_2": {
      "rows": 3300,
      "base": {
       "accuracy": 0.23800000000000002,
       "ece": 0.195
      },
      "adapter": {
       "accuracy": 0.97,
       "ece": 0.005
      }
     },
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "tools": {
   "run": "0.8b-tools-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 5157,
     "options": {
      "min": 4,
      "max": 136
     },
     "untrained": {
      "accuracy": 0.17859220477021523,
      "ece": 0.06290609077954537
     },
     "base": {
      "accuracy": 0.3015318983905371,
      "ece": 0.016429751122015616
     },
     "adapter": {
      "accuracy": 0.9722707000193911,
      "ece": 0.006943791699622538
     },
     "v1_2": {
      "rows": 5157,
      "base": {
       "accuracy": 0.578,
       "ece": 0.091
      },
      "adapter": {
       "accuracy": 0.9790000000000001,
       "ece": 0.004
      }
     },
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "guard": {
   "run": "0.8b-guard-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 6552,
     "options": {
      "min": 2,
      "max": 5
     },
     "untrained": {
      "accuracy": 0.43772893772893773,
      "ece": 0.06398512485249054
     },
     "base": {
      "accuracy": 0.4938949938949939,
      "ece": 0.1575024591009097
     },
     "adapter": {
      "accuracy": 0.9815323565323565,
      "ece": 0.004219618972013314
     },
     "v1_2": {
      "rows": 6552,
      "base": {
       "accuracy": 0.469,
       "ece": 0.28
      },
      "adapter": {
       "accuracy": 0.9840000000000001,
       "ece": 0.004
      }
     },
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "ground": {
   "run": "0.8b-ground-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 4160,
     "options": {
      "min": 4,
      "max": 41
     },
     "untrained": {
      "accuracy": 0.28870192307692305,
      "ece": 0.060628500769551674
     },
     "base": {
      "accuracy": 0.509375,
      "ece": 0.07913504769070705
     },
     "adapter": {
      "accuracy": 0.9663461538461539,
      "ece": 0.00680882300334636
     },
     "v1_2": {
      "rows": 4160,
      "base": {
       "accuracy": 0.49,
       "ece": 0.237
      },
      "adapter": {
       "accuracy": 0.97,
       "ece": 0.012
      }
     },
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "triage": {
   "run": "0.8b-triage-20261003-0149",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 7256,
     "options": {
      "min": 2,
      "max": 36
     },
     "untrained": {
      "accuracy": 0.44060088202866593,
      "ece": 0.09787958044445541
     },
     "base": {
      "accuracy": 0.47588202866593166,
      "ece": 0.058708662754586236
     },
     "adapter": {
      "accuracy": 0.9148291069459757,
      "ece": 0.015101962956236503
     },
     "v1_2": {
      "rows": 7256,
      "base": {
       "accuracy": 0.6709999999999999,
       "ece": 0.026
      },
      "adapter": {
       "accuracy": 0.9179999999999999,
       "ece": 0.009
      }
     },
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/retrained-adapters.table.json"
  },
  "sanctions": {
   "run": "0.8b-sanctions-20261003-1217",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 4909,
     "options": {
      "min": 3,
      "max": 3
     },
     "untrained": {
      "accuracy": 0.34365451212059483,
      "ece": 0.010262019907618602
     },
     "base": {
      "accuracy": 0.6826237522917091,
      "ece": 0.07968005776162645
     },
     "adapter": {
      "accuracy": 0.9995925850478713,
      "ece": 0.0010277376235659922
     },
     "v1_2": null,
     "groups": []
    }
   ],
   "source": "results/sources/v1.3/new-adapters.table.json"
  },
  "trading-desk": {
   "run": "0.8b-trading-desk-20261003-1525",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 5000,
     "options": {
      "min": 2,
      "max": 3
     },
     "untrained": {
      "accuracy": 0.3858,
      "ece": 0.01489035542520305
     },
     "base": {
      "accuracy": 0.4106,
      "ece": 0.114698115014766
     },
     "adapter": {
      "accuracy": 0.9814,
      "ece": 0.009943304474960767
     },
     "v1_2": null,
     "groups": [
      {
       "group": "decision: direction",
       "rows": 1000,
       "untrained": {
        "accuracy": 0.358,
        "ece": 0.012380824757211861
       },
       "base": {
        "accuracy": 0.349,
        "ece": 0.1610522695722192
       },
       "adapter": {
        "accuracy": 0.999,
        "ece": 0.002493643930658213
       }
      },
      {
       "group": "decision: order",
       "rows": 1000,
       "untrained": {
        "accuracy": 0.341,
        "ece": 0.01707430466230353
       },
       "base": {
        "accuracy": 0.371,
        "ece": 0.2519168204848874
       },
       "adapter": {
        "accuracy": 1,
        "ece": 0.00027392565499750887
       }
      },
      {
       "group": "decision: risk_off",
       "rows": 1000,
       "untrained": {
        "accuracy": 0.557,
        "ece": 0.04531037271605831
       },
       "base": {
        "accuracy": 0.632,
        "ece": 0.025327032972125418
       },
       "adapter": {
        "accuracy": 1,
        "ece": 0.000280211364184324
       }
      },
      {
       "group": "decision: route",
       "rows": 1000,
       "untrained": {
        "accuracy": 0.34,
        "ece": 0.001748072849023119
       },
       "base": {
        "accuracy": 0.328,
        "ece": 0.08712384460222625
       },
       "adapter": {
        "accuracy": 0.953,
        "ece": 0.018601892391565397
       }
      },
      {
       "group": "decision: size",
       "rows": 1000,
       "untrained": {
        "accuracy": 0.333,
        "ece": 0.022699851655842318
       },
       "base": {
        "accuracy": 0.373,
        "ece": 0.08003148602905057
       },
       "adapter": {
        "accuracy": 0.955,
        "ece": 0.03354186206786604
       }
      }
     ]
    }
   ],
   "source": "results/sources/v1.3/new-adapters.table.json"
  },
  "soc": {
   "run": "0.8b-soc-20261003-1942",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 4929,
     "options": {
      "min": 4,
      "max": 5
     },
     "untrained": {
      "accuracy": 0.22134307161696085,
      "ece": 0.011299265591491197
     },
     "base": {
      "accuracy": 0.33840535605599514,
      "ece": 0.031898361529129594
     },
     "adapter": {
      "accuracy": 0.9411645364171232,
      "ece": 0.013609359267820732
     },
     "v1_2": null,
     "groups": [
      {
       "group": "alert source: endpoint",
       "rows": 1500,
       "untrained": {
        "accuracy": 0.23,
        "ece": 0.02371933538348683
       },
       "base": {
        "accuracy": 0.3466666666666667,
        "ece": 0.03209083117858784
       },
       "adapter": {
        "accuracy": 0.9626666666666667,
        "ece": 0.017908501934457102
       }
      },
      {
       "group": "alert source: insider",
       "rows": 1429,
       "untrained": {
        "accuracy": 0.2225332400279916,
        "ece": 0.015250576573020222
       },
       "base": {
        "accuracy": 0.2967109867039888,
        "ece": 0.058614057079479594
       },
       "adapter": {
        "accuracy": 0.8782365290412876,
        "ece": 0.01905055930639796
       }
      },
      {
       "group": "alert source: network",
       "rows": 2000,
       "untrained": {
        "accuracy": 0.214,
        "ece": 0.010939854994703941
       },
       "base": {
        "accuracy": 0.362,
        "ece": 0.09575521477920258
       },
       "adapter": {
        "accuracy": 0.97,
        "ece": 0.01110764495970884
       }
      },
      {
       "group": "label: block",
       "rows": 673,
       "untrained": {
        "accuracy": 0.3447251114413076,
        "ece": 0.12101302606758772
       },
       "base": {
        "accuracy": 0.3447251114413076,
        "ece": 0.06716116300122903
       },
       "adapter": {
        "accuracy": 0.9539375928677564,
        "ece": 0.01868424453231881
       }
      },
      {
       "group": "label: escalate",
       "rows": 1085,
       "untrained": {
        "accuracy": 0.3244239631336406,
        "ece": 0.09136028805178445
       },
       "base": {
        "accuracy": 0.1585253456221198,
        "ece": 0.13350782749516518
       },
       "adapter": {
        "accuracy": 0.9520737327188941,
        "ece": 0.03387379371362807
       }
      },
      {
       "group": "label: ignore",
       "rows": 1165,
       "untrained": {
        "accuracy": 0.06866952789699571,
        "ece": 0.16215495023395815
       },
       "base": {
        "accuracy": 0.6377682403433477,
        "ece": 0.2860521722982204
       },
       "adapter": {
        "accuracy": 0.9201716738197425,
        "ece": 0.02341114659738691
       }
      },
      {
       "group": "label: investigate",
       "rows": 1237,
       "untrained": {
        "accuracy": 0.18593371059013744,
        "ece": 0.050612482924153525
       },
       "base": {
        "accuracy": 0.15198059822150364,
        "ece": 0.14788056840690308
       },
       "adapter": {
        "accuracy": 0.9248181083265966,
        "ece": 0.015934144051085005
       }
      },
      {
       "group": "label: isolate",
       "rows": 769,
       "untrained": {
        "accuracy": 0.25617685305591675,
        "ece": 0.027733891938512528
       },
       "base": {
        "accuracy": 0.43302990897269183,
        "ece": 0.18742143543589254
       },
       "adapter": {
        "accuracy": 0.9726918075422627,
        "ece": 0.026021208155593115
       }
      }
     ]
    }
   ],
   "source": "results/sources/v1.3/new-adapters.table.json"
  },
  "aml": {
   "run": "0.8b-aml-v2-20261004-0713",
   "adapter_bytes": 41459776,
   "tests": [
    {
     "test": "test",
     "rows": 5120,
     "options": {
      "min": 2,
      "max": 8
     },
     "untrained": {
      "accuracy": 0.36015625,
      "ece": 0.022105448833337274
     },
     "base": {
      "accuracy": 0.4052734375,
      "ece": 0.10304039761207712
     },
     "adapter": {
      "accuracy": 0.9501953125,
      "ece": 0.01191735409237603
     },
     "v1_2": null,
     "groups": [
      {
       "group": "question: action",
       "rows": 2386,
       "untrained": {
        "accuracy": 0.2732606873428332,
        "ece": 0.00856262098903864
       },
       "base": {
        "accuracy": 0.22380553227158423,
        "ece": 0.17365571131772503
       },
       "adapter": {
        "accuracy": 0.940905280804694,
        "ece": 0.010226042576831417
       }
      },
      {
       "group": "question: pattern",
       "rows": 348,
       "untrained": {
        "accuracy": 0.13793103448275862,
        "ece": 0.004270092428096821
       },
       "base": {
        "accuracy": 0.1781609195402299,
        "ece": 0.024957911726174467
       },
       "adapter": {
        "accuracy": 0.9971264367816092,
        "ece": 0.005754187679714456
       }
      },
      {
       "group": "question: suspicious",
       "rows": 2386,
       "untrained": {
        "accuracy": 0.47946353730092206,
        "ece": 0.03824957761184533
       },
       "base": {
        "accuracy": 0.6198658843252305,
        "ece": 0.04551253252852873
       },
       "adapter": {
        "accuracy": 0.9526404023470243,
        "ece": 0.016224937160150034
       }
      }
     ]
    },
    {
     "test": "test-amlworld-v1",
     "rows": 4500,
     "options": {
      "min": 2,
      "max": 2
     },
     "untrained": {
      "accuracy": 0.48933333333333334,
      "ece": 0.02327642562842963
     },
     "base": {
      "accuracy": 0.6426666666666667,
      "ece": 0.04394163979324305
     },
     "adapter": {
      "accuracy": 0.6002222222222222,
      "ece": 0.2435080222824059
     },
     "v1_2": null,
     "groups": [
      {
       "group": "question: amlworld_window",
       "rows": 4500,
       "untrained": {
        "accuracy": 0.48933333333333334,
        "ece": 0.02327642562842963
       },
       "base": {
        "accuracy": 0.6426666666666667,
        "ece": 0.04394163979324305
       },
       "adapter": {
        "accuracy": 0.6002222222222222,
        "ece": 0.2435080222824059
       }
      }
     ]
    }
   ],
   "source": "results/sources/v1.3/new-adapters.table.json"
  }
 },
 "gguf": {
  "llama_cpp_commit": "cb7934c52ca8710994b2ecc19775ebefcfdb8d01",
  "files": [
   {
    "file": "loras/aml.gguf",
    "bytes": 168603072
   },
   {
    "file": "loras/code-router.gguf",
    "bytes": 168603072
   },
   {
    "file": "loras/code.gguf",
    "bytes": 168603072
   },
   {
    "file": "loras/emotion.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/ground.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/guard.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/intrusion-v2.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/legal-clauses.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/nav.gguf",
    "bytes": 168603072
   },
   {
    "file": "loras/sanctions.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/soc.gguf",
    "bytes": 168603072
   },
   {
    "file": "loras/spam.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/support-intents.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/tools.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/trading-desk.gguf",
    "bytes": 168603104
   },
   {
    "file": "loras/triage.gguf",
    "bytes": 168603104
   },
   {
    "file": "models/base-q4_k_m.gguf",
    "bytes": 672329312
   },
   {
    "file": "models/base-q8_0.gguf",
    "bytes": 1082015328
   }
  ],
  "adapters": [
   {
    "adapter": "emotion",
    "test": "test",
    "split": "test",
    "rows": 5408,
    "float": {
     "accuracy": 0.6046597633136095,
     "ece": 0.017823078475914834
    },
    "q8_0": {
     "accuracy": 0.6009615384615384,
     "ece": 0.011982500410557753
    },
    "q4_k_m": {
     "accuracy": 0.6022559171597633,
     "ece": 0.017232059536928173
    },
    "float_source": null
   },
   {
    "adapter": "ground",
    "test": "test",
    "split": "test",
    "rows": 4160,
    "float": {
     "accuracy": 0.9663461538461539,
     "ece": 0.00680882300334636
    },
    "q8_0": {
     "accuracy": 0.9661057692307692,
     "ece": 0.008980308970886596
    },
    "q4_k_m": {
     "accuracy": 0.963701923076923,
     "ece": 0.010131548476852872
    },
    "float_source": null
   },
   {
    "adapter": "guard",
    "test": "test",
    "split": "test",
    "rows": 6552,
    "float": {
     "accuracy": 0.9815323565323565,
     "ece": 0.004219618972013314
    },
    "q8_0": {
     "accuracy": 0.9816849816849816,
     "ece": 0.00375631970955116
    },
    "q4_k_m": {
     "accuracy": 0.9806166056166056,
     "ece": 0.0024962549781453237
    },
    "float_source": null
   },
   {
    "adapter": "legal-clauses",
    "test": "test",
    "split": "test",
    "rows": 9895,
    "float": {
     "accuracy": 0.8363820111167256,
     "ece": 0.010929339202633614
    },
    "q8_0": {
     "accuracy": 0.8377968671045983,
     "ece": 0.010733964126753802
    },
    "q4_k_m": {
     "accuracy": 0.8335522991409803,
     "ece": 0.010187670963920486
    },
    "float_source": null
   },
   {
    "adapter": "legal-clauses",
    "test": "test-novel",
    "split": "test",
    "rows": 7357,
    "float": {
     "accuracy": 0.8143264917765394,
     "ece": 0.014832325678416778
    },
    "q8_0": {
     "accuracy": 0.8155498165012913,
     "ece": 0.015389610806925275
    },
    "q4_k_m": {
     "accuracy": 0.812423542204703,
     "ece": 0.015425355664125258
    },
    "float_source": null
   },
   {
    "adapter": "nav",
    "test": "heldout_synthetic",
    "split": "test",
    "rows": 3300,
    "float": {
     "accuracy": 0.9733333333333334,
     "ece": 0.006490042433820258
    },
    "q8_0": {
     "accuracy": 0.9733333333333334,
     "ece": 0.006812013583884202
    },
    "q4_k_m": {
     "accuracy": 0.9736363636363636,
     "ece": 0.009197052161908224
    },
    "float_source": null
   },
   {
    "adapter": "nav",
    "test": "heldout_synthetic-v12",
    "split": "test",
    "rows": 3300,
    "float": {
     "accuracy": 0.9733333333333334,
     "ece": 0.0060272813996041594
    },
    "q8_0": {
     "accuracy": 0.9733333333333334,
     "ece": 0.007798014024974912
    },
    "q4_k_m": {
     "accuracy": 0.973030303030303,
     "ece": 0.007651213484835406
    },
    "float_source": null
   },
   {
    "adapter": "spam",
    "test": "test",
    "split": "test",
    "rows": 3897,
    "float": {
     "accuracy": 0.9812676417757249,
     "ece": 0.005544432685541584
    },
    "q8_0": {
     "accuracy": 0.9810110341288171,
     "ece": 0.004017224671908634
    },
    "q4_k_m": {
     "accuracy": 0.9781883500128303,
     "ece": 0.005357673517108378
    },
    "float_source": null
   },
   {
    "adapter": "spam",
    "test": "test-v12",
    "split": "test",
    "rows": 3603,
    "float": {
     "accuracy": 0.980571745767416,
     "ece": 0.006531537137028663
    },
    "q8_0": {
     "accuracy": 0.9802941992783791,
     "ece": 0.004506030554021337
    },
    "q4_k_m": {
     "accuracy": 0.9764085484318623,
     "ece": 0.006552053891551335
    },
    "float_source": null
   },
   {
    "adapter": "support-intents",
    "test": "test",
    "split": "test",
    "rows": 5577,
    "float": {
     "accuracy": 0.9634211941904249,
     "ece": 0.0028617700344859593
    },
    "q8_0": {
     "accuracy": 0.9634211941904249,
     "ece": 0.005739304334904006
    },
    "q4_k_m": {
     "accuracy": 0.9618074233458849,
     "ece": 0.004815255279826123
    },
    "float_source": null
   },
   {
    "adapter": "tools",
    "test": "test",
    "split": "test",
    "rows": 5157,
    "float": {
     "accuracy": 0.9722707000193911,
     "ece": 0.006943791699622538
    },
    "q8_0": {
     "accuracy": 0.9720767888307156,
     "ece": 0.005149080196386522
    },
    "q4_k_m": {
     "accuracy": 0.9705254993213108,
     "ece": 0.010300712373231036
    },
    "float_source": null
   },
   {
    "adapter": "triage",
    "test": "test",
    "split": "test",
    "rows": 7256,
    "float": {
     "accuracy": 0.9148291069459757,
     "ece": 0.015101962956236503
    },
    "q8_0": {
     "accuracy": 0.9148291069459757,
     "ece": 0.015082111304974477
    },
    "q4_k_m": {
     "accuracy": 0.9122105843439912,
     "ece": 0.014211492208869475
    },
    "float_source": null
   },
   {
    "adapter": "trading-desk",
    "test": "test",
    "split": "test",
    "rows": 5000,
    "float": {
     "accuracy": 0.9814,
     "ece": 0.009943304474960767
    },
    "q8_0": {
     "accuracy": 0.9812,
     "ece": 0.009795993442581696
    },
    "q4_k_m": {
     "accuracy": 0.978,
     "ece": 0.010770269978213403
    },
    "float_source": null
   },
   {
    "adapter": "sanctions",
    "test": "test",
    "split": "test",
    "rows": 4909,
    "float": {
     "accuracy": 0.9995925850478713,
     "ece": 0.0010277376235659922
    },
    "q8_0": {
     "accuracy": 0.9995925850478713,
     "ece": 0.001239407874421216
    },
    "q4_k_m": {
     "accuracy": 0.9989814626196781,
     "ece": 0.0014719844545841645
    },
    "float_source": null
   },
   {
    "adapter": "soc",
    "test": "test",
    "split": "test",
    "rows": 4929,
    "float": {
     "accuracy": 0.9411645364171232,
     "ece": 0.013609359267820732
    },
    "q8_0": {
     "accuracy": 0.9395414891458713,
     "ece": 0.013153477851619372
    },
    "q4_k_m": {
     "accuracy": 0.9367011564211808,
     "ece": 0.009382341025025531
    },
    "float_source": null
   },
   {
    "adapter": "aml",
    "test": "test",
    "split": "test",
    "rows": 5120,
    "float": {
     "accuracy": 0.9501953125,
     "ece": 0.01191735409237603
    },
    "q8_0": {
     "accuracy": 0.9498046875,
     "ece": 0.01244638380079681
    },
    "q4_k_m": {
     "accuracy": 0.946484375,
     "ece": 0.014923798381059679
    },
    "float_source": null
   },
   {
    "adapter": "aml",
    "test": "test-amlworld-v1",
    "split": "test",
    "rows": 4500,
    "float": {
     "accuracy": 0.6002222222222222,
     "ece": 0.2435080222824059
    },
    "q8_0": {
     "accuracy": 0.5975555555555555,
     "ece": 0.24556798797277518
    },
    "q4_k_m": {
     "accuracy": 0.6068888888888889,
     "ece": 0.22149966262846854
    },
    "float_source": null
   },
   {
    "adapter": "code",
    "test": "development",
    "split": "development",
    "rows": 991,
    "float": {
     "accuracy": 0.9041372351160444,
     "ece": 0.022776550248966427
    },
    "q8_0": {
     "accuracy": 0.9041372351160444,
     "ece": 0.029625917143096685
    },
    "q4_k_m": {
     "accuracy": 0.9101917255297679,
     "ece": 0.028540085809215427
    },
    "float_source": "full precision from the trainer's own evaluation on the development split"
   },
   {
    "adapter": "code-router",
    "test": "development",
    "split": "development",
    "rows": 987,
    "float": {
     "accuracy": 0.6443768996960486,
     "ece": 0.06283172167507822
    },
    "q8_0": {
     "accuracy": 0.6443768996960486,
     "ece": 0.0625095464441003
    },
    "q4_k_m": {
     "accuracy": 0.6382978723404256,
     "ece": 0.06695919412990656
    },
    "float_source": "full precision from the trainer's own evaluation on the development split"
   }
  ],
  "code_rules": {
   "code": {
    "q8_0": 0.9989909182643795,
    "q4_k_m": 0.9788092835519677,
    "rows": 991
   },
   "router": {
    "q8_0": 0.9918946301925026,
    "q4_k_m": 0.9432624113475178,
    "rows": 987
   }
  },
  "base_alone": [
   {
    "test": "panel",
    "rows": 4599,
    "float": {
     "accuracy": 0.7859999999999999,
     "ece": 0.028
    },
    "q8_0": {
     "accuracy": 0.7859999999999999,
     "ece": 0.025
    },
    "q4_k_m": {
     "accuracy": 0.7809999999999999,
     "ece": 0.02
    }
   },
   {
    "test": "jevbench-hard",
    "rows": 105,
    "float": {
     "accuracy": 0.47600000000000003,
     "ece": 0.171
    },
    "q8_0": {
     "accuracy": 0.47600000000000003,
     "ece": 0.203
    },
    "q4_k_m": {
     "accuracy": 0.514,
     "ece": 0.15
    }
   },
   {
    "test": "documents",
    "rows": 2009,
    "float": {
     "accuracy": 0.6559999999999999,
     "ece": 0.077
    },
    "q8_0": {
     "accuracy": 0.653,
     "ece": 0.076
    },
    "q4_k_m": {
     "accuracy": 0.64,
     "ece": 0.07
    }
   },
   {
    "test": "voice",
    "rows": 3324,
    "float": {
     "accuracy": 0.898,
     "ece": 0.058
    },
    "q8_0": {
     "accuracy": 0.899,
     "ece": 0.059
    },
    "q4_k_m": {
     "accuracy": 0.893,
     "ece": 0.074
    }
   },
   {
    "test": "longlists-v2",
    "rows": 1886,
    "float": {
     "accuracy": 0.934,
     "ece": 0.013
    },
    "q8_0": {
     "accuracy": 0.934,
     "ece": 0.018
    },
    "q4_k_m": {
     "accuracy": 0.9309999999999999,
     "ece": 0.019
    }
   }
  ],
  "serving": [
   {
    "device": "one H100 (GPU 0), CUDA build",
    "way": "library (jeff-logits)",
    "interleaved_ms": 53.9,
    "grouped_ms": 42.6,
    "switching_ms": 11.3
   },
   {
    "device": "one H100 (GPU 0), CUDA build",
    "way": "llama-server",
    "interleaved_ms": 94.7,
    "grouped_ms": 74.9,
    "switching_ms": 19.8
   },
   {
    "device": "CPU, 16 threads",
    "way": "library (jeff-logits)",
    "interleaved_ms": 1511.5,
    "grouped_ms": 1344.7,
    "switching_ms": 166.8
   },
   {
    "device": "CPU, 16 threads",
    "way": "llama-server",
    "interleaved_ms": 1275.9,
    "grouped_ms": 1281.4,
    "switching_ms": -5.5
   }
  ],
  "source": "results/sources/v1.3/gguf-results-dynamic.json, gguf-results-new.json, gguf-code-rules.json, gguf-RESULTS.md, gguf-file-sizes.txt"
 },
 "jeff_code": {
  "about": "Jeff-Code results supplied by the owner on 2026-10-05, as numbers: the final evaluation report of 18:23 (jeff-pi results/imitation/eval-tonight.md). Not copied from a result file. The per-benchmark rows come from the same report (arms a1-baseline and a4-jeff06, commit bd44418be), handed over by the jeff-dev session on 2026-10-05: SWE-rebench and Terminal-Bench Pro ran twice, and each has its two rounds combined into one row, with intervals resampling whole tasks over both rounds. Both arms run the Jeff-Code agent (a fork of Pi) with Qwen3.8-27B; one with Jeff's two LoRA adapters on the fixed Jeff v1.3 base (code: information-gathering steps ahead of Qwen; code-router: whether Qwen thinks hard on a turn), one with Qwen alone in the same build with every Jeff feature off, thinking at full on every turn and no thinking limit.",
  "adapters": {
   "steps": "code",
   "router": "code-router"
  },
  "supplied_by": "owner",
  "date": "2026-10-05",
  "status": "final",
  "thresholds": {
   "step": 0.4,
   "thinking": 0.6,
   "thinking_rule": "thinking off unless P(xhigh), out of the four levels off/low/medium/xhigh, >= 0.6"
  },
  "benchmarks": [
   "SWE-bench Verified",
   "SWE-rebench",
   "Terminal-Bench Pro",
   "Terminal-Bench 2.0",
   "SkillsBench",
   "Harbor Index"
  ],
  "paired_tasks": 1242,
  "design": "each task run in both settings side by side, at the same time on the same Qwen server, and compared pair by pair",
  "pass_rate": {
   "with_jeff": 0.624,
   "qwen_alone": 0.628,
   "paired_difference_points": -0.2,
   "ci95_points": [
    -2.6,
    2.1
   ]
  },
  "time": {
   "per_task": {
    "ratio": 0.68,
    "ci95": [
     0.64,
     0.72
    ]
   },
   "median_task_ratio": 0.7,
   "total": {
    "ratio": 0.86,
    "ci95": [
     0.8,
     0.93
    ]
   },
   "by_benchmark": [
    {
     "benchmark": "SWE-bench Verified",
     "rounds": 1,
     "paired_tasks": 486,
     "time": {
      "ratio": 0.63,
      "ci95": [
       0.57,
       0.69
      ]
     },
     "pass": {
      "qwen_alone": 0.706,
      "with_jeff": 0.708,
      "difference_points": 0.4,
      "ci95_points": [
       -3.5,
       4.3
      ]
     }
    },
    {
     "benchmark": "SWE-rebench",
     "rounds": 2,
     "paired_tasks": 370,
     "time": {
      "ratio": 0.66,
      "ci95": [
       0.61,
       0.73
      ]
     },
     "pass": {
      "qwen_alone": 0.589,
      "with_jeff": 0.581,
      "difference_points": -0.8,
      "ci95_points": [
       -5.7,
       3.5
      ]
     }
    },
    {
     "benchmark": "Terminal-Bench Pro",
     "rounds": 2,
     "paired_tasks": 195,
     "time": {
      "ratio": 0.64,
      "ci95": [
       0.55,
       0.76
      ]
     },
     "pass": {
      "qwen_alone": 0.612,
      "with_jeff": 0.628,
      "difference_points": 1.5,
      "ci95_points": [
       -4.6,
       7.7
      ]
     }
    },
    {
     "benchmark": "Terminal-Bench 2.0",
     "rounds": 1,
     "paired_tasks": 108,
     "time": {
      "ratio": 0.96,
      "ci95": [
       0.78,
       1.16
      ]
     },
     "pass": {
      "qwen_alone": 0.759,
      "with_jeff": 0.7,
      "difference_points": -4.6,
      "ci95_points": [
       -12.1,
       3.7
      ]
     }
    },
    {
     "benchmark": "SkillsBench",
     "rounds": 1,
     "paired_tasks": 42,
     "time": {
      "ratio": 0.91,
      "ci95": [
       0.68,
       1.2
      ]
     },
     "pass": {
      "qwen_alone": 0.286,
      "with_jeff": 0.31,
      "difference_points": 2.4,
      "ci95_points": [
       -11.9,
       16.7
      ]
     }
    },
    {
     "benchmark": "Harbor Index",
     "rounds": 1,
     "paired_tasks": 41,
     "time": {
      "ratio": 0.71,
      "ci95": [
       0.51,
       0.99
      ]
     },
     "pass": {
      "qwen_alone": 0.122,
      "with_jeff": 0.098,
      "difference_points": -2.4,
      "ci95_points": [
       -12.2,
       7.3
      ]
     }
    }
   ]
  },
  "thinking_off": {
   "points": -7.6,
   "ci95_points": [
    -10.6,
    -4.5
   ],
   "worst": {
    "benchmark": "Terminal-Bench 2.0",
    "points": -13.5
   },
   "note": "Thinking off throughout is faster still but clearly worse: Jeff's decisions are what keep the quality."
  },
  "looser_threshold": {
   "thinking": 0.7,
   "points": -4.5
  },
  "long_tasks": {
   "share": 0.05,
   "minutes_longer": 30,
   "solved_with_jeff": 26,
   "solved_alone": 24
  },
  "excluded": {
   "pairs": 27,
   "reason": "where the system killed a session (for example, out of memory), plus the few sessions still running when the report was made"
  },
  "offline_accuracy": {
   "step": 0.92,
   "router": 0.68,
   "on": "held-out scoring tasks",
   "source": "reported by the Jeff-Code session: practically the same as the full fine-tune compared offline (step 92%, router 68% for both)"
  },
  "source": "results/sources/v1.3/jeff-code-owner-supplied.json (owner-supplied)"
 },
 "memory": {
  "device": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition",
  "base_gb": 1.737057792,
  "with_adapters": [
   {
    "adapters": [
     "guard",
     "tools",
     "ground"
    ],
    "gb": 1.83352064,
    "median_ms": 31.910685007460415
   },
   {
    "adapters": [
     "emotion",
     "ground",
     "guard",
     "legal-clauses",
     "nav",
     "spam",
     "support-intents",
     "tools",
     "triage",
     "trading-desk",
     "aml",
     "sanctions",
     "soc",
     "code",
     "code-router"
    ],
    "gb": 2.085154304,
    "median_ms": 32.119691953994334
   }
  ],
  "source": "results/sources/v1.3/memory-v1.3-three-adapters.json, memory-v1.3-fifteen-adapters.json"
 },
 "speed_gpu": {
  "device": "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition",
  "jeff_runs": [
   "fast-v13-r2"
  ],
  "qwen": "Qwen3.8-27B, NVFP4 weights, vLLM, prefix caching off, thinking off, one request at a time",
  "tasks": [
   {
    "task": "guard",
    "prompt_tokens": 305,
    "jeff_ms": 31.065989984199405,
    "qwen_ms": 120.86065988522023,
    "qwen_rows": 100
   },
   {
    "task": "triage",
    "prompt_tokens": 295,
    "jeff_ms": 30.90202691964805,
    "qwen_ms": 120.54731434211135,
    "qwen_rows": 100
   },
   {
    "task": "support-intents",
    "prompt_tokens": 1091,
    "jeff_ms": 31.288221012800932,
    "qwen_ms": 159.44447937887162,
    "qwen_rows": 100
   },
   {
    "task": "tools",
    "prompt_tokens": 1302,
    "jeff_ms": 32.477544038556516,
    "qwen_ms": 209.1925422893837,
    "qwen_rows": 100
   },
   {
    "task": "ground",
    "prompt_tokens": 804,
    "jeff_ms": 32.59990701917559,
    "qwen_ms": 237.5625392282003,
    "qwen_rows": 123
   },
   {
    "task": "nav",
    "prompt_tokens": 946,
    "jeff_ms": 32.13568194769323,
    "qwen_ms": 189.22198957763612,
    "qwen_rows": 100
   },
   {
    "task": "emotion",
    "prompt_tokens": 590,
    "jeff_ms": 31.609109952114522,
    "qwen_ms": 146.09270483022556,
    "qwen_rows": 100
   },
   {
    "task": "spam",
    "prompt_tokens": 255,
    "jeff_ms": 30.628912965767086,
    "qwen_ms": 123.08650280232541,
    "qwen_rows": 100
   },
   {
    "task": "legal-clauses",
    "prompt_tokens": 2574,
    "jeff_ms": 37.953450926579535,
    "qwen_ms": 325.5783415073529,
    "qwen_rows": 100
   }
  ],
  "source": "results/sources/v1.3/latency-jeff-rtx.json, latency-qwen27b-rtx.json, prompt-tokens-jeff.json"
 },
 "pending": [
  "code: test scores (untrained, v1.3 base, v1.3 + adapter)",
  "code-router: test scores (untrained, v1.3 base, v1.3 + adapter)",
  "jeff-serve speed with prompt reuse",
  "Calibration charts, commonest confusions and accuracy per answer for each v1.3 adapter",
  "Example responses recorded from the v1.3 adapters",
  "Jeff-Code: the detailed result files behind the owner-supplied numbers (time per session)",
  "Jeff's memory on a Mac"
 ],
 "sources": {
  "retrained_adapters": "results/sources/v1.3/retrained-adapters.table.json",
  "new_adapters": "results/sources/v1.3/new-adapters.table.json",
  "general_panel": "results/sources/v1.3/panel-v1.3-base.json, panel-v1.2-base.json",
  "gguf": "results/sources/v1.3/gguf-results-dynamic.json, gguf-results-new.json, gguf-code-rules.json, gguf-RESULTS.md",
  "adapter_sizes": "results/sources/v1.3/adapter-sizes.json",
  "option_counts": "results/sources/v1.3/option-counts.json",
  "jeff_code": "results/sources/v1.3/jeff-code-owner-supplied.json (supplied by the owner)",
  "memory": "results/sources/v1.3/memory-v1.3-three-adapters.json, memory-v1.3-fifteen-adapters.json",
  "speed_gpu": "results/sources/v1.3/latency-jeff-rtx.json, latency-qwen27b-rtx.json, prompt-tokens-jeff.json"
 }
}