| { |
| "objective": -0.5460977168843815, |
| "paired_flip": { |
| "pairs": 80, |
| "flip_rate": 0.55, |
| "both_correct_rate": 0.45, |
| "invariant_pairs": 32, |
| "invariance_rate": 0.9375, |
| "invariant_both_correct_rate": 0.46875 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 1264, |
| "nll": 0.5215492407482734, |
| "acc": 0.7943037974683544, |
| "ece": 0.04048827875950509, |
| "brier": 0.27717776074983425, |
| "mean_conf": 0.8284767038904833, |
| "confident_error_rate": 0.028481012658227847, |
| "coverage_at_0_9": 0.5561708860759493, |
| "accuracy_at_0_9": 0.9487908961593172, |
| "coverage_at_5pct_error": 0.555379746835443, |
| "coverage_at_1pct_error": 0.006329113924050633, |
| "aurc": 0.07067902977682425, |
| "error_rate_at_0_9": 0.051209103840682786, |
| "confidence_bias": 0.03417290642212889, |
| "top_bins": { |
| "0.9": { |
| "n": 703, |
| "errors": 36, |
| "error_rate": 0.051209103840682786 |
| }, |
| "0.95": { |
| "n": 646, |
| "errors": 27, |
| "error_rate": 0.04179566563467492 |
| }, |
| "0.99": { |
| "n": 512, |
| "errors": 13, |
| "error_rate": 0.025390625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9588607594936709, |
| "confidence_cutoff": 0.961561151958084 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.866600790513834, |
| "confidence_cutoff": 0.5841912218049162 |
| } |
| }, |
| "score_mae": 0.4606092452508017, |
| "ranked_probability_score": 0.07134033027857928 |
| }, |
| "tasks": { |
| "agnews": { |
| "n": 80, |
| "nll": 0.7253619615276804, |
| "acc": 0.825, |
| "ece": 0.12435274294907851, |
| "brier": 0.29748699997093375, |
| "mean_conf": 0.9467044212167703, |
| "confident_error_rate": 0.1125, |
| "coverage_at_0_9": 0.8375, |
| "accuracy_at_0_9": 0.8656716417910447, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.025, |
| "aurc": 0.0891676305179602, |
| "error_rate_at_0_9": 0.13432835820895522, |
| "confidence_bias": 0.12170442121677039, |
| "top_bins": { |
| "0.9": { |
| "n": 67, |
| "errors": 9, |
| "error_rate": 0.13432835820895522 |
| }, |
| "0.95": { |
| "n": 60, |
| "errors": 5, |
| "error_rate": 0.08333333333333333 |
| }, |
| "0.99": { |
| "n": 47, |
| "errors": 3, |
| "error_rate": 0.06382978723404255 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.9968383335724911 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.9297638903048708 |
| } |
| } |
| }, |
| "agnews_yn": { |
| "n": 160, |
| "nll": 0.27274937177459047, |
| "acc": 0.8875, |
| "ece": 0.07205353407763021, |
| "brier": 0.16638284259750016, |
| "mean_conf": 0.9492791941714842, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.81875, |
| "accuracy_at_0_9": 0.9694656488549618, |
| "coverage_at_5pct_error": 0.8375, |
| "coverage_at_1pct_error": 0.625, |
| "aurc": 0.019553552474504545, |
| "error_rate_at_0_9": 0.030534351145038167, |
| "confidence_bias": 0.06177919417148425, |
| "top_bins": { |
| "0.9": { |
| "n": 131, |
| "errors": 4, |
| "error_rate": 0.030534351145038167 |
| }, |
| "0.95": { |
| "n": 122, |
| "errors": 3, |
| "error_rate": 0.02459016393442623 |
| }, |
| "0.99": { |
| "n": 97, |
| "errors": 1, |
| "error_rate": 0.010309278350515464 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9946307004104364 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9765625, |
| "confidence_cutoff": 0.9334476480940993 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 0.9079064857116548, |
| "acc": 0.675, |
| "ece": 0.09082075584790608, |
| "brier": 0.48731382188074246, |
| "mean_conf": 0.6600554720427965, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.1125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.2375, |
| "aurc": 0.19069566998937534, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.01494452795720358, |
| "top_bins": { |
| "0.9": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.6450436740808796 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.71875, |
| "confidence_cutoff": 0.48814962988854316 |
| } |
| }, |
| "score_mae": 0.4846846425037448, |
| "ranked_probability_score": 0.07588150660688607 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.6355525688857636, |
| "acc": 0.8375, |
| "ece": 0.07048704096779947, |
| "brier": 0.24473580199909098, |
| "mean_conf": 0.8527902873638666, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.65, |
| "accuracy_at_0_9": 0.9615384615384616, |
| "coverage_at_5pct_error": 0.65, |
| "coverage_at_1pct_error": 0.5625, |
| "aurc": 0.03531804637364592, |
| "error_rate_at_0_9": 0.038461538461538464, |
| "confidence_bias": 0.015290287363866617, |
| "top_bins": { |
| "0.9": { |
| "n": 52, |
| "errors": 2, |
| "error_rate": 0.038461538461538464 |
| }, |
| "0.95": { |
| "n": 47, |
| "errors": 1, |
| "error_rate": 0.02127659574468085 |
| }, |
| "0.99": { |
| "n": 32, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9819397006054039 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.64788273781478 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.5330866725434655, |
| "acc": 0.7875, |
| "ece": 0.11078594898530592, |
| "brier": 0.32181604785225043, |
| "mean_conf": 0.8668040159586322, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.5875, |
| "accuracy_at_0_9": 0.8936170212765957, |
| "coverage_at_5pct_error": 0.35, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.10319650984358723, |
| "error_rate_at_0_9": 0.10638297872340426, |
| "confidence_bias": 0.07930401595863223, |
| "top_bins": { |
| "0.9": { |
| "n": 47, |
| "errors": 5, |
| "error_rate": 0.10638297872340426 |
| }, |
| "0.95": { |
| "n": 40, |
| "errors": 4, |
| "error_rate": 0.1 |
| }, |
| "0.99": { |
| "n": 17, |
| "errors": 1, |
| "error_rate": 0.058823529411764705 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.960176618093949 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.7061129387549795 |
| } |
| } |
| }, |
| "composition_atom": { |
| "n": 16, |
| "nll": 0.6868630202544777, |
| "acc": 0.5625, |
| "ece": 0.04684587795814621, |
| "brier": 0.49376454489058674, |
| "mean_conf": 0.5156541220418538, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.125, |
| "coverage_at_1pct_error": 0.125, |
| "aurc": 0.40027177683427684, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.04684587795814621, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 0.5555555555555556, |
| "confidence_cutoff": 0.5104723876996271 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.6153846153846154, |
| "confidence_cutoff": 0.5015301704406738 |
| } |
| } |
| }, |
| "composition_conditional": { |
| "n": 16, |
| "nll": 0.692639150220755, |
| "acc": 0.625, |
| "ece": 0.11956254686423151, |
| "brier": 0.4994918868193602, |
| "mean_conf": 0.5054374531357685, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.1875, |
| "coverage_at_1pct_error": 0.1875, |
| "aurc": 0.36212017149517156, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.11956254686423151, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.375, |
| "confidence_cutoff": 0.5062627045433907 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.6153846153846154, |
| "confidence_cutoff": 0.5018942356109619 |
| } |
| } |
| }, |
| "composition_conjunction": { |
| "n": 16, |
| "nll": 0.6807725943312114, |
| "acc": 0.75, |
| "ece": 0.24073381449590237, |
| "brier": 0.4876279322120418, |
| "mean_conf": 0.5092661855040976, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.5625, |
| "coverage_at_1pct_error": 0.5625, |
| "aurc": 0.07196657509157509, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.24073381449590237, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.5111942734958448 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.8461538461538461, |
| "confidence_cutoff": 0.506822407245636 |
| } |
| } |
| }, |
| "composition_disjunction": { |
| "n": 16, |
| "nll": 0.6845496087519543, |
| "acc": 0.625, |
| "ece": 0.10940085910388908, |
| "brier": 0.4914097560792531, |
| "mean_conf": 0.5155991408961109, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.125, |
| "coverage_at_1pct_error": 0.125, |
| "aurc": 0.3235514485514486, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.10940085910388908, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5133245877430088 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.6153846153846154, |
| "confidence_cutoff": 0.5098766532577081 |
| } |
| } |
| }, |
| "composition_exception": { |
| "n": 16, |
| "nll": 0.6999479317489916, |
| "acc": 0.3125, |
| "ece": 0.19967369551773084, |
| "brier": 0.5067955467438511, |
| "mean_conf": 0.5121736955177308, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.6147534756909757, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.19967369551773084, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.5091820359230042 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.38461538461538464, |
| "confidence_cutoff": 0.5007553100585938 |
| } |
| } |
| }, |
| "composition_negation": { |
| "n": 16, |
| "nll": 0.7255260889583809, |
| "acc": 0.3125, |
| "ece": 0.21004319515882908, |
| "brier": 0.5322663751971064, |
| "mean_conf": 0.5225431951588291, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.8313863220113219, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.21004319515882908, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.125, |
| "confidence_cutoff": 0.5219886458512737 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.38461538461538464, |
| "confidence_cutoff": 0.5036663563152062 |
| } |
| } |
| }, |
| "composition_nested_and": { |
| "n": 16, |
| "nll": 0.6977710170099051, |
| "acc": 0.4375, |
| "ece": 0.07430293594870152, |
| "brier": 0.5046165383435348, |
| "mean_conf": 0.5118029359487015, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.6242112921800421, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.07430293594870152, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5625, |
| "accuracy": 0.4444444444444444, |
| "confidence_cutoff": 0.5065592376423109 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.5384615384615384, |
| "confidence_cutoff": 0.5032630862750462 |
| } |
| } |
| }, |
| "composition_nested_or": { |
| "n": 16, |
| "nll": 0.6933208400499358, |
| "acc": 0.375, |
| "ece": 0.13961227642659346, |
| "brier": 0.5001786152290317, |
| "mean_conf": 0.5146122764265935, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.1875, |
| "coverage_at_1pct_error": 0.1875, |
| "aurc": 0.4065724206349206, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.13961227642659346, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5179873704910278 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.42857142857142855, |
| "confidence_cutoff": 0.5046656131744385 |
| } |
| } |
| }, |
| "contrastive_age_eligibility": { |
| "n": 24, |
| "nll": 0.0019957258438999047, |
| "acc": 1.0, |
| "ece": 0.001986453041521785, |
| "brier": 3.665966146860536e-05, |
| "mean_conf": 0.9980135469584782, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.001986453041521785, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 23, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9991325300448652 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9977411031723022 |
| } |
| } |
| }, |
| "contrastive_quantity_limit": { |
| "n": 24, |
| "nll": 0.2953036070445502, |
| "acc": 0.875, |
| "ece": 0.08720234763968863, |
| "brier": 0.19980341420601166, |
| "mean_conf": 0.7877976523603114, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.8333333333333334, |
| "coverage_at_1pct_error": 0.75, |
| "aurc": 0.020863970284736872, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0872023476396886, |
| "top_bins": { |
| "0.9": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999568032339159 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 0.95, |
| "confidence_cutoff": 0.5362308549929372 |
| } |
| } |
| }, |
| "contrastive_return_window": { |
| "n": 24, |
| "nll": 0.6873419627659144, |
| "acc": 0.4166666666666667, |
| "ece": 0.19777798255950627, |
| "brier": 0.4963496570710506, |
| "mean_conf": 0.6110676657072444, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.125, |
| "coverage_at_1pct_error": 0.125, |
| "aurc": 0.3518959178145307, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.19440099904057767, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5833333333333334, |
| "confidence_cutoff": 0.606834017823099 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5422527466861756 |
| } |
| } |
| }, |
| "contrastive_spend_threshold": { |
| "n": 24, |
| "nll": 0.00036495022640064607, |
| "acc": 1.0, |
| "ece": 0.00036484651323320794, |
| "brier": 4.018898659036875e-07, |
| "mean_conf": 0.9996351534867668, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.00036484651323320794, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998236097452727 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994718056567214 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.421653557685442, |
| "acc": 0.9625, |
| "ece": 0.038590716499222494, |
| "brier": 0.0749984906704935, |
| "mean_conf": 0.9977291409078489, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 0.9620253164556962, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.1, |
| "aurc": 0.029371556564368212, |
| "error_rate_at_0_9": 0.0379746835443038, |
| "confidence_bias": 0.0352291409078489, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 3, |
| "error_rate": 0.0379746835443038 |
| }, |
| "0.95": { |
| "n": 79, |
| "errors": 3, |
| "error_rate": 0.0379746835443038 |
| }, |
| "0.99": { |
| "n": 78, |
| "errors": 3, |
| "error_rate": 0.038461538461538464 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999933016730673 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9999585960145762 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.33141699422220733, |
| "acc": 0.9125, |
| "ece": 0.05920801680085907, |
| "brier": 0.14056258642338354, |
| "mean_conf": 0.9717080168008593, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.9459459459459459, |
| "coverage_at_5pct_error": 0.85, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.046590820152253506, |
| "error_rate_at_0_9": 0.05405405405405406, |
| "confidence_bias": 0.0592080168008593, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 4, |
| "error_rate": 0.05405405405405406 |
| }, |
| "0.95": { |
| "n": 73, |
| "errors": 4, |
| "error_rate": 0.0547945205479452 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 2, |
| "error_rate": 0.03278688524590164 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9990444188595242 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.953125, |
| "confidence_cutoff": 0.9856349119042156 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.6238560572386006, |
| "acc": 0.775, |
| "ece": 0.098545258596266, |
| "brier": 0.33638415666106714, |
| "mean_conf": 0.8037648115227398, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.3625, |
| "accuracy_at_0_9": 0.8620689655172413, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.025, |
| "aurc": 0.1461657498448564, |
| "error_rate_at_0_9": 0.13793103448275862, |
| "confidence_bias": 0.028764811522739797, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 4, |
| "error_rate": 0.13793103448275862 |
| }, |
| "0.95": { |
| "n": 23, |
| "errors": 2, |
| "error_rate": 0.08695652173913043 |
| }, |
| "0.99": { |
| "n": 10, |
| "errors": 1, |
| "error_rate": 0.1 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.85, |
| "confidence_cutoff": 0.8368562883553604 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.6399822597195601 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 0.9312577104784421, |
| "acc": 0.6375, |
| "ece": 0.058047224154593555, |
| "brier": 0.517886063473853, |
| "mean_conf": 0.6155482010149003, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.28378138053885427, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.02195179898509969, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7, |
| "confidence_cutoff": 0.6130724488092862 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.671875, |
| "confidence_cutoff": 0.4991830745359043 |
| } |
| }, |
| "score_mae": 0.5024858053525012, |
| "ranked_probability_score": 0.07990343060681752 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.31463663041157075, |
| "acc": 0.95, |
| "ece": 0.05342332897478768, |
| "brier": 0.10129446206548179, |
| "mean_conf": 0.9916042689599147, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 0.9487179487179487, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.5625, |
| "aurc": 0.012206716405033919, |
| "error_rate_at_0_9": 0.05128205128205128, |
| "confidence_bias": 0.04160426895991476, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 4, |
| "error_rate": 0.05128205128205128 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 4, |
| "error_rate": 0.05128205128205128 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 2, |
| "error_rate": 0.027777777777777776 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994122098980961 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9962499645174014 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 80, |
| "nll": 0.7093590738165945, |
| "acc": 0.675, |
| "ece": 0.09823356777966172, |
| "brier": 0.4286367817783625, |
| "mean_conf": 0.7157254348808417, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.1625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.2, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.17643766746740774, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.04072543488084168, |
| "top_bins": { |
| "0.9": { |
| "n": 13, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.825, |
| "confidence_cutoff": 0.7173906410209289 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.71875, |
| "confidence_cutoff": 0.5784675950234985 |
| } |
| }, |
| "score_mae": 0.39465728789615917, |
| "ranked_probability_score": 0.058236053622034226 |
| }, |
| "yelp_yn": { |
| "n": 80, |
| "nll": 0.15311162372276566, |
| "acc": 0.95, |
| "ece": 0.04194291267597787, |
| "brier": 0.08344044292524996, |
| "mean_conf": 0.9285674559777203, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.984375, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.5625, |
| "aurc": 0.010052232246343523, |
| "error_rate_at_0_9": 0.015625, |
| "confidence_bias": -0.021432544022279676, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 1, |
| "error_rate": 0.015625 |
| }, |
| "0.95": { |
| "n": 53, |
| "errors": 1, |
| "error_rate": 0.018867924528301886 |
| }, |
| "0.99": { |
| "n": 39, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9892096070827485 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9120315109484305 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 1264, |
| "nll": 0.5215492407482734, |
| "acc": 0.7943037974683544, |
| "ece": 0.04048827875950509, |
| "brier": 0.27717776074983425, |
| "mean_conf": 0.8284767038904833, |
| "confident_error_rate": 0.028481012658227847, |
| "coverage_at_0_9": 0.5561708860759493, |
| "accuracy_at_0_9": 0.9487908961593172, |
| "coverage_at_5pct_error": 0.555379746835443, |
| "coverage_at_1pct_error": 0.006329113924050633, |
| "aurc": 0.07067902977682425, |
| "error_rate_at_0_9": 0.051209103840682786, |
| "confidence_bias": 0.03417290642212889, |
| "top_bins": { |
| "0.9": { |
| "n": 703, |
| "errors": 36, |
| "error_rate": 0.051209103840682786 |
| }, |
| "0.95": { |
| "n": 646, |
| "errors": 27, |
| "error_rate": 0.04179566563467492 |
| }, |
| "0.99": { |
| "n": 512, |
| "errors": 13, |
| "error_rate": 0.025390625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9588607594936709, |
| "confidence_cutoff": 0.961561151958084 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.866600790513834, |
| "confidence_cutoff": 0.5841912218049162 |
| } |
| }, |
| "score_mae": 0.4606092452508017, |
| "ranked_probability_score": 0.07134033027857928 |
| }, |
| "none_absent": { |
| "n": 60, |
| "nll": 0.543251177325835, |
| "acc": 0.85, |
| "ece": 0.03515603322900176, |
| "brier": 0.22331079271298393, |
| "mean_conf": 0.8600576005410744, |
| "confident_error_rate": 0.03333333333333333, |
| "coverage_at_0_9": 0.6333333333333333, |
| "accuracy_at_0_9": 0.9473684210526315, |
| "coverage_at_5pct_error": 0.7166666666666667, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.11368855700250266, |
| "error_rate_at_0_9": 0.05263157894736842, |
| "confidence_bias": 0.010057600541074407, |
| "top_bins": { |
| "0.9": { |
| "n": 38, |
| "errors": 2, |
| "error_rate": 0.05263157894736842 |
| }, |
| "0.95": { |
| "n": 31, |
| "errors": 2, |
| "error_rate": 0.06451612903225806 |
| }, |
| "0.99": { |
| "n": 18, |
| "errors": 1, |
| "error_rate": 0.05555555555555555 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9333333333333333, |
| "confidence_cutoff": 0.9619653601083108 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9166666666666666, |
| "confidence_cutoff": 0.683343371252518 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 60, |
| "nll": 0.9005504316398635, |
| "acc": 0.7333333333333333, |
| "ece": 0.09230758815850672, |
| "brier": 0.32249124714796495, |
| "mean_conf": 0.8206757401298138, |
| "confident_error_rate": 0.016666666666666666, |
| "coverage_at_0_9": 0.5666666666666667, |
| "accuracy_at_0_9": 0.9705882352941176, |
| "coverage_at_5pct_error": 0.5833333333333334, |
| "coverage_at_1pct_error": 0.016666666666666666, |
| "aurc": 0.11577228698943155, |
| "error_rate_at_0_9": 0.029411764705882353, |
| "confidence_bias": 0.08734240679648053, |
| "top_bins": { |
| "0.9": { |
| "n": 34, |
| "errors": 1, |
| "error_rate": 0.029411764705882353 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 1, |
| "error_rate": 0.06666666666666667 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9666666666666667, |
| "confidence_cutoff": 0.9339339032600245 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8541666666666666, |
| "confidence_cutoff": 0.6044387108750882 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 84, |
| "nll": 0.4678339065572589, |
| "acc": 0.9166666666666666, |
| "ece": 0.03085079906002683, |
| "brier": 0.1315219130880203, |
| "mean_conf": 0.9279063634937833, |
| "confident_error_rate": 0.023809523809523808, |
| "coverage_at_0_9": 0.7976190476190477, |
| "accuracy_at_0_9": 0.9701492537313433, |
| "coverage_at_5pct_error": 0.9047619047619048, |
| "coverage_at_1pct_error": 0.09523809523809523, |
| "aurc": 0.03856923320884028, |
| "error_rate_at_0_9": 0.029850746268656716, |
| "confidence_bias": 0.011239696827116674, |
| "top_bins": { |
| "0.9": { |
| "n": 67, |
| "errors": 2, |
| "error_rate": 0.029850746268656716 |
| }, |
| "0.95": { |
| "n": 60, |
| "errors": 2, |
| "error_rate": 0.03333333333333333 |
| }, |
| "0.99": { |
| "n": 48, |
| "errors": 1, |
| "error_rate": 0.020833333333333332 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9761904761904762, |
| "confidence_cutoff": 0.995227212706841 |
| }, |
| "0.8": { |
| "coverage": 0.8095238095238095, |
| "accuracy": 0.9705882352941176, |
| "confidence_cutoff": 0.8928713597792232 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": {}, |
| "permutation": { |
| "n": 60, |
| "mean_max_delta": 0.03779500413383731, |
| "flip_rate": 0.016666666666666666 |
| }, |
| "temperature": 1.319507910772894, |
| "calibrated_clean": { |
| "n": 1264, |
| "nll": 0.4919571472502258, |
| "acc": 0.7943037974683544, |
| "ece": 0.028814258275892082, |
| "brier": 0.2749206448948224, |
| "mean_conf": 0.7989016595250444, |
| "confident_error_rate": 0.020569620253164556, |
| "coverage_at_0_9": 0.5055379746835443, |
| "accuracy_at_0_9": 0.9593114241001565, |
| "coverage_at_5pct_error": 0.5545886075949367, |
| "coverage_at_1pct_error": 0.005537974683544304, |
| "aurc": 0.07095897819981735, |
| "error_rate_at_0_9": 0.0406885758998435, |
| "confidence_bias": 0.004597862056689994, |
| "top_bins": { |
| "0.9": { |
| "n": 639, |
| "errors": 26, |
| "error_rate": 0.0406885758998435 |
| }, |
| "0.95": { |
| "n": 558, |
| "errors": 18, |
| "error_rate": 0.03225806451612903 |
| }, |
| "0.99": { |
| "n": 367, |
| "errors": 6, |
| "error_rate": 0.01634877384196185 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9588607594936709, |
| "confidence_cutoff": 0.9097529357933202 |
| }, |
| "0.8": { |
| "coverage": 0.8006329113924051, |
| "accuracy": 0.8656126482213439, |
| "confidence_cutoff": 0.5412959766037492 |
| } |
| }, |
| "score_mae": 0.47272708593819684, |
| "ranked_probability_score": 0.07114286982306338 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 1204, |
| "requested_questions": 1468, |
| "evaluated_records": 1204, |
| "evaluated_questions": 1468, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 343.6476665083319, |
| "p95": 1776.4106375863773 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.319507910772894, |
| "logits_recorded": true |
| }, |
| "transfer": { |
| "objective": -0.9407704885894344, |
| "paired_flip": { |
| "pairs": 64, |
| "flip_rate": 0.0625, |
| "both_correct_rate": 0.015625, |
| "invariant_pairs": 24, |
| "invariance_rate": 0.9166666666666666, |
| "invariant_both_correct_rate": 0.5 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 656, |
| "nll": 0.9098667547927495, |
| "acc": 0.5685975609756098, |
| "ece": 0.15184069475083165, |
| "brier": 0.5496946473177795, |
| "mean_conf": 0.7148065244031183, |
| "confident_error_rate": 0.06402439024390244, |
| "coverage_at_0_9": 0.3003048780487805, |
| "accuracy_at_0_9": 0.7868020304568528, |
| "coverage_at_5pct_error": 0.13414634146341464, |
| "coverage_at_1pct_error": 0.0701219512195122, |
| "aurc": 0.2726527308927074, |
| "error_rate_at_0_9": 0.2131979695431472, |
| "confidence_bias": 0.14620896342750855, |
| "top_bins": { |
| "0.9": { |
| "n": 197, |
| "errors": 42, |
| "error_rate": 0.2131979695431472 |
| }, |
| "0.95": { |
| "n": 126, |
| "errors": 14, |
| "error_rate": 0.1111111111111111 |
| }, |
| "0.99": { |
| "n": 39, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6707317073170732, |
| "confidence_cutoff": 0.707867643471588 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.6, |
| "confidence_cutoff": 0.5144661954363468 |
| } |
| }, |
| "score_mae": 0.6961418739313883, |
| "ranked_probability_score": 0.30817160000540955 |
| }, |
| "tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.6833422496464963, |
| "acc": 0.5625, |
| "ece": 0.048464233979846005, |
| "brier": 0.4902079366809463, |
| "mean_conf": 0.514035766020154, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.09375, |
| "coverage_at_1pct_error": 0.09375, |
| "aurc": 0.29136050867534974, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.048464233979846005, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.5107448405471349 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 0.5925925925925926, |
| "confidence_cutoff": 0.503829673046758 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.6940730345557493, |
| "acc": 0.46875, |
| "ece": 0.04352010198047873, |
| "brier": 0.5009236559654398, |
| "mean_conf": 0.5122701019804787, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5496196217669959, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04352010198047873, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5166306187730659 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.4230769230769231, |
| "confidence_cutoff": 0.501518502876714 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.6980906968444973, |
| "acc": 0.46875, |
| "ece": 0.054383457719069384, |
| "brier": 0.5049256050377603, |
| "mean_conf": 0.5231334577190694, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5820981501169419, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.054383457719069384, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.375, |
| "confidence_cutoff": 0.5229583382606506 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.5769230769230769, |
| "confidence_cutoff": 0.5096097144368206 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 1.0339701294493764, |
| "acc": 0.5, |
| "ece": 0.4144774026307121, |
| "brier": 0.7161675205460087, |
| "mean_conf": 0.9144774026307122, |
| "confident_error_rate": 0.2, |
| "coverage_at_0_9": 0.7, |
| "accuracy_at_0_9": 0.7142857142857143, |
| "coverage_at_5pct_error": 0.525, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.15959830910365297, |
| "error_rate_at_0_9": 0.2857142857142857, |
| "confidence_bias": 0.4144774026307122, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 8, |
| "error_rate": 0.2857142857142857 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 1, |
| "error_rate": 0.047619047619047616 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9799628239498235 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.8601182103157043 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 2.2505586636609918, |
| "acc": 0.275, |
| "ece": 0.6290899198958707, |
| "brier": 1.197235252147983, |
| "mean_conf": 0.9040899198958708, |
| "confident_error_rate": 0.475, |
| "coverage_at_0_9": 0.7, |
| "accuracy_at_0_9": 0.32142857142857145, |
| "coverage_at_5pct_error": 0.05, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.54830887260108, |
| "error_rate_at_0_9": 0.6785714285714286, |
| "confidence_bias": 0.6290899198958708, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 19, |
| "error_rate": 0.6785714285714286 |
| }, |
| "0.95": { |
| "n": 9, |
| "errors": 4, |
| "error_rate": 0.4444444444444444 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.45, |
| "confidence_cutoff": 0.9255054173738333 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.3125, |
| "confidence_cutoff": 0.8918043030731547 |
| } |
| }, |
| "score_mae": 0.6961418739313883, |
| "ranked_probability_score": 0.30817160000540955 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.2057542293509105, |
| "acc": 0.5625, |
| "ece": 0.06829600936372701, |
| "brier": 0.5855855421120933, |
| "mean_conf": 0.5838921344491043, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.05, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.2847309733715896, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.02139213444910426, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7, |
| "confidence_cutoff": 0.5569462304170179 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.4726999258969467 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 1.1222459884756266, |
| "acc": 0.525, |
| "ece": 0.12375383831039756, |
| "brier": 0.5704439813752953, |
| "mean_conf": 0.5562091193375563, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.1125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.2375, |
| "aurc": 0.2604475173843487, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.031209119337556324, |
| "top_bins": { |
| "0.9": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7, |
| "confidence_cutoff": 0.5009298640715034 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.5625, |
| "confidence_cutoff": 0.3604842722415924 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.9816472399168571, |
| "acc": 0.2875, |
| "ece": 0.35924160997690124, |
| "brier": 0.7544718149675231, |
| "mean_conf": 0.6467416099769012, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.8563656691854259, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.35924160997690124, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.1, |
| "confidence_cutoff": 0.6299932786820316 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.5468162135238583 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.6201317262227193, |
| "acc": 0.6875, |
| "ece": 0.11840502530533466, |
| "brier": 0.41161498583927425, |
| "mean_conf": 0.7757451323961314, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.2875, |
| "accuracy_at_0_9": 0.8695652173913043, |
| "coverage_at_5pct_error": 0.025, |
| "coverage_at_1pct_error": 0.025, |
| "aurc": 0.21007195888747154, |
| "error_rate_at_0_9": 0.13043478260869565, |
| "confidence_bias": 0.08824513239613141, |
| "top_bins": { |
| "0.9": { |
| "n": 23, |
| "errors": 3, |
| "error_rate": 0.13043478260869565 |
| }, |
| "0.95": { |
| "n": 10, |
| "errors": 2, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.8032595395556161 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.765625, |
| "confidence_cutoff": 0.6199027115135577 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.22354972299184625, |
| "acc": 0.925, |
| "ece": 0.06789633455655048, |
| "brier": 0.12012071486848477, |
| "mean_conf": 0.8789441216334387, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.675, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.9, |
| "coverage_at_1pct_error": 0.75, |
| "aurc": 0.01049294556907073, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.04605587836656133, |
| "top_bins": { |
| "0.9": { |
| "n": 54, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 47, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 22, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9726309307076344 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.7994060744804564 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.8351116933687053, |
| "acc": 0.6875, |
| "ece": 0.22180319280009556, |
| "brier": 0.5101348034224664, |
| "mean_conf": 0.8908219907612654, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.675, |
| "accuracy_at_0_9": 0.7777777777777778, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.20980111038804497, |
| "error_rate_at_0_9": 0.2222222222222222, |
| "confidence_bias": 0.20332199076126545, |
| "top_bins": { |
| "0.9": { |
| "n": 54, |
| "errors": 12, |
| "error_rate": 0.2222222222222222 |
| }, |
| "0.95": { |
| "n": 34, |
| "errors": 7, |
| "error_rate": 0.20588235294117646 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.9442175626754761 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.703125, |
| "confidence_cutoff": 0.8313192624640026 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 656, |
| "nll": 0.9098667547927495, |
| "acc": 0.5685975609756098, |
| "ece": 0.15184069475083165, |
| "brier": 0.5496946473177795, |
| "mean_conf": 0.7148065244031183, |
| "confident_error_rate": 0.06402439024390244, |
| "coverage_at_0_9": 0.3003048780487805, |
| "accuracy_at_0_9": 0.7868020304568528, |
| "coverage_at_5pct_error": 0.13414634146341464, |
| "coverage_at_1pct_error": 0.0701219512195122, |
| "aurc": 0.2726527308927074, |
| "error_rate_at_0_9": 0.2131979695431472, |
| "confidence_bias": 0.14620896342750855, |
| "top_bins": { |
| "0.9": { |
| "n": 197, |
| "errors": 42, |
| "error_rate": 0.2131979695431472 |
| }, |
| "0.95": { |
| "n": 126, |
| "errors": 14, |
| "error_rate": 0.1111111111111111 |
| }, |
| "0.99": { |
| "n": 39, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6707317073170732, |
| "confidence_cutoff": 0.707867643471588 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.6, |
| "confidence_cutoff": 0.5144661954363468 |
| } |
| }, |
| "score_mae": 0.6961418739313883, |
| "ranked_probability_score": 0.30817160000540955 |
| }, |
| "none_absent": { |
| "n": 36, |
| "nll": 0.839222556790093, |
| "acc": 0.6388888888888888, |
| "ece": 0.13519060875689604, |
| "brier": 0.4790985720376279, |
| "mean_conf": 0.6663556845610538, |
| "confident_error_rate": 0.027777777777777776, |
| "coverage_at_0_9": 0.16666666666666666, |
| "accuracy_at_0_9": 0.8333333333333334, |
| "coverage_at_5pct_error": 0.08333333333333333, |
| "coverage_at_1pct_error": 0.08333333333333333, |
| "aurc": 0.24216988587274996, |
| "error_rate_at_0_9": 0.16666666666666666, |
| "confidence_bias": 0.027466795672164923, |
| "top_bins": { |
| "0.9": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 1, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7222222222222222, |
| "confidence_cutoff": 0.692634932148079 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.7586206896551724, |
| "confidence_cutoff": 0.46444892796790427 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 36, |
| "nll": 1.0702554096724084, |
| "acc": 0.6111111111111112, |
| "ece": 0.0824644297325398, |
| "brier": 0.4845346826960828, |
| "mean_conf": 0.6615826453720488, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.2222222222222222, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.3888888888888889, |
| "coverage_at_1pct_error": 0.3888888888888889, |
| "aurc": 0.13647701567895199, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.050471534260937645, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9444444444444444, |
| "confidence_cutoff": 0.678112416947823 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.6896551724137931, |
| "confidence_cutoff": 0.4202956126164172 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 36, |
| "nll": 0.7676115841056179, |
| "acc": 0.6944444444444444, |
| "ece": 0.13343139218862612, |
| "brier": 0.3608910203704398, |
| "mean_conf": 0.7179161874094808, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3055555555555556, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.5555555555555556, |
| "coverage_at_1pct_error": 0.5277777777777778, |
| "aurc": 0.08668255163693558, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.023471742965036402, |
| "top_bins": { |
| "0.9": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.7402256166503357 |
| }, |
| "0.8": { |
| "coverage": 0.8055555555555556, |
| "accuracy": 0.7931034482758621, |
| "confidence_cutoff": 0.5095580033363343 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.6833422496464963, |
| "acc": 0.5625, |
| "ece": 0.048464233979846005, |
| "brier": 0.4902079366809463, |
| "mean_conf": 0.514035766020154, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.09375, |
| "coverage_at_1pct_error": 0.09375, |
| "aurc": 0.29136050867534974, |
| "error_rate_at_0_9": null, |
| "confidence_bias": -0.048464233979846005, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.5107448405471349 |
| }, |
| "0.8": { |
| "coverage": 0.84375, |
| "accuracy": 0.5925925925925926, |
| "confidence_cutoff": 0.503829673046758 |
| } |
| } |
| }, |
| "composition_held_conditional": { |
| "n": 32, |
| "nll": 0.6940730345557493, |
| "acc": 0.46875, |
| "ece": 0.04352010198047873, |
| "brier": 0.5009236559654398, |
| "mean_conf": 0.5122701019804787, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5496196217669959, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04352010198047873, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5166306187730659 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.4230769230769231, |
| "confidence_cutoff": 0.501518502876714 |
| } |
| } |
| }, |
| "composition_held_or_not": { |
| "n": 32, |
| "nll": 0.6980906968444973, |
| "acc": 0.46875, |
| "ece": 0.054383457719069384, |
| "brier": 0.5049256050377603, |
| "mean_conf": 0.5231334577190694, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5820981501169419, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.054383457719069384, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.375, |
| "confidence_cutoff": 0.5229583382606506 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.5769230769230769, |
| "confidence_cutoff": 0.5096097144368206 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 40, |
| "nll": 1.0339701294493764, |
| "acc": 0.5, |
| "ece": 0.4144774026307121, |
| "brier": 0.7161675205460087, |
| "mean_conf": 0.9144774026307122, |
| "confident_error_rate": 0.2, |
| "coverage_at_0_9": 0.7, |
| "accuracy_at_0_9": 0.7142857142857143, |
| "coverage_at_5pct_error": 0.525, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.15959830910365297, |
| "error_rate_at_0_9": 0.2857142857142857, |
| "confidence_bias": 0.4144774026307122, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 8, |
| "error_rate": 0.2857142857142857 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 1, |
| "error_rate": 0.047619047619047616 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9799628239498235 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.8601182103157043 |
| } |
| } |
| }, |
| "contrastive_deadline": { |
| "n": 40, |
| "nll": 2.2505586636609918, |
| "acc": 0.275, |
| "ece": 0.6290899198958707, |
| "brier": 1.197235252147983, |
| "mean_conf": 0.9040899198958708, |
| "confident_error_rate": 0.475, |
| "coverage_at_0_9": 0.7, |
| "accuracy_at_0_9": 0.32142857142857145, |
| "coverage_at_5pct_error": 0.05, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.54830887260108, |
| "error_rate_at_0_9": 0.6785714285714286, |
| "confidence_bias": 0.6290899198958708, |
| "top_bins": { |
| "0.9": { |
| "n": 28, |
| "errors": 19, |
| "error_rate": 0.6785714285714286 |
| }, |
| "0.95": { |
| "n": 9, |
| "errors": 4, |
| "error_rate": 0.4444444444444444 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.45, |
| "confidence_cutoff": 0.9255054173738333 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.3125, |
| "confidence_cutoff": 0.8918043030731547 |
| } |
| }, |
| "score_mae": 0.6961418739313883, |
| "ranked_probability_score": 0.30817160000540955 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.2057542293509105, |
| "acc": 0.5625, |
| "ece": 0.06829600936372701, |
| "brier": 0.5855855421120933, |
| "mean_conf": 0.5838921344491043, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.05, |
| "coverage_at_1pct_error": 0.05, |
| "aurc": 0.2847309733715896, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.02139213444910426, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7, |
| "confidence_cutoff": 0.5569462304170179 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.4726999258969467 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 1.1222459884756266, |
| "acc": 0.525, |
| "ece": 0.12375383831039756, |
| "brier": 0.5704439813752953, |
| "mean_conf": 0.5562091193375563, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.1125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.2375, |
| "aurc": 0.2604475173843487, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.031209119337556324, |
| "top_bins": { |
| "0.9": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7, |
| "confidence_cutoff": 0.5009298640715034 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.5625, |
| "confidence_cutoff": 0.3604842722415924 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.9816472399168571, |
| "acc": 0.2875, |
| "ece": 0.35924160997690124, |
| "brier": 0.7544718149675231, |
| "mean_conf": 0.6467416099769012, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.8563656691854259, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.35924160997690124, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.1, |
| "confidence_cutoff": 0.6299932786820316 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.5468162135238583 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.6201317262227193, |
| "acc": 0.6875, |
| "ece": 0.11840502530533466, |
| "brier": 0.41161498583927425, |
| "mean_conf": 0.7757451323961314, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.2875, |
| "accuracy_at_0_9": 0.8695652173913043, |
| "coverage_at_5pct_error": 0.025, |
| "coverage_at_1pct_error": 0.025, |
| "aurc": 0.21007195888747154, |
| "error_rate_at_0_9": 0.13043478260869565, |
| "confidence_bias": 0.08824513239613141, |
| "top_bins": { |
| "0.9": { |
| "n": 23, |
| "errors": 3, |
| "error_rate": 0.13043478260869565 |
| }, |
| "0.95": { |
| "n": 10, |
| "errors": 2, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.8032595395556161 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.765625, |
| "confidence_cutoff": 0.6199027115135577 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.22354972299184625, |
| "acc": 0.925, |
| "ece": 0.06789633455655048, |
| "brier": 0.12012071486848477, |
| "mean_conf": 0.8789441216334387, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.675, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.9, |
| "coverage_at_1pct_error": 0.75, |
| "aurc": 0.01049294556907073, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.04605587836656133, |
| "top_bins": { |
| "0.9": { |
| "n": 54, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 47, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 22, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9726309307076344 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.7994060744804564 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.8351116933687053, |
| "acc": 0.6875, |
| "ece": 0.22180319280009556, |
| "brier": 0.5101348034224664, |
| "mean_conf": 0.8908219907612654, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.675, |
| "accuracy_at_0_9": 0.7777777777777778, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.20980111038804497, |
| "error_rate_at_0_9": 0.2222222222222222, |
| "confidence_bias": 0.20332199076126545, |
| "top_bins": { |
| "0.9": { |
| "n": 54, |
| "errors": 12, |
| "error_rate": 0.2222222222222222 |
| }, |
| "0.95": { |
| "n": 34, |
| "errors": 7, |
| "error_rate": 0.20588235294117646 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.9442175626754761 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.703125, |
| "confidence_cutoff": 0.8313192624640026 |
| } |
| } |
| } |
| }, |
| "permutation": { |
| "n": 36, |
| "mean_max_delta": 0.07768421211680508, |
| "flip_rate": 0.08333333333333333 |
| }, |
| "temperature": 1.319507910772894, |
| "calibrated_clean": { |
| "n": 656, |
| "nll": 0.8448515581702054, |
| "acc": 0.5685975609756098, |
| "ece": 0.11046275792106883, |
| "brier": 0.5254015776080041, |
| "mean_conf": 0.6726607987326776, |
| "confident_error_rate": 0.021341463414634148, |
| "coverage_at_0_9": 0.19207317073170732, |
| "accuracy_at_0_9": 0.8888888888888888, |
| "coverage_at_5pct_error": 0.12804878048780488, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.2767632520963531, |
| "error_rate_at_0_9": 0.1111111111111111, |
| "confidence_bias": 0.10406323775706783, |
| "top_bins": { |
| "0.9": { |
| "n": 126, |
| "errors": 14, |
| "error_rate": 0.1111111111111111 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 3, |
| "error_rate": 0.04477611940298507 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6646341463414634, |
| "confidence_cutoff": 0.6451680660218214 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.6114285714285714, |
| "confidence_cutoff": 0.5037052906313757 |
| } |
| }, |
| "score_mae": 0.6894445881767614, |
| "ranked_probability_score": 0.28422154802672034 |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 764, |
| "requested_questions": 764, |
| "evaluated_records": 764, |
| "evaluated_questions": 764, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 400.28454147977754, |
| "p95": 713.417099986691 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.319507910772894, |
| "logits_recorded": true |
| }, |
| "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1" |
| }, |
| "provenance": { |
| "config": { |
| "base": "google/gemma-4-E2B", |
| "base_revision": "d29ff6b45f081a49ee2733a859c9c9c2d95d1a6f", |
| "seed": 0, |
| "epochs": 1, |
| "lr": 0.0001, |
| "batch": 4, |
| "accum": 2, |
| "dtype": "bf16", |
| "weights_dtype": "bf16", |
| "checkpointing": 1, |
| "p_none_pair": 0.25 |
| }, |
| "config_sha256": "355241763405ebfaa97b051898d14a98695c5ef0ed47b6877082fca47adf1506", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "d12ab7f0765de0f29364f3411b99f06bb80b1cbf120ca9ed2b03316ec766e828", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "c2129026ab56395f78c7d3071410541d5423dd7d97f7371518570572f206c376", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "d18d5f4f02d079344d61c2a31882c524c90012ebfdb660bf9160e145f37e3d42", |
| "kev/train.py": "e103e99d3a68ab99eb58f135320fbc46b18514fcc2b195d1c6879633836e99b9", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "7a13e9d29b3fc68f8236a0c1e4d67079b40a2a328a296f7ced1022817dff82aa", |
| "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "git_commit": "ccd6e8b646068a6689792d76e3cae3698e9a015a", |
| "platform": "HF Jobs l4x1 (job 6ab426ec52d0dbd7f1d86bac)", |
| "torch": "2.8.0+cu128", |
| "device": "cuda", |
| "gpu": "NVIDIA L4", |
| "legacy_checkpoint": false, |
| "measured_checkpoint": { |
| "requested": "runs/g4-e2b-e1-fp32/00-trial-0/checkpoint", |
| "resolved": "runs/g4-e2b-e1-fp32/00-trial-0/checkpoint", |
| "head_sha256": "80bcdddf6f671e9901e3805f8c311492c93d86ac63af0440d519b2d45bb90361", |
| "adapter_sha256": "93a489ba6fd4eddb766e9f9bbd70fbec04babe33c7ecd3925f5f9973e325410a", |
| "inference_temperature": 1.0 |
| }, |
| "eval_device": "mps", |
| "resumed_evaluation": true, |
| "resumed_source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "d12ab7f0765de0f29364f3411b99f06bb80b1cbf120ca9ed2b03316ec766e828", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "c2129026ab56395f78c7d3071410541d5423dd7d97f7371518570572f206c376", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "d18d5f4f02d079344d61c2a31882c524c90012ebfdb660bf9160e145f37e3d42", |
| "kev/train.py": "e103e99d3a68ab99eb58f135320fbc46b18514fcc2b195d1c6879633836e99b9", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "7a13e9d29b3fc68f8236a0c1e4d67079b40a2a328a296f7ced1022817dff82aa", |
| "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "resumed_with_changed_non_evaluator_files": [], |
| "resumed_git_commit": "ccd6e8b646068a6689792d76e3cae3698e9a015a" |
| }, |
| "mechanism_checks": { |
| "n": 8, |
| "packed_max_delta": 0.0, |
| "sibling_max_delta": 1.8417835235595703e-05, |
| "tolerance": 0.001, |
| "passed": true |
| }, |
| "gates": { |
| "passed": false, |
| "checks": { |
| "complete_coverage": true, |
| "isolation_and_packing": true, |
| "transfer_complete": true, |
| "heldout_pairs_at_least_70pct": false, |
| "transfer_confident_errors_below_10pct": true |
| }, |
| "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." |
| }, |
| "calibration_fit": { |
| "temperature": 1.319507910772894, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "e8e669ff8b32eca59da46675dbd5a5ec138895480cdd475ed499f8d58249c0a2", |
| "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2", |
| "n": 1148 |
| }, |
| "wall_seconds": 1409.0520514999516, |
| "promotable": false, |
| "test_evaluated": false, |
| "training_resources": { |
| "wall_seconds": 4865.690968990326, |
| "records_seen": 15142, |
| "requested_records": 12506, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 1564, |
| "forward_tokens": 3079220, |
| "peak_device_bytes": 14623992320, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 4, |
| "peak_rss_bytes": 11216687104 |
| } |
| } |
|
|
