{
  "objective": -0.5460977168843815,
  "paired_flip": {
    "pairs": 80,
    "flip_rate": 0.55,
    "both_correct_rate": 0.45,
    "invariant_pairs": 32,
    "invariance_rate": 0.9375,
    "invariant_both_correct_rate": 0.46875
  },
  "unknowable": null,
  "clean": {
    "n": 1264,
    "nll": 0.5215492407482734,
    "acc": 0.7943037974683544,
    "ece": 0.04048827875950509,
    "brier": 0.27717776074983425,
    "mean_conf": 0.8284767038904833,
    "confident_error_rate": 0.028481012658227847,
    "coverage_at_0_9": 0.5561708860759493,
    "accuracy_at_0_9": 0.9487908961593172,
    "coverage_at_5pct_error": 0.555379746835443,
    "coverage_at_1pct_error": 0.006329113924050633,
    "aurc": 0.07067902977682425,
    "error_rate_at_0_9": 0.051209103840682786,
    "confidence_bias": 0.03417290642212889,
    "top_bins": {
      "0.9": {
        "n": 703,
        "errors": 36,
        "error_rate": 0.051209103840682786
      },
      "0.95": {
        "n": 646,
        "errors": 27,
        "error_rate": 0.04179566563467492
      },
      "0.99": {
        "n": 512,
        "errors": 13,
        "error_rate": 0.025390625
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9588607594936709,
        "confidence_cutoff": 0.961561151958084
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.866600790513834,
        "confidence_cutoff": 0.5841912218049162
      }
    },
    "score_mae": 0.4606092452508017,
    "ranked_probability_score": 0.07134033027857928
  },
  "tasks": {
    "agnews": {
      "n": 80,
      "nll": 0.7253619615276804,
      "acc": 0.825,
      "ece": 0.12435274294907851,
      "brier": 0.29748699997093375,
      "mean_conf": 0.9467044212167703,
      "confident_error_rate": 0.1125,
      "coverage_at_0_9": 0.8375,
      "accuracy_at_0_9": 0.8656716417910447,
      "coverage_at_5pct_error": 0.5,
      "coverage_at_1pct_error": 0.025,
      "aurc": 0.0891676305179602,
      "error_rate_at_0_9": 0.13432835820895522,
      "confidence_bias": 0.12170442121677039,
      "top_bins": {
        "0.9": {
          "n": 67,
          "errors": 9,
          "error_rate": 0.13432835820895522
        },
        "0.95": {
          "n": 60,
          "errors": 5,
          "error_rate": 0.08333333333333333
        },
        "0.99": {
          "n": 47,
          "errors": 3,
          "error_rate": 0.06382978723404255
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.95,
          "confidence_cutoff": 0.9968383335724911
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.875,
          "confidence_cutoff": 0.9297638903048708
        }
      }
    },
    "agnews_yn": {
      "n": 160,
      "nll": 0.27274937177459047,
      "acc": 0.8875,
      "ece": 0.07205353407763021,
      "brier": 0.16638284259750016,
      "mean_conf": 0.9492791941714842,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.81875,
      "accuracy_at_0_9": 0.9694656488549618,
      "coverage_at_5pct_error": 0.8375,
      "coverage_at_1pct_error": 0.625,
      "aurc": 0.019553552474504545,
      "error_rate_at_0_9": 0.030534351145038167,
      "confidence_bias": 0.06177919417148425,
      "top_bins": {
        "0.9": {
          "n": 131,
          "errors": 4,
          "error_rate": 0.030534351145038167
        },
        "0.95": {
          "n": 122,
          "errors": 3,
          "error_rate": 0.02459016393442623
        },
        "0.99": {
          "n": 97,
          "errors": 1,
          "error_rate": 0.010309278350515464
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9946307004104364
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9765625,
          "confidence_cutoff": 0.9334476480940993
        }
      }
    },
    "amazon": {
      "n": 80,
      "nll": 0.9079064857116548,
      "acc": 0.675,
      "ece": 0.09082075584790608,
      "brier": 0.48731382188074246,
      "mean_conf": 0.6600554720427965,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.1125,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.25,
      "coverage_at_1pct_error": 0.2375,
      "aurc": 0.19069566998937534,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.01494452795720358,
      "top_bins": {
        "0.9": {
          "n": 9,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 3,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.775,
          "confidence_cutoff": 0.6450436740808796
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.71875,
          "confidence_cutoff": 0.48814962988854316
        }
      },
      "score_mae": 0.4846846425037448,
      "ranked_probability_score": 0.07588150660688607
    },
    "banking77": {
      "n": 80,
      "nll": 0.6355525688857636,
      "acc": 0.8375,
      "ece": 0.07048704096779947,
      "brier": 0.24473580199909098,
      "mean_conf": 0.8527902873638666,
      "confident_error_rate": 0.025,
      "coverage_at_0_9": 0.65,
      "accuracy_at_0_9": 0.9615384615384616,
      "coverage_at_5pct_error": 0.65,
      "coverage_at_1pct_error": 0.5625,
      "aurc": 0.03531804637364592,
      "error_rate_at_0_9": 0.038461538461538464,
      "confidence_bias": 0.015290287363866617,
      "top_bins": {
        "0.9": {
          "n": 52,
          "errors": 2,
          "error_rate": 0.038461538461538464
        },
        "0.95": {
          "n": 47,
          "errors": 1,
          "error_rate": 0.02127659574468085
        },
        "0.99": {
          "n": 32,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9819397006054039
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.921875,
          "confidence_cutoff": 0.64788273781478
        }
      }
    },
    "boolq": {
      "n": 80,
      "nll": 0.5330866725434655,
      "acc": 0.7875,
      "ece": 0.11078594898530592,
      "brier": 0.32181604785225043,
      "mean_conf": 0.8668040159586322,
      "confident_error_rate": 0.0625,
      "coverage_at_0_9": 0.5875,
      "accuracy_at_0_9": 0.8936170212765957,
      "coverage_at_5pct_error": 0.35,
      "coverage_at_1pct_error": 0.15,
      "aurc": 0.10319650984358723,
      "error_rate_at_0_9": 0.10638297872340426,
      "confidence_bias": 0.07930401595863223,
      "top_bins": {
        "0.9": {
          "n": 47,
          "errors": 5,
          "error_rate": 0.10638297872340426
        },
        "0.95": {
          "n": 40,
          "errors": 4,
          "error_rate": 0.1
        },
        "0.99": {
          "n": 17,
          "errors": 1,
          "error_rate": 0.058823529411764705
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9,
          "confidence_cutoff": 0.960176618093949
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.8125,
          "confidence_cutoff": 0.7061129387549795
        }
      }
    },
    "composition_atom": {
      "n": 16,
      "nll": 0.6868630202544777,
      "acc": 0.5625,
      "ece": 0.04684587795814621,
      "brier": 0.49376454489058674,
      "mean_conf": 0.5156541220418538,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.125,
      "coverage_at_1pct_error": 0.125,
      "aurc": 0.40027177683427684,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.04684587795814621,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 0.5555555555555556,
          "confidence_cutoff": 0.5104723876996271
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6153846153846154,
          "confidence_cutoff": 0.5015301704406738
        }
      }
    },
    "composition_conditional": {
      "n": 16,
      "nll": 0.692639150220755,
      "acc": 0.625,
      "ece": 0.11956254686423151,
      "brier": 0.4994918868193602,
      "mean_conf": 0.5054374531357685,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.1875,
      "coverage_at_1pct_error": 0.1875,
      "aurc": 0.36212017149517156,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.11956254686423151,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.375,
          "confidence_cutoff": 0.5062627045433907
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6153846153846154,
          "confidence_cutoff": 0.5018942356109619
        }
      }
    },
    "composition_conjunction": {
      "n": 16,
      "nll": 0.6807725943312114,
      "acc": 0.75,
      "ece": 0.24073381449590237,
      "brier": 0.4876279322120418,
      "mean_conf": 0.5092661855040976,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.5625,
      "coverage_at_1pct_error": 0.5625,
      "aurc": 0.07196657509157509,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.24073381449590237,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.5111942734958448
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.8461538461538461,
          "confidence_cutoff": 0.506822407245636
        }
      }
    },
    "composition_disjunction": {
      "n": 16,
      "nll": 0.6845496087519543,
      "acc": 0.625,
      "ece": 0.10940085910388908,
      "brier": 0.4914097560792531,
      "mean_conf": 0.5155991408961109,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.125,
      "coverage_at_1pct_error": 0.125,
      "aurc": 0.3235514485514486,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.10940085910388908,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.5,
          "confidence_cutoff": 0.5133245877430088
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.6153846153846154,
          "confidence_cutoff": 0.5098766532577081
        }
      }
    },
    "composition_exception": {
      "n": 16,
      "nll": 0.6999479317489916,
      "acc": 0.3125,
      "ece": 0.19967369551773084,
      "brier": 0.5067955467438511,
      "mean_conf": 0.5121736955177308,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0625,
      "coverage_at_1pct_error": 0.0625,
      "aurc": 0.6147534756909757,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.19967369551773084,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.25,
          "confidence_cutoff": 0.5091820359230042
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.38461538461538464,
          "confidence_cutoff": 0.5007553100585938
        }
      }
    },
    "composition_negation": {
      "n": 16,
      "nll": 0.7255260889583809,
      "acc": 0.3125,
      "ece": 0.21004319515882908,
      "brier": 0.5322663751971064,
      "mean_conf": 0.5225431951588291,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.8313863220113219,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.21004319515882908,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.125,
          "confidence_cutoff": 0.5219886458512737
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.38461538461538464,
          "confidence_cutoff": 0.5036663563152062
        }
      }
    },
    "composition_nested_and": {
      "n": 16,
      "nll": 0.6977710170099051,
      "acc": 0.4375,
      "ece": 0.07430293594870152,
      "brier": 0.5046165383435348,
      "mean_conf": 0.5118029359487015,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.6242112921800421,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.07430293594870152,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5625,
          "accuracy": 0.4444444444444444,
          "confidence_cutoff": 0.5065592376423109
        },
        "0.8": {
          "coverage": 0.8125,
          "accuracy": 0.5384615384615384,
          "confidence_cutoff": 0.5032630862750462
        }
      }
    },
    "composition_nested_or": {
      "n": 16,
      "nll": 0.6933208400499358,
      "acc": 0.375,
      "ece": 0.13961227642659346,
      "brier": 0.5001786152290317,
      "mean_conf": 0.5146122764265935,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.1875,
      "coverage_at_1pct_error": 0.1875,
      "aurc": 0.4065724206349206,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.13961227642659346,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.5,
          "confidence_cutoff": 0.5179873704910278
        },
        "0.8": {
          "coverage": 0.875,
          "accuracy": 0.42857142857142855,
          "confidence_cutoff": 0.5046656131744385
        }
      }
    },
    "contrastive_age_eligibility": {
      "n": 24,
      "nll": 0.0019957258438999047,
      "acc": 1.0,
      "ece": 0.001986453041521785,
      "brier": 3.665966146860536e-05,
      "mean_conf": 0.9980135469584782,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.001986453041521785,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 23,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9991325300448652
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9977411031723022
        }
      }
    },
    "contrastive_quantity_limit": {
      "n": 24,
      "nll": 0.2953036070445502,
      "acc": 0.875,
      "ece": 0.08720234763968863,
      "brier": 0.19980341420601166,
      "mean_conf": 0.7877976523603114,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.5,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.8333333333333334,
      "coverage_at_1pct_error": 0.75,
      "aurc": 0.020863970284736872,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.0872023476396886,
      "top_bins": {
        "0.9": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.999568032339159
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 0.95,
          "confidence_cutoff": 0.5362308549929372
        }
      }
    },
    "contrastive_return_window": {
      "n": 24,
      "nll": 0.6873419627659144,
      "acc": 0.4166666666666667,
      "ece": 0.19777798255950627,
      "brier": 0.4963496570710506,
      "mean_conf": 0.6110676657072444,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.125,
      "coverage_at_1pct_error": 0.125,
      "aurc": 0.3518959178145307,
      "error_rate_at_0_9": null,
      "confidence_bias": 0.19440099904057767,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.5833333333333334,
          "confidence_cutoff": 0.606834017823099
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 0.5,
          "confidence_cutoff": 0.5422527466861756
        }
      }
    },
    "contrastive_spend_threshold": {
      "n": 24,
      "nll": 0.00036495022640064607,
      "acc": 1.0,
      "ece": 0.00036484651323320794,
      "brier": 4.018898659036875e-07,
      "mean_conf": 0.9996351534867668,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 1.0,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 1.0,
      "aurc": 0.0,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": -0.00036484651323320794,
      "top_bins": {
        "0.9": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 24,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9998236097452727
        },
        "0.8": {
          "coverage": 0.8333333333333334,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9994718056567214
        }
      }
    },
    "dbpedia14": {
      "n": 80,
      "nll": 0.421653557685442,
      "acc": 0.9625,
      "ece": 0.038590716499222494,
      "brier": 0.0749984906704935,
      "mean_conf": 0.9977291409078489,
      "confident_error_rate": 0.0375,
      "coverage_at_0_9": 0.9875,
      "accuracy_at_0_9": 0.9620253164556962,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.1,
      "aurc": 0.029371556564368212,
      "error_rate_at_0_9": 0.0379746835443038,
      "confidence_bias": 0.0352291409078489,
      "top_bins": {
        "0.9": {
          "n": 79,
          "errors": 3,
          "error_rate": 0.0379746835443038
        },
        "0.95": {
          "n": 79,
          "errors": 3,
          "error_rate": 0.0379746835443038
        },
        "0.99": {
          "n": 78,
          "errors": 3,
          "error_rate": 0.038461538461538464
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9999933016730673
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9999585960145762
        }
      }
    },
    "imdb": {
      "n": 80,
      "nll": 0.33141699422220733,
      "acc": 0.9125,
      "ece": 0.05920801680085907,
      "brier": 0.14056258642338354,
      "mean_conf": 0.9717080168008593,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.925,
      "accuracy_at_0_9": 0.9459459459459459,
      "coverage_at_5pct_error": 0.85,
      "coverage_at_1pct_error": 0.05,
      "aurc": 0.046590820152253506,
      "error_rate_at_0_9": 0.05405405405405406,
      "confidence_bias": 0.0592080168008593,
      "top_bins": {
        "0.9": {
          "n": 74,
          "errors": 4,
          "error_rate": 0.05405405405405406
        },
        "0.95": {
          "n": 73,
          "errors": 4,
          "error_rate": 0.0547945205479452
        },
        "0.99": {
          "n": 61,
          "errors": 2,
          "error_rate": 0.03278688524590164
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.975,
          "confidence_cutoff": 0.9990444188595242
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.953125,
          "confidence_cutoff": 0.9856349119042156
        }
      }
    },
    "mnli": {
      "n": 80,
      "nll": 0.6238560572386006,
      "acc": 0.775,
      "ece": 0.098545258596266,
      "brier": 0.33638415666106714,
      "mean_conf": 0.8037648115227398,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.3625,
      "accuracy_at_0_9": 0.8620689655172413,
      "coverage_at_5pct_error": 0.25,
      "coverage_at_1pct_error": 0.025,
      "aurc": 0.1461657498448564,
      "error_rate_at_0_9": 0.13793103448275862,
      "confidence_bias": 0.028764811522739797,
      "top_bins": {
        "0.9": {
          "n": 29,
          "errors": 4,
          "error_rate": 0.13793103448275862
        },
        "0.95": {
          "n": 23,
          "errors": 2,
          "error_rate": 0.08695652173913043
        },
        "0.99": {
          "n": 10,
          "errors": 1,
          "error_rate": 0.1
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.85,
          "confidence_cutoff": 0.8368562883553604
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.8125,
          "confidence_cutoff": 0.6399822597195601
        }
      }
    },
    "sst5": {
      "n": 80,
      "nll": 0.9312577104784421,
      "acc": 0.6375,
      "ece": 0.058047224154593555,
      "brier": 0.517886063473853,
      "mean_conf": 0.6155482010149003,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.0,
      "accuracy_at_0_9": null,
      "coverage_at_5pct_error": 0.0,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.28378138053885427,
      "error_rate_at_0_9": null,
      "confidence_bias": -0.02195179898509969,
      "top_bins": {
        "0.9": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.95": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.7,
          "confidence_cutoff": 0.6130724488092862
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.671875,
          "confidence_cutoff": 0.4991830745359043
        }
      },
      "score_mae": 0.5024858053525012,
      "ranked_probability_score": 0.07990343060681752
    },
    "trec": {
      "n": 80,
      "nll": 0.31463663041157075,
      "acc": 0.95,
      "ece": 0.05342332897478768,
      "brier": 0.10129446206548179,
      "mean_conf": 0.9916042689599147,
      "confident_error_rate": 0.05,
      "coverage_at_0_9": 0.975,
      "accuracy_at_0_9": 0.9487179487179487,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.5625,
      "aurc": 0.012206716405033919,
      "error_rate_at_0_9": 0.05128205128205128,
      "confidence_bias": 0.04160426895991476,
      "top_bins": {
        "0.9": {
          "n": 78,
          "errors": 4,
          "error_rate": 0.05128205128205128
        },
        "0.95": {
          "n": 78,
          "errors": 4,
          "error_rate": 0.05128205128205128
        },
        "0.99": {
          "n": 72,
          "errors": 2,
          "error_rate": 0.027777777777777776
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9994122098980961
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.96875,
          "confidence_cutoff": 0.9962499645174014
        }
      }
    },
    "yelp": {
      "n": 80,
      "nll": 0.7093590738165945,
      "acc": 0.675,
      "ece": 0.09823356777966172,
      "brier": 0.4286367817783625,
      "mean_conf": 0.7157254348808417,
      "confident_error_rate": 0.0,
      "coverage_at_0_9": 0.1625,
      "accuracy_at_0_9": 1.0,
      "coverage_at_5pct_error": 0.2,
      "coverage_at_1pct_error": 0.2,
      "aurc": 0.17643766746740774,
      "error_rate_at_0_9": 0.0,
      "confidence_bias": 0.04072543488084168,
      "top_bins": {
        "0.9": {
          "n": 13,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.95": {
          "n": 8,
          "errors": 0,
          "error_rate": 0.0
        },
        "0.99": {
          "n": 0,
          "errors": 0,
          "error_rate": null
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.825,
          "confidence_cutoff": 0.7173906410209289
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.71875,
          "confidence_cutoff": 0.5784675950234985
        }
      },
      "score_mae": 0.39465728789615917,
      "ranked_probability_score": 0.058236053622034226
    },
    "yelp_yn": {
      "n": 80,
      "nll": 0.15311162372276566,
      "acc": 0.95,
      "ece": 0.04194291267597787,
      "brier": 0.08344044292524996,
      "mean_conf": 0.9285674559777203,
      "confident_error_rate": 0.0125,
      "coverage_at_0_9": 0.8,
      "accuracy_at_0_9": 0.984375,
      "coverage_at_5pct_error": 1.0,
      "coverage_at_1pct_error": 0.5625,
      "aurc": 0.010052232246343523,
      "error_rate_at_0_9": 0.015625,
      "confidence_bias": -0.021432544022279676,
      "top_bins": {
        "0.9": {
          "n": 64,
          "errors": 1,
          "error_rate": 0.015625
        },
        "0.95": {
          "n": 53,
          "errors": 1,
          "error_rate": 0.018867924528301886
        },
        "0.99": {
          "n": 39,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 1.0,
          "confidence_cutoff": 0.9892096070827485
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.984375,
          "confidence_cutoff": 0.9120315109484305
        }
      }
    }
  },
  "variants": {
    "clean": {
      "n": 1264,
      "nll": 0.5215492407482734,
      "acc": 0.7943037974683544,
      "ece": 0.04048827875950509,
      "brier": 0.27717776074983425,
      "mean_conf": 0.8284767038904833,
      "confident_error_rate": 0.028481012658227847,
      "coverage_at_0_9": 0.5561708860759493,
      "accuracy_at_0_9": 0.9487908961593172,
      "coverage_at_5pct_error": 0.555379746835443,
      "coverage_at_1pct_error": 0.006329113924050633,
      "aurc": 0.07067902977682425,
      "error_rate_at_0_9": 0.051209103840682786,
      "confidence_bias": 0.03417290642212889,
      "top_bins": {
        "0.9": {
          "n": 703,
          "errors": 36,
          "error_rate": 0.051209103840682786
        },
        "0.95": {
          "n": 646,
          "errors": 27,
          "error_rate": 0.04179566563467492
        },
        "0.99": {
          "n": 512,
          "errors": 13,
          "error_rate": 0.025390625
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9588607594936709,
          "confidence_cutoff": 0.961561151958084
        },
        "0.8": {
          "coverage": 0.8006329113924051,
          "accuracy": 0.866600790513834,
          "confidence_cutoff": 0.5841912218049162
        }
      },
      "score_mae": 0.4606092452508017,
      "ranked_probability_score": 0.07134033027857928
    },
    "none_absent": {
      "n": 60,
      "nll": 0.543251177325835,
      "acc": 0.85,
      "ece": 0.03515603322900176,
      "brier": 0.22331079271298393,
      "mean_conf": 0.8600576005410744,
      "confident_error_rate": 0.03333333333333333,
      "coverage_at_0_9": 0.6333333333333333,
      "accuracy_at_0_9": 0.9473684210526315,
      "coverage_at_5pct_error": 0.7166666666666667,
      "coverage_at_1pct_error": 0.0,
      "aurc": 0.11368855700250266,
      "error_rate_at_0_9": 0.05263157894736842,
      "confidence_bias": 0.010057600541074407,
      "top_bins": {
        "0.9": {
          "n": 38,
          "errors": 2,
          "error_rate": 0.05263157894736842
        },
        "0.95": {
          "n": 31,
          "errors": 2,
          "error_rate": 0.06451612903225806
        },
        "0.99": {
          "n": 18,
          "errors": 1,
          "error_rate": 0.05555555555555555
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9333333333333333,
          "confidence_cutoff": 0.9619653601083108
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.9166666666666666,
          "confidence_cutoff": 0.683343371252518
        }
      }
    },
    "none_present": {
      "n": 60,
      "nll": 0.9005504316398635,
      "acc": 0.7333333333333333,
      "ece": 0.09230758815850672,
      "brier": 0.32249124714796495,
      "mean_conf": 0.8206757401298138,
      "confident_error_rate": 0.016666666666666666,
      "coverage_at_0_9": 0.5666666666666667,
      "accuracy_at_0_9": 0.9705882352941176,
      "coverage_at_5pct_error": 0.5833333333333334,
      "coverage_at_1pct_error": 0.016666666666666666,
      "aurc": 0.11577228698943155,
      "error_rate_at_0_9": 0.029411764705882353,
      "confidence_bias": 0.08734240679648053,
      "top_bins": {
        "0.9": {
          "n": 34,
          "errors": 1,
          "error_rate": 0.029411764705882353
        },
        "0.95": {
          "n": 27,
          "errors": 1,
          "error_rate": 0.037037037037037035
        },
        "0.99": {
          "n": 15,
          "errors": 1,
          "error_rate": 0.06666666666666667
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9666666666666667,
          "confidence_cutoff": 0.9339339032600245
        },
        "0.8": {
          "coverage": 0.8,
          "accuracy": 0.8541666666666666,
          "confidence_cutoff": 0.6044387108750882
        }
      }
    },
    "permuted": {
      "n": 84,
      "nll": 0.4678339065572589,
      "acc": 0.9166666666666666,
      "ece": 0.03085079906002683,
      "brier": 0.1315219130880203,
      "mean_conf": 0.9279063634937833,
      "confident_error_rate": 0.023809523809523808,
      "coverage_at_0_9": 0.7976190476190477,
      "accuracy_at_0_9": 0.9701492537313433,
      "coverage_at_5pct_error": 0.9047619047619048,
      "coverage_at_1pct_error": 0.09523809523809523,
      "aurc": 0.03856923320884028,
      "error_rate_at_0_9": 0.029850746268656716,
      "confidence_bias": 0.011239696827116674,
      "top_bins": {
        "0.9": {
          "n": 67,
          "errors": 2,
          "error_rate": 0.029850746268656716
        },
        "0.95": {
          "n": 60,
          "errors": 2,
          "error_rate": 0.03333333333333333
        },
        "0.99": {
          "n": 48,
          "errors": 1,
          "error_rate": 0.020833333333333332
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.9761904761904762,
          "confidence_cutoff": 0.995227212706841
        },
        "0.8": {
          "coverage": 0.8095238095238095,
          "accuracy": 0.9705882352941176,
          "confidence_cutoff": 0.8928713597792232
        }
      }
    }
  },
  "heldout_tasks": {},
  "permutation": {
    "n": 60,
    "mean_max_delta": 0.03779500413383731,
    "flip_rate": 0.016666666666666666
  },
  "temperature": 1.319507910772894,
  "calibrated_clean": {
    "n": 1264,
    "nll": 0.4919571472502258,
    "acc": 0.7943037974683544,
    "ece": 0.028814258275892082,
    "brier": 0.2749206448948224,
    "mean_conf": 0.7989016595250444,
    "confident_error_rate": 0.020569620253164556,
    "coverage_at_0_9": 0.5055379746835443,
    "accuracy_at_0_9": 0.9593114241001565,
    "coverage_at_5pct_error": 0.5545886075949367,
    "coverage_at_1pct_error": 0.005537974683544304,
    "aurc": 0.07095897819981735,
    "error_rate_at_0_9": 0.0406885758998435,
    "confidence_bias": 0.004597862056689994,
    "top_bins": {
      "0.9": {
        "n": 639,
        "errors": 26,
        "error_rate": 0.0406885758998435
      },
      "0.95": {
        "n": 558,
        "errors": 18,
        "error_rate": 0.03225806451612903
      },
      "0.99": {
        "n": 367,
        "errors": 6,
        "error_rate": 0.01634877384196185
      }
    },
    "selective": {
      "0.5": {
        "coverage": 0.5,
        "accuracy": 0.9588607594936709,
        "confidence_cutoff": 0.9097529357933202
      },
      "0.8": {
        "coverage": 0.8006329113924051,
        "accuracy": 0.8656126482213439,
        "confidence_cutoff": 0.5412959766037492
      }
    },
    "score_mae": 0.47272708593819684,
    "ranked_probability_score": 0.07114286982306338
  },
  "metric_policy": {
    "version": 2,
    "selective_ties": "whole_confidence_groups",
    "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
    "aurc": "right-step integral over whole confidence groups",
    "confident_error_rate": "high-confidence errors divided by all questions",
    "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
    "nll": "exact from logits when recorded; otherwise from floored probabilities",
    "nll_floor": 1e-09,
    "renormalize_returned_probabilities": true,
    "raw_sums_outside_1e_5": 0,
    "returned_zeros": 0
  },
  "coverage": {
    "requested_records": 1204,
    "requested_questions": 1468,
    "evaluated_records": 1204,
    "evaluated_questions": 1468,
    "rejected_records": 0,
    "truncated_records": 0
  },
  "latency_ms": {
    "median": 343.6476665083319,
    "p95": 1776.4106375863773
  },
  "calibration": {
    "inference_temperature": 1.0,
    "additional_temperature": 1.319507910772894,
    "logits_recorded": true
  },
  "transfer": {
    "objective": -0.9407704885894344,
    "paired_flip": {
      "pairs": 64,
      "flip_rate": 0.0625,
      "both_correct_rate": 0.015625,
      "invariant_pairs": 24,
      "invariance_rate": 0.9166666666666666,
      "invariant_both_correct_rate": 0.5
    },
    "unknowable": null,
    "clean": {
      "n": 656,
      "nll": 0.9098667547927495,
      "acc": 0.5685975609756098,
      "ece": 0.15184069475083165,
      "brier": 0.5496946473177795,
      "mean_conf": 0.7148065244031183,
      "confident_error_rate": 0.06402439024390244,
      "coverage_at_0_9": 0.3003048780487805,
      "accuracy_at_0_9": 0.7868020304568528,
      "coverage_at_5pct_error": 0.13414634146341464,
      "coverage_at_1pct_error": 0.0701219512195122,
      "aurc": 0.2726527308927074,
      "error_rate_at_0_9": 0.2131979695431472,
      "confidence_bias": 0.14620896342750855,
      "top_bins": {
        "0.9": {
          "n": 197,
          "errors": 42,
          "error_rate": 0.2131979695431472
        },
        "0.95": {
          "n": 126,
          "errors": 14,
          "error_rate": 0.1111111111111111
        },
        "0.99": {
          "n": 39,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.6707317073170732,
          "confidence_cutoff": 0.707867643471588
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.6,
          "confidence_cutoff": 0.5144661954363468
        }
      },
      "score_mae": 0.6961418739313883,
      "ranked_probability_score": 0.30817160000540955
    },
    "tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.6833422496464963,
        "acc": 0.5625,
        "ece": 0.048464233979846005,
        "brier": 0.4902079366809463,
        "mean_conf": 0.514035766020154,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.09375,
        "coverage_at_1pct_error": 0.09375,
        "aurc": 0.29136050867534974,
        "error_rate_at_0_9": null,
        "confidence_bias": -0.048464233979846005,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.5107448405471349
          },
          "0.8": {
            "coverage": 0.84375,
            "accuracy": 0.5925925925925926,
            "confidence_cutoff": 0.503829673046758
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.6940730345557493,
        "acc": 0.46875,
        "ece": 0.04352010198047873,
        "brier": 0.5009236559654398,
        "mean_conf": 0.5122701019804787,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.5496196217669959,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.04352010198047873,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.5,
            "confidence_cutoff": 0.5166306187730659
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.4230769230769231,
            "confidence_cutoff": 0.501518502876714
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.6980906968444973,
        "acc": 0.46875,
        "ece": 0.054383457719069384,
        "brier": 0.5049256050377603,
        "mean_conf": 0.5231334577190694,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.5820981501169419,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.054383457719069384,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.375,
            "confidence_cutoff": 0.5229583382606506
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.5769230769230769,
            "confidence_cutoff": 0.5096097144368206
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 1.0339701294493764,
        "acc": 0.5,
        "ece": 0.4144774026307121,
        "brier": 0.7161675205460087,
        "mean_conf": 0.9144774026307122,
        "confident_error_rate": 0.2,
        "coverage_at_0_9": 0.7,
        "accuracy_at_0_9": 0.7142857142857143,
        "coverage_at_5pct_error": 0.525,
        "coverage_at_1pct_error": 0.5,
        "aurc": 0.15959830910365297,
        "error_rate_at_0_9": 0.2857142857142857,
        "confidence_bias": 0.4144774026307122,
        "top_bins": {
          "0.9": {
            "n": 28,
            "errors": 8,
            "error_rate": 0.2857142857142857
          },
          "0.95": {
            "n": 21,
            "errors": 1,
            "error_rate": 0.047619047619047616
          },
          "0.99": {
            "n": 12,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9799628239498235
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.625,
            "confidence_cutoff": 0.8601182103157043
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 2.2505586636609918,
        "acc": 0.275,
        "ece": 0.6290899198958707,
        "brier": 1.197235252147983,
        "mean_conf": 0.9040899198958708,
        "confident_error_rate": 0.475,
        "coverage_at_0_9": 0.7,
        "accuracy_at_0_9": 0.32142857142857145,
        "coverage_at_5pct_error": 0.05,
        "coverage_at_1pct_error": 0.05,
        "aurc": 0.54830887260108,
        "error_rate_at_0_9": 0.6785714285714286,
        "confidence_bias": 0.6290899198958708,
        "top_bins": {
          "0.9": {
            "n": 28,
            "errors": 19,
            "error_rate": 0.6785714285714286
          },
          "0.95": {
            "n": 9,
            "errors": 4,
            "error_rate": 0.4444444444444444
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.45,
            "confidence_cutoff": 0.9255054173738333
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.3125,
            "confidence_cutoff": 0.8918043030731547
          }
        },
        "score_mae": 0.6961418739313883,
        "ranked_probability_score": 0.30817160000540955
      },
      "emotion": {
        "n": 80,
        "nll": 1.2057542293509105,
        "acc": 0.5625,
        "ece": 0.06829600936372701,
        "brier": 0.5855855421120933,
        "mean_conf": 0.5838921344491043,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.05,
        "coverage_at_1pct_error": 0.05,
        "aurc": 0.2847309733715896,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.02139213444910426,
        "top_bins": {
          "0.9": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7,
            "confidence_cutoff": 0.5569462304170179
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.625,
            "confidence_cutoff": 0.4726999258969467
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.1222459884756266,
        "acc": 0.525,
        "ece": 0.12375383831039756,
        "brier": 0.5704439813752953,
        "mean_conf": 0.5562091193375563,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.1125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.25,
        "coverage_at_1pct_error": 0.2375,
        "aurc": 0.2604475173843487,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.031209119337556324,
        "top_bins": {
          "0.9": {
            "n": 9,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7,
            "confidence_cutoff": 0.5009298640715034
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.5625,
            "confidence_cutoff": 0.3604842722415924
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 0.9816472399168571,
        "acc": 0.2875,
        "ece": 0.35924160997690124,
        "brier": 0.7544718149675231,
        "mean_conf": 0.6467416099769012,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.8563656691854259,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.35924160997690124,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.1,
            "confidence_cutoff": 0.6299932786820316
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.25,
            "confidence_cutoff": 0.5468162135238583
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.6201317262227193,
        "acc": 0.6875,
        "ece": 0.11840502530533466,
        "brier": 0.41161498583927425,
        "mean_conf": 0.7757451323961314,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.2875,
        "accuracy_at_0_9": 0.8695652173913043,
        "coverage_at_5pct_error": 0.025,
        "coverage_at_1pct_error": 0.025,
        "aurc": 0.21007195888747154,
        "error_rate_at_0_9": 0.13043478260869565,
        "confidence_bias": 0.08824513239613141,
        "top_bins": {
          "0.9": {
            "n": 23,
            "errors": 3,
            "error_rate": 0.13043478260869565
          },
          "0.95": {
            "n": 10,
            "errors": 2,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.775,
            "confidence_cutoff": 0.8032595395556161
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.765625,
            "confidence_cutoff": 0.6199027115135577
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.22354972299184625,
        "acc": 0.925,
        "ece": 0.06789633455655048,
        "brier": 0.12012071486848477,
        "mean_conf": 0.8789441216334387,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.675,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.9,
        "coverage_at_1pct_error": 0.75,
        "aurc": 0.01049294556907073,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.04605587836656133,
        "top_bins": {
          "0.9": {
            "n": 54,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 47,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 22,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9726309307076344
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.984375,
            "confidence_cutoff": 0.7994060744804564
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.8351116933687053,
        "acc": 0.6875,
        "ece": 0.22180319280009556,
        "brier": 0.5101348034224664,
        "mean_conf": 0.8908219907612654,
        "confident_error_rate": 0.15,
        "coverage_at_0_9": 0.675,
        "accuracy_at_0_9": 0.7777777777777778,
        "coverage_at_5pct_error": 0.0625,
        "coverage_at_1pct_error": 0.0625,
        "aurc": 0.20980111038804497,
        "error_rate_at_0_9": 0.2222222222222222,
        "confidence_bias": 0.20332199076126545,
        "top_bins": {
          "0.9": {
            "n": 54,
            "errors": 12,
            "error_rate": 0.2222222222222222
          },
          "0.95": {
            "n": 34,
            "errors": 7,
            "error_rate": 0.20588235294117646
          },
          "0.99": {
            "n": 3,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.775,
            "confidence_cutoff": 0.9442175626754761
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.703125,
            "confidence_cutoff": 0.8313192624640026
          }
        }
      }
    },
    "variants": {
      "clean": {
        "n": 656,
        "nll": 0.9098667547927495,
        "acc": 0.5685975609756098,
        "ece": 0.15184069475083165,
        "brier": 0.5496946473177795,
        "mean_conf": 0.7148065244031183,
        "confident_error_rate": 0.06402439024390244,
        "coverage_at_0_9": 0.3003048780487805,
        "accuracy_at_0_9": 0.7868020304568528,
        "coverage_at_5pct_error": 0.13414634146341464,
        "coverage_at_1pct_error": 0.0701219512195122,
        "aurc": 0.2726527308927074,
        "error_rate_at_0_9": 0.2131979695431472,
        "confidence_bias": 0.14620896342750855,
        "top_bins": {
          "0.9": {
            "n": 197,
            "errors": 42,
            "error_rate": 0.2131979695431472
          },
          "0.95": {
            "n": 126,
            "errors": 14,
            "error_rate": 0.1111111111111111
          },
          "0.99": {
            "n": 39,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.6707317073170732,
            "confidence_cutoff": 0.707867643471588
          },
          "0.8": {
            "coverage": 0.8003048780487805,
            "accuracy": 0.6,
            "confidence_cutoff": 0.5144661954363468
          }
        },
        "score_mae": 0.6961418739313883,
        "ranked_probability_score": 0.30817160000540955
      },
      "none_absent": {
        "n": 36,
        "nll": 0.839222556790093,
        "acc": 0.6388888888888888,
        "ece": 0.13519060875689604,
        "brier": 0.4790985720376279,
        "mean_conf": 0.6663556845610538,
        "confident_error_rate": 0.027777777777777776,
        "coverage_at_0_9": 0.16666666666666666,
        "accuracy_at_0_9": 0.8333333333333334,
        "coverage_at_5pct_error": 0.08333333333333333,
        "coverage_at_1pct_error": 0.08333333333333333,
        "aurc": 0.24216988587274996,
        "error_rate_at_0_9": 0.16666666666666666,
        "confidence_bias": 0.027466795672164923,
        "top_bins": {
          "0.9": {
            "n": 6,
            "errors": 1,
            "error_rate": 0.16666666666666666
          },
          "0.95": {
            "n": 5,
            "errors": 1,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7222222222222222,
            "confidence_cutoff": 0.692634932148079
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.7586206896551724,
            "confidence_cutoff": 0.46444892796790427
          }
        }
      },
      "none_present": {
        "n": 36,
        "nll": 1.0702554096724084,
        "acc": 0.6111111111111112,
        "ece": 0.0824644297325398,
        "brier": 0.4845346826960828,
        "mean_conf": 0.6615826453720488,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.2222222222222222,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.3888888888888889,
        "coverage_at_1pct_error": 0.3888888888888889,
        "aurc": 0.13647701567895199,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.050471534260937645,
        "top_bins": {
          "0.9": {
            "n": 8,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.9444444444444444,
            "confidence_cutoff": 0.678112416947823
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.6896551724137931,
            "confidence_cutoff": 0.4202956126164172
          }
        }
      },
      "permuted": {
        "n": 36,
        "nll": 0.7676115841056179,
        "acc": 0.6944444444444444,
        "ece": 0.13343139218862612,
        "brier": 0.3608910203704398,
        "mean_conf": 0.7179161874094808,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.3055555555555556,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.5555555555555556,
        "coverage_at_1pct_error": 0.5277777777777778,
        "aurc": 0.08668255163693558,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.023471742965036402,
        "top_bins": {
          "0.9": {
            "n": 11,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 9,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 4,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.7402256166503357
          },
          "0.8": {
            "coverage": 0.8055555555555556,
            "accuracy": 0.7931034482758621,
            "confidence_cutoff": 0.5095580033363343
          }
        }
      }
    },
    "heldout_tasks": {
      "composition_held_and_or": {
        "n": 32,
        "nll": 0.6833422496464963,
        "acc": 0.5625,
        "ece": 0.048464233979846005,
        "brier": 0.4902079366809463,
        "mean_conf": 0.514035766020154,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.09375,
        "coverage_at_1pct_error": 0.09375,
        "aurc": 0.29136050867534974,
        "error_rate_at_0_9": null,
        "confidence_bias": -0.048464233979846005,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.625,
            "confidence_cutoff": 0.5107448405471349
          },
          "0.8": {
            "coverage": 0.84375,
            "accuracy": 0.5925925925925926,
            "confidence_cutoff": 0.503829673046758
          }
        }
      },
      "composition_held_conditional": {
        "n": 32,
        "nll": 0.6940730345557493,
        "acc": 0.46875,
        "ece": 0.04352010198047873,
        "brier": 0.5009236559654398,
        "mean_conf": 0.5122701019804787,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.5496196217669959,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.04352010198047873,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.5,
            "confidence_cutoff": 0.5166306187730659
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.4230769230769231,
            "confidence_cutoff": 0.501518502876714
          }
        }
      },
      "composition_held_or_not": {
        "n": 32,
        "nll": 0.6980906968444973,
        "acc": 0.46875,
        "ece": 0.054383457719069384,
        "brier": 0.5049256050377603,
        "mean_conf": 0.5231334577190694,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.5820981501169419,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.054383457719069384,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.375,
            "confidence_cutoff": 0.5229583382606506
          },
          "0.8": {
            "coverage": 0.8125,
            "accuracy": 0.5769230769230769,
            "confidence_cutoff": 0.5096097144368206
          }
        }
      },
      "contrastive_authorization": {
        "n": 40,
        "nll": 1.0339701294493764,
        "acc": 0.5,
        "ece": 0.4144774026307121,
        "brier": 0.7161675205460087,
        "mean_conf": 0.9144774026307122,
        "confident_error_rate": 0.2,
        "coverage_at_0_9": 0.7,
        "accuracy_at_0_9": 0.7142857142857143,
        "coverage_at_5pct_error": 0.525,
        "coverage_at_1pct_error": 0.5,
        "aurc": 0.15959830910365297,
        "error_rate_at_0_9": 0.2857142857142857,
        "confidence_bias": 0.4144774026307122,
        "top_bins": {
          "0.9": {
            "n": 28,
            "errors": 8,
            "error_rate": 0.2857142857142857
          },
          "0.95": {
            "n": 21,
            "errors": 1,
            "error_rate": 0.047619047619047616
          },
          "0.99": {
            "n": 12,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9799628239498235
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.625,
            "confidence_cutoff": 0.8601182103157043
          }
        }
      },
      "contrastive_deadline": {
        "n": 40,
        "nll": 2.2505586636609918,
        "acc": 0.275,
        "ece": 0.6290899198958707,
        "brier": 1.197235252147983,
        "mean_conf": 0.9040899198958708,
        "confident_error_rate": 0.475,
        "coverage_at_0_9": 0.7,
        "accuracy_at_0_9": 0.32142857142857145,
        "coverage_at_5pct_error": 0.05,
        "coverage_at_1pct_error": 0.05,
        "aurc": 0.54830887260108,
        "error_rate_at_0_9": 0.6785714285714286,
        "confidence_bias": 0.6290899198958708,
        "top_bins": {
          "0.9": {
            "n": 28,
            "errors": 19,
            "error_rate": 0.6785714285714286
          },
          "0.95": {
            "n": 9,
            "errors": 4,
            "error_rate": 0.4444444444444444
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.45,
            "confidence_cutoff": 0.9255054173738333
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.3125,
            "confidence_cutoff": 0.8918043030731547
          }
        },
        "score_mae": 0.6961418739313883,
        "ranked_probability_score": 0.30817160000540955
      },
      "emotion": {
        "n": 80,
        "nll": 1.2057542293509105,
        "acc": 0.5625,
        "ece": 0.06829600936372701,
        "brier": 0.5855855421120933,
        "mean_conf": 0.5838921344491043,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.05,
        "coverage_at_1pct_error": 0.05,
        "aurc": 0.2847309733715896,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.02139213444910426,
        "top_bins": {
          "0.9": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7,
            "confidence_cutoff": 0.5569462304170179
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.625,
            "confidence_cutoff": 0.4726999258969467
          }
        }
      },
      "mmlu": {
        "n": 80,
        "nll": 1.1222459884756266,
        "acc": 0.525,
        "ece": 0.12375383831039756,
        "brier": 0.5704439813752953,
        "mean_conf": 0.5562091193375563,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.1125,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.25,
        "coverage_at_1pct_error": 0.2375,
        "aurc": 0.2604475173843487,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": 0.031209119337556324,
        "top_bins": {
          "0.9": {
            "n": 9,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 5,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.7,
            "confidence_cutoff": 0.5009298640715034
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.5625,
            "confidence_cutoff": 0.3604842722415924
          }
        }
      },
      "paws": {
        "n": 80,
        "nll": 0.9816472399168571,
        "acc": 0.2875,
        "ece": 0.35924160997690124,
        "brier": 0.7544718149675231,
        "mean_conf": 0.6467416099769012,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.0,
        "accuracy_at_0_9": null,
        "coverage_at_5pct_error": 0.0,
        "coverage_at_1pct_error": 0.0,
        "aurc": 0.8563656691854259,
        "error_rate_at_0_9": null,
        "confidence_bias": 0.35924160997690124,
        "top_bins": {
          "0.9": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.95": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          },
          "0.99": {
            "n": 0,
            "errors": 0,
            "error_rate": null
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.1,
            "confidence_cutoff": 0.6299932786820316
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.25,
            "confidence_cutoff": 0.5468162135238583
          }
        }
      },
      "qnli": {
        "n": 80,
        "nll": 0.6201317262227193,
        "acc": 0.6875,
        "ece": 0.11840502530533466,
        "brier": 0.41161498583927425,
        "mean_conf": 0.7757451323961314,
        "confident_error_rate": 0.0375,
        "coverage_at_0_9": 0.2875,
        "accuracy_at_0_9": 0.8695652173913043,
        "coverage_at_5pct_error": 0.025,
        "coverage_at_1pct_error": 0.025,
        "aurc": 0.21007195888747154,
        "error_rate_at_0_9": 0.13043478260869565,
        "confidence_bias": 0.08824513239613141,
        "top_bins": {
          "0.9": {
            "n": 23,
            "errors": 3,
            "error_rate": 0.13043478260869565
          },
          "0.95": {
            "n": 10,
            "errors": 2,
            "error_rate": 0.2
          },
          "0.99": {
            "n": 1,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.775,
            "confidence_cutoff": 0.8032595395556161
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.765625,
            "confidence_cutoff": 0.6199027115135577
          }
        }
      },
      "sciq": {
        "n": 80,
        "nll": 0.22354972299184625,
        "acc": 0.925,
        "ece": 0.06789633455655048,
        "brier": 0.12012071486848477,
        "mean_conf": 0.8789441216334387,
        "confident_error_rate": 0.0,
        "coverage_at_0_9": 0.675,
        "accuracy_at_0_9": 1.0,
        "coverage_at_5pct_error": 0.9,
        "coverage_at_1pct_error": 0.75,
        "aurc": 0.01049294556907073,
        "error_rate_at_0_9": 0.0,
        "confidence_bias": -0.04605587836656133,
        "top_bins": {
          "0.9": {
            "n": 54,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.95": {
            "n": 47,
            "errors": 0,
            "error_rate": 0.0
          },
          "0.99": {
            "n": 22,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 1.0,
            "confidence_cutoff": 0.9726309307076344
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.984375,
            "confidence_cutoff": 0.7994060744804564
          }
        }
      },
      "tweet_offensive": {
        "n": 80,
        "nll": 0.8351116933687053,
        "acc": 0.6875,
        "ece": 0.22180319280009556,
        "brier": 0.5101348034224664,
        "mean_conf": 0.8908219907612654,
        "confident_error_rate": 0.15,
        "coverage_at_0_9": 0.675,
        "accuracy_at_0_9": 0.7777777777777778,
        "coverage_at_5pct_error": 0.0625,
        "coverage_at_1pct_error": 0.0625,
        "aurc": 0.20980111038804497,
        "error_rate_at_0_9": 0.2222222222222222,
        "confidence_bias": 0.20332199076126545,
        "top_bins": {
          "0.9": {
            "n": 54,
            "errors": 12,
            "error_rate": 0.2222222222222222
          },
          "0.95": {
            "n": 34,
            "errors": 7,
            "error_rate": 0.20588235294117646
          },
          "0.99": {
            "n": 3,
            "errors": 0,
            "error_rate": 0.0
          }
        },
        "selective": {
          "0.5": {
            "coverage": 0.5,
            "accuracy": 0.775,
            "confidence_cutoff": 0.9442175626754761
          },
          "0.8": {
            "coverage": 0.8,
            "accuracy": 0.703125,
            "confidence_cutoff": 0.8313192624640026
          }
        }
      }
    },
    "permutation": {
      "n": 36,
      "mean_max_delta": 0.07768421211680508,
      "flip_rate": 0.08333333333333333
    },
    "temperature": 1.319507910772894,
    "calibrated_clean": {
      "n": 656,
      "nll": 0.8448515581702054,
      "acc": 0.5685975609756098,
      "ece": 0.11046275792106883,
      "brier": 0.5254015776080041,
      "mean_conf": 0.6726607987326776,
      "confident_error_rate": 0.021341463414634148,
      "coverage_at_0_9": 0.19207317073170732,
      "accuracy_at_0_9": 0.8888888888888888,
      "coverage_at_5pct_error": 0.12804878048780488,
      "coverage_at_1pct_error": 0.0625,
      "aurc": 0.2767632520963531,
      "error_rate_at_0_9": 0.1111111111111111,
      "confidence_bias": 0.10406323775706783,
      "top_bins": {
        "0.9": {
          "n": 126,
          "errors": 14,
          "error_rate": 0.1111111111111111
        },
        "0.95": {
          "n": 67,
          "errors": 3,
          "error_rate": 0.04477611940298507
        },
        "0.99": {
          "n": 12,
          "errors": 0,
          "error_rate": 0.0
        }
      },
      "selective": {
        "0.5": {
          "coverage": 0.5,
          "accuracy": 0.6646341463414634,
          "confidence_cutoff": 0.6451680660218214
        },
        "0.8": {
          "coverage": 0.8003048780487805,
          "accuracy": 0.6114285714285714,
          "confidence_cutoff": 0.5037052906313757
        }
      },
      "score_mae": 0.6894445881767614,
      "ranked_probability_score": 0.28422154802672034
    },
    "metric_policy": {
      "version": 2,
      "selective_ties": "whole_confidence_groups",
      "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee",
      "aurc": "right-step integral over whole confidence groups",
      "confident_error_rate": "high-confidence errors divided by all questions",
      "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9",
      "nll": "exact from logits when recorded; otherwise from floored probabilities",
      "nll_floor": 1e-09,
      "renormalize_returned_probabilities": true,
      "raw_sums_outside_1e_5": 0,
      "returned_zeros": 0
    },
    "coverage": {
      "requested_records": 764,
      "requested_questions": 764,
      "evaluated_records": 764,
      "evaluated_questions": 764,
      "rejected_records": 0,
      "truncated_records": 0
    },
    "latency_ms": {
      "median": 400.28454147977754,
      "p95": 713.417099986691
    },
    "calibration": {
      "inference_temperature": 1.0,
      "additional_temperature": 1.319507910772894,
      "logits_recorded": true
    },
    "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1"
  },
  "provenance": {
    "config": {
      "base": "google/gemma-4-E2B",
      "base_revision": "d29ff6b45f081a49ee2733a859c9c9c2d95d1a6f",
      "seed": 0,
      "epochs": 1,
      "lr": 0.0001,
      "batch": 4,
      "accum": 2,
      "dtype": "bf16",
      "weights_dtype": "bf16",
      "checkpointing": 1,
      "p_none_pair": 0.25
    },
    "config_sha256": "355241763405ebfaa97b051898d14a98695c5ef0ed47b6877082fca47adf1506",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
      "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
      "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
      "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0",
      "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
      "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59",
      "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
      "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320",
      "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
      "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
      "kev/experiment.py": "d12ab7f0765de0f29364f3411b99f06bb80b1cbf120ca9ed2b03316ec766e828",
      "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
      "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9",
      "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
      "kev/model.py": "c2129026ab56395f78c7d3071410541d5423dd7d97f7371518570572f206c376",
      "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
      "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3",
      "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
      "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735",
      "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
      "kev/suite.py": "d18d5f4f02d079344d61c2a31882c524c90012ebfdb660bf9160e145f37e3d42",
      "kev/train.py": "e103e99d3a68ab99eb58f135320fbc46b18514fcc2b195d1c6879633836e99b9",
      "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
      "modal_app.py": "7a13e9d29b3fc68f8236a0c1e4d67079b40a2a328a296f7ced1022817dff82aa",
      "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d",
      "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
    },
    "git_commit": "ccd6e8b646068a6689792d76e3cae3698e9a015a",
    "platform": "HF Jobs l4x1 (job 6ab426ec52d0dbd7f1d86bac)",
    "torch": "2.8.0+cu128",
    "device": "cuda",
    "gpu": "NVIDIA L4",
    "legacy_checkpoint": false,
    "measured_checkpoint": {
      "requested": "runs/g4-e2b-e1-fp32/00-trial-0/checkpoint",
      "resolved": "runs/g4-e2b-e1-fp32/00-trial-0/checkpoint",
      "head_sha256": "80bcdddf6f671e9901e3805f8c311492c93d86ac63af0440d519b2d45bb90361",
      "adapter_sha256": "93a489ba6fd4eddb766e9f9bbd70fbec04babe33c7ecd3925f5f9973e325410a",
      "inference_temperature": 1.0
    },
    "eval_device": "mps",
    "resumed_evaluation": true,
    "resumed_source_hashes": {
      "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
      "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb",
      "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582",
      "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a",
      "kev/benchmark.py": "9c442efd8b6b852c574a646192c61ee91afa1306e3b2978bb6391ae1b91a3fa0",
      "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519",
      "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59",
      "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435",
      "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536",
      "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf",
      "kev/data.py": "ef1be396e7218ba5c94f0f67de2ceff23a39656b57199cb3aa3192cde53e8320",
      "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4",
      "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c",
      "kev/experiment.py": "d12ab7f0765de0f29364f3411b99f06bb80b1cbf120ca9ed2b03316ec766e828",
      "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243",
      "kev/metrics.py": "dba8d90550999edcd642581fb4da0367bd4b39d08471815184fa732d11e137b9",
      "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909",
      "kev/model.py": "c2129026ab56395f78c7d3071410541d5423dd7d97f7371518570572f206c376",
      "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02",
      "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3",
      "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa",
      "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735",
      "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3",
      "kev/suite.py": "d18d5f4f02d079344d61c2a31882c524c90012ebfdb660bf9160e145f37e3d42",
      "kev/train.py": "e103e99d3a68ab99eb58f135320fbc46b18514fcc2b195d1c6879633836e99b9",
      "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763",
      "modal_app.py": "7a13e9d29b3fc68f8236a0c1e4d67079b40a2a328a296f7ced1022817dff82aa",
      "pyproject.toml": "7c17fbe9efcadda3eb488b59adf1b6db5938cd33af4d3bd389ce4359b73fcc7d",
      "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466"
    },
    "resumed_with_changed_non_evaluator_files": [],
    "resumed_git_commit": "ccd6e8b646068a6689792d76e3cae3698e9a015a"
  },
  "mechanism_checks": {
    "n": 8,
    "packed_max_delta": 0.0,
    "sibling_max_delta": 1.8417835235595703e-05,
    "tolerance": 0.001,
    "passed": true
  },
  "gates": {
    "passed": false,
    "checks": {
      "complete_coverage": true,
      "isolation_and_packing": true,
      "transfer_complete": true,
      "heldout_pairs_at_least_70pct": false,
      "transfer_confident_errors_below_10pct": true
    },
    "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release."
  },
  "calibration_fit": {
    "temperature": 1.319507910772894,
    "aggregation": "micro",
    "objective": "raw-logit NLL",
    "split": "calibration",
    "rows_sha256": "e8e669ff8b32eca59da46675dbd5a5ec138895480cdd475ed499f8d58249c0a2",
    "suite_sha256": "a8f50e481b7d90b97da049e0ff6a01cee2f1ed204aed61a8265af0edbb5514d2",
    "n": 1148
  },
  "wall_seconds": 1409.0520514999516,
  "promotable": false,
  "test_evaluated": false,
  "training_resources": {
    "wall_seconds": 4865.690968990326,
    "records_seen": 15142,
    "requested_records": 12506,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 1564,
    "forward_tokens": 3079220,
    "peak_device_bytes": 14623992320,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 4,
    "peak_rss_bytes": 11216687104
  }
}