| { |
| "release": "Gandalf 0.0.1", |
| "model_weights_sha256": "916370da013a428a8ed3e00791049d12b3b3fd5836969d8b0492bc5de372d5c8", |
| "all_rows_gate": { |
| "definition": { |
| "recall_at_least": 0.9, |
| "benign_false_positive_rate_at_most": 0.01, |
| "confidence": 0.95 |
| }, |
| "split": "gate_all", |
| "calibration_split": "calibration", |
| "passes": false, |
| "presets_passing": [], |
| "presets": [ |
| { |
| "preset": "identity.hate", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.7420825064182281, |
| "review_threshold": 0.4827668070793152, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 112, |
| "count": 141, |
| "rate": 0.7943262411347518, |
| "lower_bound": 0.7202325037685993 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 10, |
| "count": 879, |
| "rate": 0.011376564277588168, |
| "upper_bound": 0.020814502705574688 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 7, |
| "count": 121, |
| "rate": 0.05785123966942149, |
| "upper_bound": 0.11461150735579681 |
| }, |
| "other_unsafe": { |
| "false_positives": 3, |
| "count": 774, |
| "rate": 0.003875968992248062, |
| "upper_bound": 0.011333435287649487 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9857107125279372, |
| "sibling_unsafe": 0.9548678272082527, |
| "other_unsafe": 0.9928253339930727 |
| } |
| }, |
| { |
| "preset": "solicitation.sensitive_info", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.1611337810754776, |
| "review_threshold": 0.1611337810754776, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 59, |
| "count": 99, |
| "rate": 0.5959595959595959, |
| "lower_bound": 0.4974671538075806 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 11, |
| "count": 885, |
| "rate": 0.012429378531073447, |
| "upper_bound": 0.022119095752575677 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 160, |
| "rate": 0.0, |
| "upper_bound": 0.023447036650032713 |
| }, |
| "other_unsafe": { |
| "false_positives": 3, |
| "count": 771, |
| "rate": 0.0038910505836575876, |
| "upper_bound": 0.011377286821326028 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9120812646236375, |
| "sibling_unsafe": 0.9159090909090909, |
| "other_unsafe": 0.925559092874268 |
| } |
| }, |
| { |
| "preset": "abuse.harassment", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.6766293048858643, |
| "review_threshold": 0.13853582739830017, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 71, |
| "count": 148, |
| "rate": 0.4797297297297297, |
| "lower_bound": 0.40077634931400097 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 22, |
| "count": 885, |
| "rate": 0.024858757062146894, |
| "upper_bound": 0.03735199786654058 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 19, |
| "count": 89, |
| "rate": 0.21348314606741572, |
| "upper_bound": 0.3095304123499324 |
| }, |
| "other_unsafe": { |
| "false_positives": 43, |
| "count": 793, |
| "rate": 0.05422446406052964, |
| "upper_bound": 0.07224370244508971 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.8892579019697664, |
| "sibling_unsafe": 0.7250227755845733, |
| "other_unsafe": 0.8773729593401725 |
| } |
| }, |
| { |
| "preset": "abuse.threat", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.31466569006443024, |
| "review_threshold": 0.16331622004508972, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 9, |
| "count": 15, |
| "rate": 0.6, |
| "lower_bound": 0.357464427565077 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 18, |
| "count": 885, |
| "rate": 0.020338983050847456, |
| "upper_bound": 0.03192080218200452 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 104, |
| "count": 287, |
| "rate": 0.3623693379790941, |
| "upper_bound": 0.41946152671726716 |
| }, |
| "other_unsafe": { |
| "false_positives": 51, |
| "count": 728, |
| "rate": 0.07005494505494506, |
| "upper_bound": 0.09094152904389927 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.8773634651600754, |
| "sibling_unsafe": 0.6471544715447154, |
| "other_unsafe": 0.8507326007326007 |
| } |
| }, |
| { |
| "preset": "identity.slur", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.6240185797214508, |
| "review_threshold": 0.20658144354820251, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 112, |
| "count": 258, |
| "rate": 0.43410852713178294, |
| "lower_bound": 0.375032749729709 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 18, |
| "count": 885, |
| "rate": 0.020338983050847456, |
| "upper_bound": 0.03192080218200452 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 4, |
| "count": 90, |
| "rate": 0.044444444444444446, |
| "upper_bound": 0.1087701467709492 |
| }, |
| "other_unsafe": { |
| "false_positives": 2, |
| "count": 682, |
| "rate": 0.002932551319648094, |
| "upper_bound": 0.010628980613888749 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.8743397713835238, |
| "sibling_unsafe": 0.7653316106804479, |
| "other_unsafe": 0.9192070744958967 |
| } |
| }, |
| { |
| "preset": "abuse.personal_attack", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.5848842859268188, |
| "review_threshold": 0.16304577887058258, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 39, |
| "count": 226, |
| "rate": 0.17256637168141592, |
| "lower_bound": 0.1288810871596181 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 20, |
| "count": 885, |
| "rate": 0.022598870056497175, |
| "upper_bound": 0.03464835611231098 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 11, |
| "rate": 0.0, |
| "upper_bound": 0.2588400172488141 |
| }, |
| "other_unsafe": { |
| "false_positives": 6, |
| "count": 793, |
| "rate": 0.007566204287515763, |
| "upper_bound": 0.0164083966003853 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.7864556772161392, |
| "sibling_unsafe": 0.7936444086886565, |
| "other_unsafe": 0.8302458458413775 |
| } |
| }, |
| { |
| "preset": "violence.instruction", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.5611147284507751, |
| "review_threshold": 0.26542097330093384, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 115, |
| "count": 175, |
| "rate": 0.6571428571428571, |
| "lower_bound": 0.584117708864417 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 25, |
| "count": 885, |
| "rate": 0.02824858757062147, |
| "upper_bound": 0.04136900694643936 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 53, |
| "count": 413, |
| "rate": 0.12832929782082325, |
| "upper_bound": 0.1640445314883412 |
| }, |
| "other_unsafe": { |
| "false_positives": 5, |
| "count": 442, |
| "rate": 0.011312217194570135, |
| "upper_bound": 0.026204759836448127 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.920639225181598, |
| "sibling_unsafe": 0.8439156001383604, |
| "other_unsafe": 0.9315319974143503 |
| } |
| }, |
| { |
| "preset": "weapons.illegal", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.11735279485583305, |
| "review_threshold": 0.11735279485583305, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 46, |
| "count": 62, |
| "rate": 0.7419354838709677, |
| "lower_bound": 0.621186360610032 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 10, |
| "count": 885, |
| "rate": 0.011299435028248588, |
| "upper_bound": 0.020674255542985034 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 48, |
| "count": 521, |
| "rate": 0.09213051823416507, |
| "upper_bound": 0.12003856391347649 |
| }, |
| "other_unsafe": { |
| "false_positives": 3, |
| "count": 447, |
| "rate": 0.006711409395973154, |
| "upper_bound": 0.019544381284035284 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9664479679241844, |
| "sibling_unsafe": 0.9124821992446288, |
| "other_unsafe": 0.9743450963411994 |
| } |
| }, |
| { |
| "preset": "substances.illegal", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.26555532217025757, |
| "review_threshold": 0.12688001990318298, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 92, |
| "count": 110, |
| "rate": 0.8363636363636363, |
| "lower_bound": 0.7561132241492788 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 20, |
| "count": 885, |
| "rate": 0.022598870056497175, |
| "upper_bound": 0.03464835611231098 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 22, |
| "count": 433, |
| "rate": 0.050808314087759814, |
| "upper_bound": 0.07572784460622717 |
| }, |
| "other_unsafe": { |
| "false_positives": 7, |
| "count": 487, |
| "rate": 0.014373716632443531, |
| "upper_bound": 0.02936939336719413 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.970919363122753, |
| "sibling_unsafe": 0.936972496325845, |
| "other_unsafe": 0.9811274967332462 |
| } |
| }, |
| { |
| "preset": "sexual.explicit", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.43256138265132904, |
| "review_threshold": 0.0004432754940353334, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 69, |
| "count": 96, |
| "rate": 0.71875, |
| "lower_bound": 0.6217391618504083 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 8, |
| "count": 885, |
| "rate": 0.00903954802259887, |
| "upper_bound": 0.01773558559853817 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 5, |
| "count": 198, |
| "rate": 0.025252525252525252, |
| "upper_bound": 0.05774318065312027 |
| }, |
| "other_unsafe": { |
| "false_positives": 4, |
| "count": 725, |
| "rate": 0.005517241379310344, |
| "upper_bound": 0.014099609248300076 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9306026365348399, |
| "sibling_unsafe": 0.9128787878787878, |
| "other_unsafe": 0.9520402298850574 |
| } |
| }, |
| { |
| "preset": "sexual.harassment", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.10330371931195259, |
| "review_threshold": 0.10330371931195259, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 51, |
| "count": 231, |
| "rate": 0.22077922077922077, |
| "lower_bound": 0.17210142724225444 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 8, |
| "count": 885, |
| "rate": 0.00903954802259887, |
| "upper_bound": 0.01773558559853817 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 73, |
| "rate": 0.0, |
| "upper_bound": 0.04999375338358389 |
| }, |
| "other_unsafe": { |
| "false_positives": 4, |
| "count": 715, |
| "rate": 0.005594405594405594, |
| "upper_bound": 0.014295563368790107 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.7145840976349451, |
| "sibling_unsafe": 0.6670224752416534, |
| "other_unsafe": 0.816177761632307 |
| } |
| }, |
| { |
| "preset": "sexual.nonconsensual_imagery", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.11740976199507713, |
| "review_threshold": 0.11740976199507713, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 58, |
| "count": 205, |
| "rate": 0.28292682926829266, |
| "lower_bound": 0.2257000161883812 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 13, |
| "count": 885, |
| "rate": 0.014689265536723164, |
| "upper_bound": 0.024969364968836203 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 120, |
| "rate": 0.0, |
| "upper_bound": 0.031020271055929513 |
| }, |
| "other_unsafe": { |
| "false_positives": 0, |
| "count": 694, |
| "rate": 0.0, |
| "upper_bound": 0.005504974194716966 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.7637205456800331, |
| "sibling_unsafe": 0.7705284552845528, |
| "other_unsafe": 0.8419976101778309 |
| } |
| }, |
| { |
| "preset": "self_harm.encouragement", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.054427845403552055, |
| "review_threshold": 0.054427845403552055, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 32, |
| "count": 38, |
| "rate": 0.8421052631578947, |
| "lower_bound": 0.6958287736272311 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 20, |
| "count": 885, |
| "rate": 0.022598870056497175, |
| "upper_bound": 0.03464835611231098 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 221, |
| "rate": 0.0, |
| "upper_bound": 0.01708580618533225 |
| }, |
| "other_unsafe": { |
| "false_positives": 5, |
| "count": 771, |
| "rate": 0.00648508430609598, |
| "upper_bound": 0.015090693018716931 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9418376449598572, |
| "sibling_unsafe": 0.9381995713265063, |
| "other_unsafe": 0.9586319885316403 |
| } |
| }, |
| { |
| "preset": "self_harm.method", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.10501522198319435, |
| "review_threshold": 0.10501522198319435, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 32, |
| "count": 38, |
| "rate": 0.8421052631578947, |
| "lower_bound": 0.6958287736272311 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 19, |
| "count": 885, |
| "rate": 0.021468926553672316, |
| "upper_bound": 0.03328778358200399 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 221, |
| "rate": 0.0, |
| "upper_bound": 0.01708580618533225 |
| }, |
| "other_unsafe": { |
| "false_positives": 3, |
| "count": 771, |
| "rate": 0.0038910505836575876, |
| "upper_bound": 0.011377286821326028 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9393993458221825, |
| "sibling_unsafe": 0.9539175994284353, |
| "other_unsafe": 0.9632739436138985 |
| } |
| }, |
| { |
| "preset": "self_harm.graphic", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.10335911437869072, |
| "review_threshold": 0.10335911437869072, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 32, |
| "count": 38, |
| "rate": 0.8421052631578947, |
| "lower_bound": 0.6958287736272311 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 9, |
| "count": 885, |
| "rate": 0.010169491525423728, |
| "upper_bound": 0.019213877852455794 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 221, |
| "rate": 0.0, |
| "upper_bound": 0.01708580618533225 |
| }, |
| "other_unsafe": { |
| "false_positives": 3, |
| "count": 771, |
| "rate": 0.0038910505836575876, |
| "upper_bound": 0.011377286821326028 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9381801962533453, |
| "sibling_unsafe": 0.9493927125506073, |
| "other_unsafe": 0.9579493480783671 |
| } |
| }, |
| { |
| "preset": "self_harm.eating_disorder", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.17654937505722046, |
| "review_threshold": 0.17654937505722046, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 18, |
| "count": 106, |
| "rate": 0.16981132075471697, |
| "lower_bound": 0.110198735828486 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 15, |
| "count": 885, |
| "rate": 0.01694915254237288, |
| "upper_bound": 0.027776013490446793 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 153, |
| "rate": 0.0, |
| "upper_bound": 0.0244935017240324 |
| }, |
| "other_unsafe": { |
| "false_positives": 2, |
| "count": 771, |
| "rate": 0.0025940337224383916, |
| "upper_bound": 0.009408610809403362 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.6265216927832854, |
| "sibling_unsafe": 0.6220865704772475, |
| "other_unsafe": 0.6825612412206642 |
| } |
| }, |
| { |
| "preset": "fraud.deception", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.3355312645435333, |
| "review_threshold": 0.3355312645435333, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 14, |
| "count": 21, |
| "rate": 0.6666666666666666, |
| "lower_bound": 0.45373075657210293 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 17, |
| "count": 885, |
| "rate": 0.0192090395480226, |
| "upper_bound": 0.030546909615707128 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 71, |
| "count": 496, |
| "rate": 0.1431451612903226, |
| "upper_bound": 0.17671314467435695 |
| }, |
| "other_unsafe": { |
| "false_positives": 6, |
| "count": 513, |
| "rate": 0.011695906432748537, |
| "upper_bound": 0.02527979655807884 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9232714554748453, |
| "sibling_unsafe": 0.8211405529953917, |
| "other_unsafe": 0.9337231968810916 |
| } |
| }, |
| { |
| "preset": "fraud.financial_solicitation", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.19238756597042084, |
| "review_threshold": 0.16374947130680084, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 2, |
| "count": 14, |
| "rate": 0.14285714285714285, |
| "lower_bound": 0.040093073737921175 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 15, |
| "count": 885, |
| "rate": 0.01694915254237288, |
| "upper_bound": 0.027776013490446793 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 24, |
| "count": 496, |
| "rate": 0.04838709677419355, |
| "upper_bound": 0.07098754667357766 |
| }, |
| "other_unsafe": { |
| "false_positives": 2, |
| "count": 520, |
| "rate": 0.0038461538461538464, |
| "upper_bound": 0.013914044143853842 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.8067796610169492, |
| "sibling_unsafe": 0.6621543778801844, |
| "other_unsafe": 0.8744505494505495 |
| } |
| }, |
| { |
| "preset": "identity.impersonation", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.09971436113119125, |
| "review_threshold": 0.09971436113119125, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 0, |
| "count": 15, |
| "rate": 0.0, |
| "lower_bound": 0.0 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 10, |
| "count": 885, |
| "rate": 0.011299435028248588, |
| "upper_bound": 0.020674255542985034 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 2, |
| "count": 154, |
| "rate": 0.012987012987012988, |
| "upper_bound": 0.04611149782084855 |
| }, |
| "other_unsafe": { |
| "false_positives": 4, |
| "count": 861, |
| "rate": 0.004645760743321719, |
| "upper_bound": 0.011884163267938112 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.4903954802259887, |
| "sibling_unsafe": 0.5333333333333333, |
| "other_unsafe": 0.6277971351142083 |
| } |
| }, |
| { |
| "preset": "security.malware", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.1383366957306862, |
| "review_threshold": 0.1383366957306862, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 7, |
| "count": 8, |
| "rate": 0.875, |
| "lower_bound": 0.5291051942301386 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 8, |
| "count": 885, |
| "rate": 0.00903954802259887, |
| "upper_bound": 0.01773558559853817 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 7, |
| "count": 23, |
| "rate": 0.30434782608695654, |
| "upper_bound": 0.5086612906057261 |
| }, |
| "other_unsafe": { |
| "false_positives": 41, |
| "count": 999, |
| "rate": 0.04104104104104104, |
| "upper_bound": 0.05520302182999091 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9923728813559322, |
| "sibling_unsafe": 0.8967391304347826, |
| "other_unsafe": 0.9751001001001001 |
| } |
| }, |
| { |
| "preset": "crime.facilitation", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.7706991136074066, |
| "review_threshold": 0.38170838356018066, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 272, |
| "count": 508, |
| "rate": 0.5354330708661418, |
| "lower_bound": 0.49195822604350786 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 29, |
| "count": 885, |
| "rate": 0.0327683615819209, |
| "upper_bound": 0.046664742133047075 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 12, |
| "count": 121, |
| "rate": 0.09917355371900827, |
| "upper_bound": 0.16537073890179066 |
| }, |
| "other_unsafe": { |
| "false_positives": 6, |
| "count": 401, |
| "rate": 0.014962593516209476, |
| "upper_bound": 0.03225539988401834 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.9220739356732951, |
| "sibling_unsafe": 0.8421455066050628, |
| "other_unsafe": 0.9576845288353918 |
| } |
| }, |
| { |
| "preset": "ip.piracy", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.1751590594649315, |
| "review_threshold": 0.046332523226737976, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 0, |
| "count": 16, |
| "rate": 0.0, |
| "lower_bound": 0.0 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 10, |
| "count": 885, |
| "rate": 0.011299435028248588, |
| "upper_bound": 0.020674255542985034 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 18, |
| "rate": 0.0, |
| "upper_bound": 0.1758845505823749 |
| }, |
| "other_unsafe": { |
| "false_positives": 31, |
| "count": 996, |
| "rate": 0.03112449799196787, |
| "upper_bound": 0.04383977742376964 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.6040960451977401, |
| "sibling_unsafe": 0.28125, |
| "other_unsafe": 0.5664533132530121 |
| } |
| }, |
| { |
| "preset": "privacy.doxxing", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.15790241211652756, |
| "review_threshold": 0.15790241211652756, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 59, |
| "count": 114, |
| "rate": 0.5175438596491229, |
| "lower_bound": 0.4267487861701612 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 18, |
| "count": 885, |
| "rate": 0.020338983050847456, |
| "upper_bound": 0.03192080218200452 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 0, |
| "count": 145, |
| "rate": 0.0, |
| "upper_bound": 0.02580998860533614 |
| }, |
| "other_unsafe": { |
| "false_positives": 2, |
| "count": 771, |
| "rate": 0.0025940337224383916, |
| "upper_bound": 0.009408610809403362 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.8061552185548617, |
| "sibling_unsafe": 0.8352692075015125, |
| "other_unsafe": 0.8625275900516531 |
| } |
| }, |
| { |
| "preset": "extremism.promotion", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.4289531111717224, |
| "review_threshold": 0.40473344922065735, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": null |
| }, |
| "recall": { |
| "caught": 191, |
| "count": 704, |
| "rate": 0.2713068181818182, |
| "lower_bound": 0.23976848576370147 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 16, |
| "count": 885, |
| "rate": 0.01807909604519774, |
| "upper_bound": 0.029165532886600988 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 2, |
| "count": 70, |
| "rate": 0.02857142857142857, |
| "upper_bound": 0.09832443512391814 |
| }, |
| "other_unsafe": { |
| "false_positives": 7, |
| "count": 256, |
| "rate": 0.02734375, |
| "upper_bound": 0.055356383601243266 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.7400343477144324, |
| "sibling_unsafe": 0.5946022727272727, |
| "other_unsafe": 0.7830422141335227 |
| } |
| }, |
| { |
| "preset": "profanity.strong", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.39609213173389435, |
| "review_threshold": 0.39609213173389435, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 58, |
| "count": 98, |
| "rate": 0.5918367346938775, |
| "lower_bound": 0.49285190219841635 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 17, |
| "count": 885, |
| "rate": 0.0192090395480226, |
| "upper_bound": 0.030546909615707128 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 7, |
| "count": 250, |
| "rate": 0.028, |
| "upper_bound": 0.056658466945870976 |
| }, |
| "other_unsafe": { |
| "false_positives": 4, |
| "count": 682, |
| "rate": 0.005865102639296188, |
| "upper_bound": 0.014982713351166122 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.8979591836734694, |
| "sibling_unsafe": 0.7991428571428572, |
| "other_unsafe": 0.9460919265066731 |
| } |
| }, |
| { |
| "preset": "advice.dangerous", |
| "passes": false, |
| "thresholds": { |
| "action_threshold": 0.4932190179824829, |
| "review_threshold": 0.4932190179824829, |
| "selection": "action: lowest calibration threshold within the benign limit; review: 95% calibration recall", |
| "note": "review threshold reached the action threshold; the review band is empty" |
| }, |
| "recall": { |
| "caught": 4, |
| "count": 92, |
| "rate": 0.043478260869565216, |
| "lower_bound": 0.01703544061158062 |
| }, |
| "false_positive_rate": { |
| "safe": { |
| "false_positives": 11, |
| "count": 885, |
| "rate": 0.012429378531073447, |
| "upper_bound": 0.022119095752575677 |
| }, |
| "sibling_unsafe": { |
| "false_positives": 13, |
| "count": 140, |
| "rate": 0.09285714285714286, |
| "upper_bound": 0.1523918931775859 |
| }, |
| "other_unsafe": { |
| "false_positives": 23, |
| "count": 798, |
| "rate": 0.02882205513784461, |
| "upper_bound": 0.04287782694807783 |
| } |
| }, |
| "auroc_vs": { |
| "safe": 0.5252026529108327, |
| "sibling_unsafe": 0.35993788819875777, |
| "other_unsafe": 0.6061485234826196 |
| } |
| } |
| ] |
| }, |
| "dynahate": { |
| "status": "external check on human-labelled DynaHate test rows; threshold frozen by the preset gate", |
| "test_sha256": "0eaa0bca8d509686c74ceb7fedc58a508a2614bd9509503d4b0c95afa8c7af7b", |
| "rows": 4118, |
| "threshold": 0.7420825064182281, |
| "recall_hate": { |
| "hits": 1507, |
| "count": 2267, |
| "rate": 0.6647551830613145, |
| "lower_bound": 0.6450578039324775, |
| "upper_bound": 0.6838951269985043 |
| }, |
| "flag_rate_nothate": { |
| "hits": 340, |
| "count": 1851, |
| "rate": 0.1836844948676391, |
| "lower_bound": 0.1667049324987952, |
| "upper_bound": 0.20197431213319922 |
| }, |
| "auroc_hate_vs_nothate": 0.8238435953145417 |
| }, |
| "limitations": [ |
| "Source category tags are not all exact human policy judgements.", |
| "Teacher-filtered results are not independent teacher comparisons.", |
| "Generated diagnostic examples are not human acceptance evidence.", |
| "No physical-phone performance or deployment-compliance claim." |
| ] |
| } |
|
|
