{
  "holdout": "E:\\Bleeding_edge\\data\\model_suite\\q_retriever\\holdout\\q_retriever_v7_holdout.jsonl",
  "results": [
    {
      "name": "Q-RAG-50M-Sovereign",
      "preds": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0
      ],
      "latency_ms": 244.68170007069907
    },
    {
      "name": "intfloat/e5-small-v2",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        0,
        0,
        1
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        0,
        0,
        1
      ],
      "scores": [
        0.8916620016098022,
        0.8089611530303955,
        0.869024395942688,
        0.7798522710800171,
        0.8496180176734924,
        0.7867108583450317,
        0.8981657028198242,
        0.7856180667877197,
        0.8399478793144226,
        0.7826312184333801,
        0.8485733270645142,
        0.8053742051124573,
        0.6855081915855408,
        0.7221635580062866,
        0.7234992980957031,
        0.736674964427948,
        0.6867632865905762,
        0.8917654752731323,
        0.8382331132888794,
        0.8651498556137085,
        0.7689701914787292,
        0.9014170169830322,
        0.8094509243965149,
        0.8926041722297668,
        0.848368763923645,
        0.8760983347892761,
        0.8041742444038391,
        0.775821328163147,
        0.7300697565078735,
        0.851111650466919
      ],
      "calibrated_threshold": 0.8399478793144226,
      "calibrated_accuracy": 0.9,
      "oracle_threshold": 0.8399478793144226,
      "oracle_accuracy": 0.9,
      "latency_ms": 34.383225440979004,
      "display_name": "e5-small-v2",
      "kind": "embed",
      "params": "33M"
    },
    {
      "name": "BAAI/bge-small-en-v1.5",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        0,
        0,
        0,
        1
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        0,
        0,
        0,
        1
      ],
      "scores": [
        0.8222349882125854,
        0.6364327073097229,
        0.7970625758171082,
        0.5989657640457153,
        0.6827614307403564,
        0.5954609513282776,
        0.8256340026855469,
        0.4885215163230896,
        0.7420240640640259,
        0.6457233428955078,
        0.7767975926399231,
        0.5980231165885925,
        0.3933045268058777,
        0.34676414728164673,
        0.4929644465446472,
        0.4458352327346802,
        0.3490070104598999,
        0.819932222366333,
        0.6132599711418152,
        0.8541251420974731,
        0.6359962821006775,
        0.7759215831756592,
        0.7780344486236572,
        0.820052981376648,
        0.7437193989753723,
        0.7105350494384766,
        0.5850974917411804,
        0.6140377521514893,
        0.4791330397129059,
        0.702117919921875
      ],
      "calibrated_threshold": 0.6827614307403564,
      "calibrated_accuracy": 0.8666666666666667,
      "oracle_threshold": 0.6827614307403564,
      "oracle_accuracy": 0.8666666666666667,
      "latency_ms": 25.878326098124187,
      "display_name": "bge-small-en-v1.5",
      "kind": "embed",
      "params": "33M"
    },
    {
      "name": "BAAI/bge-m3",
      "preds_calibrated": [
        1,
        1,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        0,
        0,
        0,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        0,
        0,
        0,
        0,
        0
      ],
      "scores": [
        0.7952277660369873,
        0.6124909520149231,
        0.7697983980178833,
        0.5962614417076111,
        0.6063465476036072,
        0.5212503671646118,
        0.6881134510040283,
        0.4766281545162201,
        0.672714114189148,
        0.557146430015564,
        0.6927255392074585,
        0.5258240103721619,
        0.31215590238571167,
        0.341736376285553,
        0.26389601826667786,
        0.3178122639656067,
        0.28342440724372864,
        0.6562333703041077,
        0.5707750916481018,
        0.6418322324752808,
        0.4513903856277466,
        0.7424371242523193,
        0.653995931148529,
        0.8097206354141235,
        0.6390174627304077,
        0.5953260064125061,
        0.42827874422073364,
        0.4729628562927246,
        0.32285916805267334,
        0.5191959142684937
      ],
      "calibrated_threshold": 0.6063465476036072,
      "calibrated_accuracy": 0.8333333333333334,
      "oracle_threshold": 0.6418322324752808,
      "oracle_accuracy": 0.8666666666666667,
      "latency_ms": 214.86491362253827,
      "display_name": "bge-m3",
      "kind": "embed",
      "params": "568M"
    },
    {
      "name": "cross-encoder/ms-marco-MiniLM-L-6-v2",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "scores": [
        8.978658676147461,
        -5.629965305328369,
        5.152802467346191,
        -5.701674461364746,
        4.370293140411377,
        -6.5090813636779785,
        1.4527740478515625,
        -11.096714973449707,
        6.809542655944824,
        -3.0231895446777344,
        7.313239097595215,
        -10.806428909301758,
        -11.031339645385742,
        -11.17176628112793,
        -11.366830825805664,
        -11.241408348083496,
        -11.452680587768555,
        8.920427322387695,
        -3.528209686279297,
        9.688615798950195,
        -11.15275764465332,
        10.572490692138672,
        6.039632320404053,
        9.054561614990234,
        7.717207908630371,
        1.6525673866271973,
        -11.153036117553711,
        4.15381383895874,
        -9.92934799194336,
        -3.4672374725341797
      ],
      "calibrated_threshold": 0.0,
      "calibrated_accuracy": 0.9333333333333333,
      "oracle_threshold": 0.0,
      "oracle_accuracy": 0.9333333333333333,
      "latency_ms": 13.800207773844402,
      "display_name": "ms-marco-MiniLM-L-6-v2",
      "kind": "reranker",
      "params": "23M"
    },
    {
      "name": "cross-encoder/ms-marco-MiniLM-L-12-v2",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "scores": [
        9.410582542419434,
        -4.11091423034668,
        6.008073806762695,
        -3.397266387939453,
        4.543247222900391,
        -5.405708312988281,
        2.5701229572296143,
        -11.123125076293945,
        6.890682220458984,
        -3.017399787902832,
        7.121694564819336,
        -10.66909408569336,
        -11.099981307983398,
        -11.319366455078125,
        -11.270811080932617,
        -11.277335166931152,
        -11.20413589477539,
        9.082860946655273,
        -3.7397303581237793,
        8.564966201782227,
        -11.257317543029785,
        10.457947731018066,
        7.229440689086914,
        9.312555313110352,
        7.391736030578613,
        3.49010968208313,
        -11.228635787963867,
        4.7077107429504395,
        -9.112346649169922,
        -4.001871109008789
      ],
      "calibrated_threshold": 0.0,
      "calibrated_accuracy": 0.9333333333333333,
      "oracle_threshold": 0.0,
      "oracle_accuracy": 0.9333333333333333,
      "latency_ms": 24.0323543548584,
      "display_name": "ms-marco-MiniLM-L-12-v2",
      "kind": "reranker",
      "params": "33M"
    },
    {
      "name": "mixedbread-ai/mxbai-rerank-xsmall-v1",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        0,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        0,
        0,
        0
      ],
      "scores": [
        0.9609375,
        0.048309326171875,
        0.474853515625,
        0.08038330078125,
        0.78125,
        0.26806640625,
        0.55908203125,
        0.01898193359375,
        0.86181640625,
        0.1624755859375,
        0.63671875,
        0.033660888671875,
        0.0300445556640625,
        0.036773681640625,
        0.02337646484375,
        0.0095977783203125,
        0.04248046875,
        0.7998046875,
        0.188720703125,
        0.92529296875,
        0.02947998046875,
        0.99560546875,
        0.2021484375,
        0.9833984375,
        0.77099609375,
        0.75048828125,
        0.038909912109375,
        0.2333984375,
        0.3017578125,
        0.1456298828125
      ],
      "calibrated_threshold": 0.474853515625,
      "calibrated_accuracy": 0.9333333333333333,
      "oracle_threshold": 0.474853515625,
      "oracle_accuracy": 0.9333333333333333,
      "latency_ms": 247.4942366282145,
      "display_name": "mxbai-rerank-xsmall-v1",
      "kind": "reranker",
      "params": "70M"
    },
    {
      "name": "jina-reranker-v1-tiny-en",
      "display_name": "jina-reranker-v1-tiny-en",
      "kind": "reranker",
      "params": "33M",
      "error": "No module named 'transformers.onnx'"
    },
    {
      "name": "jina-reranker-v2-base-multilingual",
      "display_name": "jina-reranker-v2-base-multilingual",
      "kind": "reranker",
      "params": "278M",
      "error": "cannot import name 'create_position_ids_from_input_ids' from 'transformers.models.xlm_roberta.modeling_xlm_roberta' (E:\\Bleeding_edge\\.venv\\lib\\site-packages\\transformers\\models\\xlm_roberta\\modeling_xlm_roberta.py)"
    },
    {
      "name": "Alibaba-NLP/gte-reranker-modernbert-base",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        0,
        0,
        1,
        0,
        0
      ],
      "scores": [
        0.9771910905838013,
        0.7230595946311951,
        0.9418483376502991,
        0.8091198801994324,
        0.9287675619125366,
        0.7786911129951477,
        0.9291960597038269,
        0.8688681125640869,
        0.9533499479293823,
        0.7584269046783447,
        0.9666489958763123,
        0.8705883622169495,
        0.7627046704292297,
        0.33758971095085144,
        0.21788223087787628,
        0.15015909075737,
        0.12050151824951172,
        0.9472298622131348,
        0.8083674907684326,
        0.9470255970954895,
        0.8073157668113708,
        0.9804779887199402,
        0.7916922569274902,
        0.9685184955596924,
        0.8881627917289734,
        0.8436237573623657,
        0.4681851267814636,
        0.8731357455253601,
        0.7819688320159912,
        0.8426507115364075
      ],
      "calibrated_threshold": 0.9287675619125366,
      "calibrated_accuracy": 0.9333333333333333,
      "oracle_threshold": 0.8731357455253601,
      "oracle_accuracy": 0.9333333333333333,
      "latency_ms": 54.10958131154378,
      "display_name": "gte-reranker-modernbert-base",
      "kind": "reranker",
      "params": "149M"
    },
    {
      "name": "BAAI/bge-reranker-base",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        0,
        0,
        0,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        1,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        0,
        0,
        0
      ],
      "scores": [
        0.9999294281005859,
        0.09618812054395676,
        0.9870009422302246,
        0.09389989823102951,
        0.9907271862030029,
        0.7318733334541321,
        0.9834257364273071,
        0.002177225425839424,
        0.9397813081741333,
        0.0065153841860592365,
        0.9467273354530334,
        0.02877599559724331,
        3.731523247552104e-05,
        3.741564432857558e-05,
        3.738689702004194e-05,
        3.7443132896441966e-05,
        3.742870831047185e-05,
        0.9981344938278198,
        0.90499347448349,
        0.9990606904029846,
        0.025732049718499184,
        0.9996981620788574,
        0.7109959721565247,
        0.9999665021896362,
        0.981747031211853,
        0.8546900153160095,
        0.0024025740567594767,
        0.132459357380867,
        0.00024865291197784245,
        0.047000542283058167
      ],
      "calibrated_threshold": 0.9397813081741333,
      "calibrated_accuracy": 0.9,
      "oracle_threshold": 0.8546900153160095,
      "oracle_accuracy": 0.9,
      "latency_ms": 59.97885068257649,
      "display_name": "bge-reranker-base",
      "kind": "reranker",
      "params": "278M"
    },
    {
      "name": "BAAI/bge-reranker-large",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "scores": [
        0.9996817111968994,
        0.004382436629384756,
        0.8981118202209473,
        0.004344495479017496,
        0.9427682757377625,
        0.45768699049949646,
        0.918315052986145,
        0.0001671183854341507,
        0.9756414294242859,
        0.003535185707733035,
        0.984777569770813,
        0.2633640170097351,
        7.633084169356152e-05,
        7.688941695960239e-05,
        7.63679709052667e-05,
        7.637962698936462e-05,
        7.634925714228302e-05,
        0.9980623126029968,
        0.033723149448633194,
        0.9996077418327332,
        0.0033015920780599117,
        0.9982230067253113,
        0.19221831858158112,
        0.9996492862701416,
        0.9872984290122986,
        0.8963823914527893,
        0.0004903539665974677,
        0.6203130483627319,
        0.024030504748225212,
        0.040110133588314056
      ],
      "calibrated_threshold": 0.5,
      "calibrated_accuracy": 0.9666666666666667,
      "oracle_threshold": 0.5,
      "oracle_accuracy": 0.9666666666666667,
      "latency_ms": 174.3239164352417,
      "display_name": "bge-reranker-large",
      "kind": "reranker",
      "params": "560M"
    },
    {
      "name": "BAAI/bge-reranker-v2-m3",
      "preds_calibrated": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "preds_oracle": [
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        1,
        0,
        1,
        0,
        0
      ],
      "scores": [
        0.9998419284820557,
        0.009659526869654655,
        0.9561222195625305,
        0.01838972605764866,
        0.9276471138000488,
        0.26330816745758057,
        0.7980142831802368,
        0.0009156953310593963,
        0.977929949760437,
        0.013624642044305801,
        0.8832734823226929,
        0.13276134431362152,
        1.609356331755407e-05,
        1.6049774785642512e-05,
        1.6120136933750473e-05,
        1.668766708462499e-05,
        1.6149435396073386e-05,
        0.9898882508277893,
        0.2638988792896271,
        0.9989393353462219,
        0.009833737276494503,
        0.9994114637374878,
        0.09135906398296356,
        0.9998906850814819,
        0.9927875399589539,
        0.8755790591239929,
        0.00046630724682472646,
        0.747411847114563,
        0.002439234172925353,
        0.0015554826240986586
      ],
      "calibrated_threshold": 0.5,
      "calibrated_accuracy": 0.9666666666666667,
      "oracle_threshold": 0.5,
      "oracle_accuracy": 0.9666666666666667,
      "latency_ms": 175.28400421142578,
      "display_name": "bge-reranker-v2-m3",
      "kind": "reranker",
      "params": "568M"
    }
  ]
}