[
  {
    "name": "BART-large MNLI",
    "model": "facebook/bart-large-mnli",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.6362744923589948,
    "auc": {
      "number": 0.4784391111111111,
      "date": 0.9242776612148942,
      "string": 0.7570571789321789,
      "table": 0.6105985609760144,
      "prose": 0.891619842839355
    },
    "pairs_per_s": 24.5,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 1024,
    "ours": false
  },
  {
    "name": "cross-encoder DeBERTa-v3-base",
    "model": "cross-encoder/nli-deberta-v3-base",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.644738021188759,
    "auc": {
      "number": 0.44977955555555554,
      "date": 0.9497017113678303,
      "string": 0.6665088383838383,
      "table": 0.631454675168817,
      "prose": 0.8665808665808665
    },
    "pairs_per_s": 96.2,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "cross-encoder MiniLM-L6",
    "model": "cross-encoder/nli-MiniLM2-L6-H768",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.5927182536915374,
    "auc": {
      "number": 0.44786488888888887,
      "date": 0.8190866806542126,
      "string": 0.5469651875901876,
      "table": 0.5710080357488511,
      "prose": 0.8484582143118728
    },
    "pairs_per_s": 318.4,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "DeBERTa-v3-large MNLI/FEVER/ANLI",
    "model": "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.7693892982804592,
    "auc": {
      "number": 0.6419208888888889,
      "date": 0.9954498344245305,
      "string": 0.9134875541125541,
      "table": 0.7472576140517762,
      "prose": 0.9447217252095301
    },
    "pairs_per_s": 88.4,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "DeBERTa-v3-large zero-shot",
    "model": "MoritzLaurer/deberta-v3-large-zeroshot-v2.0",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.7856283280264451,
    "auc": {
      "number": 0.6575431111111111,
      "date": 0.994580879193104,
      "string": 0.9412315115440115,
      "table": 0.7659018786950342,
      "prose": 0.9286289774094653
    },
    "pairs_per_s": 89.0,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "grounding-en (ours)",
    "model": "nutrientdocs/grounding-en",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.8815400924541517,
    "auc": {
      "number": 0.9231893333333333,
      "date": 0.998429560909022,
      "string": 0.9549625721500722,
      "table": 0.8631714938307792,
      "prose": 0.9637233051867198
    },
    "pairs_per_s": 148.2,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 8192,
    "ours": true
  },
  {
    "name": "grounding-multilingual (ours)",
    "model": "nutrientdocs/grounding-multilingual",
    "config": "en",
    "open": false,
    "n": 5000,
    "overall_auc": 0.9249458851297964,
    "auc": {
      "number": 0.9689866666666667,
      "date": 0.9985970322808969,
      "string": 0.9487170815295816,
      "table": 0.9148366580575942,
      "prose": 0.9703832752613241
    },
    "pairs_per_s": 243.5,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 8192,
    "ours": true
  },
  {
    "name": "grounding-multilingual (ours)",
    "model": "nutrientdocs/grounding-multilingual",
    "config": "multi",
    "open": false,
    "n": 8421,
    "overall_auc": 0.9652247630640813,
    "auc": {
      "number": 0.9987015555555555,
      "date": null,
      "string": null,
      "table": 0.9985981272248878,
      "prose": 0.9261168118990385
    },
    "pairs_per_s": 337.7,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 8192,
    "ours": true
  },
  {
    "name": "mDeBERTa-v3-base XNLI-2mil7",
    "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.6998874981384613,
    "auc": {
      "number": 0.4928408888888889,
      "date": 0.9457582345357567,
      "string": 0.8369633838383839,
      "table": 0.664911263874525,
      "prose": 0.9090885310397505
    },
    "pairs_per_s": 215.0,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "mDeBERTa-v3-base XNLI-2mil7",
    "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7",
    "config": "multi",
    "open": true,
    "n": 8421,
    "overall_auc": 0.8937062502752443,
    "auc": {
      "number": 0.7750426666666667,
      "date": null,
      "string": null,
      "table": 0.7274667234174277,
      "prose": 0.9609073768028846
    },
    "pairs_per_s": 301.2,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "mDeBERTa-v3-base MNLI-XNLI",
    "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.6294148925341116,
    "auc": {
      "number": 0.43475644444444445,
      "date": 0.8523755656108597,
      "string": 0.8139204545454546,
      "table": 0.6084399143885807,
      "prose": 0.8505352041937407
    },
    "pairs_per_s": 214.6,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "mDeBERTa-v3-base MNLI-XNLI",
    "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
    "config": "multi",
    "open": true,
    "n": 8421,
    "overall_auc": 0.8305958149319597,
    "auc": {
      "number": 0.8063175555555555,
      "date": null,
      "string": null,
      "table": 0.7796536913790435,
      "prose": 0.8636295072115384
    },
    "pairs_per_s": 304.8,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "RoBERTa-large MNLI",
    "model": "FacebookAI/roberta-large-mnli",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.5881571031267598,
    "auc": {
      "number": 0.4420177777777778,
      "date": 0.887693066053237,
      "string": 0.5542929292929293,
      "table": 0.5558180923253898,
      "prose": 0.857927614025175
    },
    "pairs_per_s": 56.8,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "XLM-R-large XNLI",
    "model": "joeddav/xlm-roberta-large-xnli",
    "config": "en",
    "open": true,
    "n": 5000,
    "overall_auc": 0.5716570972756371,
    "auc": {
      "number": 0.44068444444444443,
      "date": 0.7327852069061402,
      "string": 0.5111381673881674,
      "table": 0.5335998274498526,
      "prose": 0.8582833704784925
    },
    "pairs_per_s": 55.7,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  },
  {
    "name": "XLM-R-large XNLI",
    "model": "joeddav/xlm-roberta-large-xnli",
    "config": "multi",
    "open": true,
    "n": 8421,
    "overall_auc": 0.8107369827807182,
    "auc": {
      "number": 0.7651982222222222,
      "date": null,
      "string": null,
      "table": 0.7150696486611979,
      "prose": 0.8689075270432692
    },
    "pairs_per_s": 80.1,
    "hardware": "NVIDIA RTX A6000",
    "ctx": 512,
    "ours": false
  }
]