| [ |
| { |
| "name": "BART-large MNLI", |
| "model": "facebook/bart-large-mnli", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.6362744923589948, |
| "auc": { |
| "number": 0.4784391111111111, |
| "date": 0.9242776612148942, |
| "string": 0.7570571789321789, |
| "table": 0.6105985609760144, |
| "prose": 0.891619842839355 |
| }, |
| "pairs_per_s": 24.5, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 1024, |
| "ours": false |
| }, |
| { |
| "name": "cross-encoder DeBERTa-v3-base", |
| "model": "cross-encoder/nli-deberta-v3-base", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.644738021188759, |
| "auc": { |
| "number": 0.44977955555555554, |
| "date": 0.9497017113678303, |
| "string": 0.6665088383838383, |
| "table": 0.631454675168817, |
| "prose": 0.8665808665808665 |
| }, |
| "pairs_per_s": 96.2, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "cross-encoder MiniLM-L6", |
| "model": "cross-encoder/nli-MiniLM2-L6-H768", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.5927182536915374, |
| "auc": { |
| "number": 0.44786488888888887, |
| "date": 0.8190866806542126, |
| "string": 0.5469651875901876, |
| "table": 0.5710080357488511, |
| "prose": 0.8484582143118728 |
| }, |
| "pairs_per_s": 318.4, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "DeBERTa-v3-large MNLI/FEVER/ANLI", |
| "model": "MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.7693892982804592, |
| "auc": { |
| "number": 0.6419208888888889, |
| "date": 0.9954498344245305, |
| "string": 0.9134875541125541, |
| "table": 0.7472576140517762, |
| "prose": 0.9447217252095301 |
| }, |
| "pairs_per_s": 88.4, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "DeBERTa-v3-large zero-shot", |
| "model": "MoritzLaurer/deberta-v3-large-zeroshot-v2.0", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.7856283280264451, |
| "auc": { |
| "number": 0.6575431111111111, |
| "date": 0.994580879193104, |
| "string": 0.9412315115440115, |
| "table": 0.7659018786950342, |
| "prose": 0.9286289774094653 |
| }, |
| "pairs_per_s": 89.0, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "grounding-en (ours)", |
| "model": "nutrientdocs/grounding-en", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.8815400924541517, |
| "auc": { |
| "number": 0.9231893333333333, |
| "date": 0.998429560909022, |
| "string": 0.9549625721500722, |
| "table": 0.8631714938307792, |
| "prose": 0.9637233051867198 |
| }, |
| "pairs_per_s": 148.2, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 8192, |
| "ours": true |
| }, |
| { |
| "name": "grounding-multilingual (ours)", |
| "model": "nutrientdocs/grounding-multilingual", |
| "config": "en", |
| "open": false, |
| "n": 5000, |
| "overall_auc": 0.9249458851297964, |
| "auc": { |
| "number": 0.9689866666666667, |
| "date": 0.9985970322808969, |
| "string": 0.9487170815295816, |
| "table": 0.9148366580575942, |
| "prose": 0.9703832752613241 |
| }, |
| "pairs_per_s": 243.5, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 8192, |
| "ours": true |
| }, |
| { |
| "name": "grounding-multilingual (ours)", |
| "model": "nutrientdocs/grounding-multilingual", |
| "config": "multi", |
| "open": false, |
| "n": 8421, |
| "overall_auc": 0.9652247630640813, |
| "auc": { |
| "number": 0.9987015555555555, |
| "date": null, |
| "string": null, |
| "table": 0.9985981272248878, |
| "prose": 0.9261168118990385 |
| }, |
| "pairs_per_s": 337.7, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 8192, |
| "ours": true |
| }, |
| { |
| "name": "mDeBERTa-v3-base XNLI-2mil7", |
| "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.6998874981384613, |
| "auc": { |
| "number": 0.4928408888888889, |
| "date": 0.9457582345357567, |
| "string": 0.8369633838383839, |
| "table": 0.664911263874525, |
| "prose": 0.9090885310397505 |
| }, |
| "pairs_per_s": 215.0, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "mDeBERTa-v3-base XNLI-2mil7", |
| "model": "MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7", |
| "config": "multi", |
| "open": true, |
| "n": 8421, |
| "overall_auc": 0.8937062502752443, |
| "auc": { |
| "number": 0.7750426666666667, |
| "date": null, |
| "string": null, |
| "table": 0.7274667234174277, |
| "prose": 0.9609073768028846 |
| }, |
| "pairs_per_s": 301.2, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "mDeBERTa-v3-base MNLI-XNLI", |
| "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.6294148925341116, |
| "auc": { |
| "number": 0.43475644444444445, |
| "date": 0.8523755656108597, |
| "string": 0.8139204545454546, |
| "table": 0.6084399143885807, |
| "prose": 0.8505352041937407 |
| }, |
| "pairs_per_s": 214.6, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "mDeBERTa-v3-base MNLI-XNLI", |
| "model": "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", |
| "config": "multi", |
| "open": true, |
| "n": 8421, |
| "overall_auc": 0.8305958149319597, |
| "auc": { |
| "number": 0.8063175555555555, |
| "date": null, |
| "string": null, |
| "table": 0.7796536913790435, |
| "prose": 0.8636295072115384 |
| }, |
| "pairs_per_s": 304.8, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "RoBERTa-large MNLI", |
| "model": "FacebookAI/roberta-large-mnli", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.5881571031267598, |
| "auc": { |
| "number": 0.4420177777777778, |
| "date": 0.887693066053237, |
| "string": 0.5542929292929293, |
| "table": 0.5558180923253898, |
| "prose": 0.857927614025175 |
| }, |
| "pairs_per_s": 56.8, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "XLM-R-large XNLI", |
| "model": "joeddav/xlm-roberta-large-xnli", |
| "config": "en", |
| "open": true, |
| "n": 5000, |
| "overall_auc": 0.5716570972756371, |
| "auc": { |
| "number": 0.44068444444444443, |
| "date": 0.7327852069061402, |
| "string": 0.5111381673881674, |
| "table": 0.5335998274498526, |
| "prose": 0.8582833704784925 |
| }, |
| "pairs_per_s": 55.7, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| }, |
| { |
| "name": "XLM-R-large XNLI", |
| "model": "joeddav/xlm-roberta-large-xnli", |
| "config": "multi", |
| "open": true, |
| "n": 8421, |
| "overall_auc": 0.8107369827807182, |
| "auc": { |
| "number": 0.7651982222222222, |
| "date": null, |
| "string": null, |
| "table": 0.7150696486611979, |
| "prose": 0.8689075270432692 |
| }, |
| "pairs_per_s": 80.1, |
| "hardware": "NVIDIA RTX A6000", |
| "ctx": 512, |
| "ours": false |
| } |
| ] |
