{
  "repo_id": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2",
  "thresholds": {
    "ppsn_min_score": 0.5,
    "other_min_score": 0.35
  },
  "artifacts": {
    "transformers": true,
    "onnx_dynamic_int8": "onnx/model_quantized.onnx",
    "transformers_inference_script": "inference_mask.py",
    "onnx_inference_script": "inference_mask_onnx.py"
  },
  "exact_qa": {
    "numeric_v2": {
      "v1": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v1",
        "overall_f1": 0.3,
        "passport_f1": 0.5,
        "routing_f1": 0.33333333333333337,
        "phone_f1": 0.0,
        "cpu_examples_per_second": 30.847927604113178
      },
      "v2_rc1": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc1",
        "overall_f1": 0.21052631578947364,
        "passport_f1": 0.0,
        "routing_f1": 0.0,
        "phone_f1": 0.5,
        "cpu_examples_per_second": 31.344761140137095
      },
      "v2_rc2": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2",
        "overall_f1": 0.896551724137931,
        "passport_f1": 0.9090909090909091,
        "routing_f1": 1.0,
        "phone_f1": 0.75,
        "cpu_examples_per_second": 30.321280764124715
      },
      "v2_rc2_onnx_q8": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2#onnx_q8",
        "overall_f1": 0.8387096774193549,
        "passport_f1": 0.7692307692307693,
        "routing_f1": 1.0,
        "phone_f1": 0.75,
        "cpu_examples_per_second": 28.129100872864512
      }
    },
    "gap_feedback_v1": {
      "v1": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v1",
        "overall_f1": 0.26666666666666666,
        "passport_f1": 0.3333333333333333,
        "routing_f1": 0.4,
        "phone_f1": 0.0,
        "cpu_examples_per_second": 30.3635921284123
      },
      "v2_rc1": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc1",
        "overall_f1": 0.13333333333333333,
        "passport_f1": 0.0,
        "routing_f1": 0.0,
        "phone_f1": 0.3333333333333333,
        "cpu_examples_per_second": 31.045050418345404
      },
      "v2_rc2": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2",
        "overall_f1": 0.8695652173913043,
        "passport_f1": 0.888888888888889,
        "routing_f1": 1.0,
        "phone_f1": 0.6666666666666666,
        "cpu_examples_per_second": 32.037976756673686
      },
      "v2_rc2_onnx_q8": {
        "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2#onnx_q8",
        "overall_f1": 0.8333333333333333,
        "passport_f1": 0.8,
        "routing_f1": 1.0,
        "phone_f1": 0.6666666666666666,
        "cpu_examples_per_second": 41.070526382683134
      }
    }
  },
  "broader": {
    "base_openmed": {
      "model": "OpenMed/OpenMed-PII-mLiteClinical-Base-135M-v1",
      "user_ppsn_f1": 0.0,
      "core_overall_f1": 0.5408805031446541,
      "edge_overall_f1": 0.05128205128205128,
      "multilingual_ppsn_f1": 0.0,
      "core_ppsn_f1": 0.0,
      "edge_ppsn_f1": 0.0,
      "edge_phone_f1": 0.14285714285714288,
      "core_cpu_examples_per_second": 33.550658935826185,
      "edge_cpu_examples_per_second": 30.571458913889202,
      "multilingual_cpu_examples_per_second": 116.83828742231812
    },
    "v1": {
      "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v1",
      "user_ppsn_f1": 0.5,
      "core_overall_f1": 0.9530201342281879,
      "edge_overall_f1": 0.5714285714285714,
      "multilingual_ppsn_f1": 0.9940119760479043,
      "core_ppsn_f1": 0.8,
      "edge_ppsn_f1": 0.5,
      "edge_phone_f1": 0.631578947368421,
      "core_cpu_examples_per_second": 27.18654336491933,
      "edge_cpu_examples_per_second": 29.208022233908082,
      "multilingual_cpu_examples_per_second": 127.0333531280346
    },
    "v2_rc1": {
      "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc1",
      "user_ppsn_f1": 1.0,
      "core_overall_f1": 0.9487179487179489,
      "edge_overall_f1": 0.8205128205128205,
      "multilingual_ppsn_f1": 0.7567567567567568,
      "core_ppsn_f1": 0.8571428571428571,
      "edge_ppsn_f1": 0.8571428571428571,
      "edge_phone_f1": 0.8,
      "core_cpu_examples_per_second": 27.619704960037197,
      "edge_cpu_examples_per_second": 28.996839164339264,
      "multilingual_cpu_examples_per_second": 111.33829609559976
    },
    "v2_rc2": {
      "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2",
      "user_ppsn_f1": 1.0,
      "core_overall_f1": 0.9554140127388535,
      "edge_overall_f1": 0.9500000000000001,
      "multilingual_ppsn_f1": 0.8038277511961722,
      "core_ppsn_f1": 0.8571428571428571,
      "edge_ppsn_f1": 0.8571428571428571,
      "edge_phone_f1": 1.0,
      "core_cpu_examples_per_second": 27.479599368654775,
      "edge_cpu_examples_per_second": 30.1890248608861,
      "multilingual_cpu_examples_per_second": 131.09557525150234
    },
    "v2_rc2_onnx_q8": {
      "model": "temsa/OpenMed-mLiteClinical-IrishCorePII-135M-v2-rc2#onnx_q8",
      "user_ppsn_f1": 1.0,
      "core_overall_f1": 0.9615384615384615,
      "edge_overall_f1": 0.9500000000000001,
      "multilingual_ppsn_f1": 0.7887323943661972,
      "core_ppsn_f1": 0.8571428571428571,
      "edge_ppsn_f1": 0.8571428571428571,
      "edge_phone_f1": 1.0,
      "core_cpu_examples_per_second": 119.06137431000792,
      "edge_cpu_examples_per_second": 69.96690039628702,
      "multilingual_cpu_examples_per_second": 86.94930904250197
    }
  }
}