v2 model: production_model/eval_metrics_v2.json

production_model/eval_metrics_v2.json

@@ -0,0 +1,221 @@
+{
+  "metrics": {
+    "Heuristic": {
+      "ndcg@5": 0.11592394487394277,
+      "ndcg@10": 0.16781153427227338,
+      "ndcg@20": 0.2319342951881642,
+      "recall@10": 0.3023517740429505,
+      "mrr": 0.21061923173981048
+    },
+    "V2 LightGBM": {
+      "ndcg@5": 0.8903084772096255,
+      "ndcg@10": 0.9186384673551364,
+      "ndcg@20": 0.9473276046637528,
+      "recall@10": 0.9362628384687208,
+      "mrr": 1.0
+    },
+    "V1 LightGBM": {
+      "ndcg@5": 0.7257547161489314,
+      "ndcg@10": 0.7895323219635818,
+      "ndcg@20": 0.8420620604377875,
+      "recall@10": 0.8752917833800186,
+      "mrr": 0.8982843137254901
+    }
+  },
+  "feature_importance": [
+    {
+      "rank": 1,
+      "feature": "candidate_num_cited_by",
+      "importance": 14985.530790269375
+    },
+    {
+      "rank": 2,
+      "feature": "age_ratio",
+      "importance": 9626.036534249783
+    },
+    {
+      "rank": 3,
+      "feature": "candidate_position",
+      "importance": 8036.926429748535
+    },
+    {
+      "rank": 4,
+      "feature": "cluster_distance_to_medoid",
+      "importance": 2655.753707051277
+    },
+    {
+      "rank": 5,
+      "feature": "position_inverse",
+      "importance": 2578.4697216153145
+    },
+    {
+      "rank": 6,
+      "feature": "ewma_negative_similarity",
+      "importance": 1595.2763053178787
+    },
+    {
+      "rank": 7,
+      "feature": "ewma_shortterm_similarity",
+      "importance": 1498.1983343958855
+    },
+    {
+      "rank": 8,
+      "feature": "qdrant_cosine_score",
+      "importance": 418.87039536237717
+    },
+    {
+      "rank": 9,
+      "feature": "citation_count_ratio",
+      "importance": 360.1048934459686
+    },
+    {
+      "rank": 10,
+      "feature": "cosine_x_citations",
+      "importance": 288.0438839793205
+    },
+    {
+      "rank": 11,
+      "feature": "query_age_days",
+      "importance": 287.9089082479477
+    },
+    {
+      "rank": 12,
+      "feature": "candidate_influential_citations",
+      "importance": 250.03656780719757
+    },
+    {
+      "rank": 13,
+      "feature": "user_days_since_last_save",
+      "importance": 244.79518991708755
+    },
+    {
+      "rank": 14,
+      "feature": "candidate_citation_count",
+      "importance": 244.56051486730576
+    },
+    {
+      "rank": 15,
+      "feature": "query_citation_count",
+      "importance": 243.8800145983696
+    },
+    {
+      "rank": 16,
+      "feature": "ewma_longterm_similarity",
+      "importance": 227.91575199365616
+    },
+    {
+      "rank": 17,
+      "feature": "query_num_references",
+      "importance": 225.5082014799118
+    },
+    {
+      "rank": 18,
+      "feature": "candidate_citations_per_year",
+      "importance": 214.10649985074997
+    },
+    {
+      "rank": 19,
+      "feature": "candidate_age_days",
+      "importance": 184.8911971449852
+    },
+    {
+      "rank": 20,
+      "feature": "cosine_x_cocitation",
+      "importance": 152.6348159313202
+    },
+    {
+      "rank": 21,
+      "feature": "onboarding_category_match",
+      "importance": 152.52679014205933
+    },
+    {
+      "rank": 22,
+      "feature": "citations_x_recency",
+      "importance": 140.38160228729248
+    },
+    {
+      "rank": 23,
+      "feature": "category_x_recency",
+      "importance": 129.21052467823029
+    },
+    {
+      "rank": 24,
+      "feature": "user_session_save_count",
+      "importance": 109.97411346435547
+    },
+    {
+      "rank": 25,
+      "feature": "cluster_importance",
+      "importance": 102.49212235212326
+    },
+    {
+      "rank": 26,
+      "feature": "year_diff",
+      "importance": 81.32582759857178
+    },
+    {
+      "rank": 27,
+      "feature": "candidate_recency_score",
+      "importance": 80.25869399309158
+    },
+    {
+      "rank": 28,
+      "feature": "cosine_x_recency",
+      "importance": 61.490856766700745
+    },
+    {
+      "rank": 29,
+      "feature": "user_total_saves",
+      "importance": 60.1797690987587
+    },
+    {
+      "rank": 30,
+      "feature": "query_log_citations",
+      "importance": 48.18304318189621
+    },
+    {
+      "rank": 31,
+      "feature": "candidate_log_citations",
+      "importance": 44.986644983291626
+    },
+    {
+      "rank": 32,
+      "feature": "co_citation_count",
+      "importance": 17.18885040283203
+    },
+    {
+      "rank": 33,
+      "feature": "is_suppressed_category",
+      "importance": 16.725749731063843
+    },
+    {
+      "rank": 34,
+      "feature": "candidate_is_newer",
+      "importance": 11.455129861831665
+    },
+    {
+      "rank": 35,
+      "feature": "same_primary_category",
+      "importance": 0.0
+    },
+    {
+      "rank": 36,
+      "feature": "shared_author_count",
+      "importance": 0.0
+    },
+    {
+      "rank": 37,
+      "feature": "user_total_dismissals",
+      "importance": 0.0
+    }
+  ],
+  "training": {
+    "best_iteration": 52,
+    "train_rows": 26494,
+    "eval_rows": 3389,
+    "train_queries": 532,
+    "eval_queries": 68,
+    "training_time_seconds": 0.7
+  },
+  "latency_ms": 0.128
+}