# Copy of wholepipeline print("--- 1. Installing All Libraries ---") print("✅ Libraries installed.") print("\n--- 2. Cloning IndicLID Repository ---") # Using your proven method of changing directories print("✅ Repository cloned.") # Navigate into the correct directory structure print("\n--- 3. Downloading and Unzipping IndicLID Models ---") print("✅ Download commands executed. Unzipping now...") print("✅ Unzip commands executed.") import os import sys import torch print("--- Applying your original add_safe_globals fix... ---") if "/content/IndicLID/Inference" not in sys.path: sys.path.append("/content/IndicLID/Inference") from transformers.models.bert.modeling_bert import ( BertModel, BertPreTrainedModel, BertForSequenceClassification, BertEmbeddings, BertEncoder, BertPooler, BertLayer, BertAttention, BertSelfAttention, BertSelfOutput, BertIntermediate, BertOutput ) from transformers.models.bert.configuration_bert import BertConfig import torch.nn as nn from torch.nn.modules.sparse import Embedding from torch.nn.modules.container import ModuleList from torch.nn.modules.linear import Linear from torch.nn.modules.normalization import LayerNorm from torch.nn.modules.dropout import Dropout torch.serialization.add_safe_globals([ BertModel, BertPreTrainedModel, BertForSequenceClassification, BertEmbeddings, BertEncoder, BertPooler, BertLayer, BertAttention, BertSelfAttention, BertSelfOutput, BertIntermediate, BertOutput, BertConfig, Embedding, ModuleList, Linear, LayerNorm, Dropout, ]) print("✅ Comprehensive safe globals added successfully.") from transformers import AutoTokenizer, AutoModelForSeq2SeqLM from IndicTransToolkit.processor import IndicProcessor from ai4bharat.IndicLID import IndicLID print("--- Loading all models into memory... ---") device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {device}") lid = IndicLID(input_threshold=0.5, roman_lid_threshold=0.6) print("✅ IndicLID model loaded successfully.") MODEL_ID = "ai4bharat/indictrans2-indic-en-1B" tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True) model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_ID, trust_remote_code=True).to(device) ip = IndicProcessor(inference=True) import gradio as gr import pandas as pd from indic_transliteration import sanscript from indic_transliteration.sanscript import transliterate import requests from typing import List, Dict, Union, Optional # YOUR EXACT IndicXlit API Code (no changes) class IndicXlitClient: """Simple client for IndicXlit Transliteration API""" def __init__(self, api_url: str = "https://awake-blowfish-liberal.ngrok-free.app"): self.api_url = api_url.rstrip('/') self.session = requests.Session() self.session.headers.update({ 'Content-Type': 'application/json', 'Accept': 'application/json' }) def health_check(self) -> dict: try: response = self.session.get(f"{self.api_url}/health") response.raise_for_status() return response.json() except Exception as e: return {"error": str(e), "status": "unhealthy"} def get_supported_languages(self) -> List[str]: try: response = self.session.get(f"{self.api_url}/languages") response.raise_for_status() data = response.json() return data.get("supported_languages", []) except Exception as e: print(f"Error getting languages: {e}") return [] def english_to_indic(self, text: str, target_languages: Union[str, List[str]], beam_width: int = 4) -> Dict[str, str]: try: payload = { "text": text, "target_languages": target_languages, "beam_width": beam_width } response = self.session.post( f"{self.api_url}/transliterate/en-to-indic", json=payload ) response.raise_for_status() result = response.json() if result.get("success"): return result.get("results", {}) else: print(f"API Error: {result}") return {} except Exception as e: print(f"Error transliterating: {e}") return {} def indic_to_english(self, text: str, source_language: str, beam_width: int = 4) -> str: try: payload = { "text": text, "source_language": source_language, "beam_width": beam_width } response = self.session.post( f"{self.api_url}/transliterate/indic-to-en", json=payload ) response.raise_for_status() result = response.json() if result.get("success"): return result.get("result", "") else: print(f"API Error: {result}") return "" except Exception as e: print(f"Error transliterating: {e}") return "" # Create a global client instance for convenience client = IndicXlitClient() # YOUR EXACT convenience functions def transliterate_from_indic(text: str, source_language: str) -> str: return client.indic_to_english(text, source_language) def transliterate_from_en(text: str, target_languages: Union[str, List[str]]) -> Dict[str, str]: return client.english_to_indic(text, target_languages) def get_supported_languages() -> List[str]: return client.get_supported_languages() def check_api_health() -> bool: health = client.health_check() return health.get("status") == "healthy" # Test API connectivity print("🔄 Testing IndicXlit API connectivity...") if check_api_health(): print("✅ IndicXlit API is healthy and ready!") supported_langs = get_supported_languages() print(f"📋 Supported languages: {supported_langs}") else: print("⚠️ IndicXlit API is not available - will use fallback methods") print("✅ IndicXlit API integration completed!") # Language mapping with only IndicXlit support (aksharamukha removed) # --- THE FINAL, VERIFIED, AND ACCURATE MASTER MAPPING DICTIONARY --- # This dictionary should replace all other mappings in our code. # It correctly handles all model-to-model code translations and exceptions. MASTER_LANGUAGE_MAPPING = { # This dictionary maps the output of IndicLID to the required codes for IndicXlit and IndicTrans2. # --- NATIVE SCRIPT INPUTS (Detected by IndicLID) --- "asm_Beng": {"name": "Assamese", "indicxlit_code": "as", "indictrans_code": "asm_Beng", "script": "bengali"}, "ben_Beng": {"name": "Bengali", "indicxlit_code": "bn", "indictrans_code": "ben_Beng", "script": "bengali"}, "brx_Deva": {"name": "Bodo", "indicxlit_code": "brx", "indictrans_code": "brx_Deva", "script": "devanagari"}, "doi_Deva": {"name": "Dogri", "indicxlit_code": None, "indictrans_code": "doi_Deva", "script": "devanagari"}, # VERIFIED: IndicXlit does not support Dogri. "eng_Latn": {"name": "English", "indicxlit_code": None, "indictrans_code": "eng_Latn", "script": None}, "guj_Gujr": {"name": "Gujarati", "indicxlit_code": "gu", "indictrans_code": "guj_Gujr", "script": "gujarati"}, "hin_Deva": {"name": "Hindi", "indicxlit_code": "hi", "indictrans_code": "hin_Deva", "script": "devanagari"}, "kan_Knda": {"name": "Kannada", "indicxlit_code": "kn", "indictrans_code": "kan_Knda", "script": "kannada"}, "kas_Arab": {"name": "Kashmiri", "indicxlit_code": "ks", "indictrans_code": "kas_Arab", "script": "urdu"}, "kas_Deva": {"name": "Kashmiri", "indicxlit_code": "ks", "indictrans_code": "kas_Deva", "script": "devanagari"}, "kok_Deva": {"name": "Konkani", "indicxlit_code": "gom", "indictrans_code": "gom_Deva", "script": "devanagari"}, # VERIFIED: IndicLID uses 'kok', IndicXlit/Trans2 use 'gom'. "mai_Deva": {"name": "Maithili", "indicxlit_code": "mai", "indictrans_code": "mai_Deva", "script": "devanagari"}, "mal_Mlym": {"name": "Malayalam","indicxlit_code": "ml", "indictrans_code": "mal_Mlym", "script": "malayalam"}, "mar_Deva": {"name": "Marathi", "indicxlit_code": "mr", "indictrans_code": "mar_Deva", "script": "devanagari"}, "mni_Beng": {"name": "Manipuri", "indicxlit_code": "mni", "indictrans_code": "mni_Beng", "script": "bengali"}, "mni_Mtei": {"name": "Manipuri", "indicxlit_code": "mni", "indictrans_code": "mni_Mtei", "script": None}, "nep_Deva": {"name": "Nepali", "indicxlit_code": "ne", "indictrans_code": "npi_Deva", "script": "devanagari"}, # VERIFIED: IndicTrans2 uses 'npi'. "ori_Orya": {"name": "Odia", "indicxlit_code": "or", "indictrans_code": "ory_Orya", "script": "oriya"}, # VERIFIED: IndicTrans2 uses 'ory'. "pan_Guru": {"name": "Punjabi", "indicxlit_code": "pa", "indictrans_code": "pan_Guru", "script": "gurmukhi"}, "san_Deva": {"name": "Sanskrit", "indicxlit_code": "sa", "indictrans_code": "san_Deva", "script": "devanagari"}, "sat_Olch": {"name": "Santali", "indicxlit_code": None, "indictrans_code": "sat_Olck", "script": None}, # VERIFIED: IndicXlit does not support Santali. "snd_Arab": {"name": "Sindhi", "indicxlit_code": "sd", "indictrans_code": "snd_Arab", "script": "urdu"}, "snd_Deva": {"name": "Sindhi", "indicxlit_code": "sd", "indictrans_code": "snd_Deva", "script": "devanagari"}, "tam_Taml": {"name": "Tamil", "indicxlit_code": "ta", "indictrans_code": "tam_Taml", "script": "tamil"}, "tel_Telu": {"name": "Telugu", "indicxlit_code": "te", "indictrans_code": "tel_Telu", "script": "telugu"}, "urd_Arab": {"name": "Urdu", "indicxlit_code": "ur", "indictrans_code": "urd_Arab", "script": "urdu"}, # --- ROMANIZED SCRIPT INPUTS (Detected by IndicLID) --- "asm_Latn": {"name": "Assamese", "indicxlit_code": "as", "indictrans_code": "asm_Beng", "script": "bengali"}, "ben_Latn": {"name": "Bengali", "indicxlit_code": "bn", "indictrans_code": "ben_Beng", "script": "bengali"}, "brx_Latn": {"name": "Bodo", "indicxlit_code": "brx", "indictrans_code": "brx_Deva", "script": "devanagari"}, "guj_Latn": {"name": "Gujarati", "indicxlit_code": "gu", "indictrans_code": "guj_Gujr", "script": "gujarati"}, "hin_Latn": {"name": "Hindi", "indicxlit_code": "hi", "indictrans_code": "hin_Deva", "script": "devanagari"}, "kan_Latn": {"name": "Kannada", "indicxlit_code": "kn", "indictrans_code": "kan_Knda", "script": "kannada"}, "kas_Latn": {"name": "Kashmiri", "indicxlit_code": "ks", "indictrans_code": "kas_Deva", "script": "devanagari"}, "kok_Latn": {"name": "Konkani", "indicxlit_code": "gom", "indictrans_code": "gom_Deva", "script": "devanagari"}, "mai_Latn": {"name": "Maithili", "indicxlit_code": "mai", "indictrans_code": "mai_Deva", "script": "devanagari"}, "mal_Latn": {"name": "Malayalam","indicxlit_code": "ml", "indictrans_code": "mal_Mlym", "script": "malayalam"}, "mar_Latn": {"name": "Marathi", "indicxlit_code": "mr", "indictrans_code": "mar_Deva", "script": "devanagari"}, "mni_Latn": {"name": "Manipuri", "indicxlit_code": "mni", "indictrans_code": "mni_Beng", "script": "bengali"}, "nep_Latn": {"name": "Nepali", "indicxlit_code": "ne", "indictrans_code": "npi_Deva", "script": "devanagari"}, "ori_Latn": {"name": "Odia", "indicxlit_code": "or", "indictrans_code": "ory_Orya", "script": "oriya"}, "pan_Latn": {"name": "Punjabi", "indicxlit_code": "pa", "indictrans_code": "pan_Guru", "script": "gurmukhi"}, "san_Latn": {"name": "Sanskrit", "indicxlit_code": "sa", "indictrans_code": "san_Deva", "script": "devanagari"}, "snd_Latn": {"name": "Sindhi", "indicxlit_code": "sd", "indictrans_code": "snd_Arab", "script": "urdu"}, "tam_Latn": {"name": "Tamil", "indicxlit_code": "ta", "indictrans_code": "tam_Taml", "script": "tamil"}, "tel_Latn": {"name": "Telugu", "indicxlit_code": "te", "indictrans_code": "tel_Telu", "script": "telugu"}, "urd_Latn": {"name": "Urdu", "indicxlit_code": "ur", "indictrans_code": "urd_Arab", "script": "urdu"}, } # KEEP our exact enhanced_transliterate_robust function (no changes) def enhanced_transliterate_robust(text, target_script): """ENHANCED transliteration function - same logic, better replacements""" try: cleaned_text = text.lower().strip() # ENHANCED replacements for better accuracy replacements = { 'kh': 'kh', 'ch': 'ch', 'th': 'th', 'ph': 'ph', 'bh': 'bh', 'dh': 'dh', 'gh': 'gh', 'jh': 'jh', 'sh': 'sh', 'zh': 'zh', 'ng': 'ng', # Added more consonants 'aa': 'A', 'ee': 'I', 'oo': 'U', 'ou': 'au', 'ai': 'ai', 'ei': 'ai' # Better vowel handling } for old, new in replacements.items(): cleaned_text = cleaned_text.replace(old, new) result = transliterate(cleaned_text, sanscript.ITRANS, target_script) return result if result else text except Exception as e: return text # IndicXlit function using OUR convenience functions def indicxlit_transliterate(text, target_lang_code): """IndicXlit API-based transliteration using YOUR convenience functions""" try: # Use OUR convenience function results = transliterate_from_en(text, target_lang_code) if results and target_lang_code in results: return results[target_lang_code], 1.0 # High confidence for API else: return text, 0.0 except Exception as e: print(f"IndicXlit error: {e}") return text, 0.0 print(" Enhanced pipeline functions loaded successfully.") # --- CELL 1: HELPER FUNCTIONS (ADD HERE) --- import difflib from collections import defaultdict, Counter def parse_confidence_safely(conf_value): """Safely converts and clamps confidence scores to the [0.0, 1.0] range.""" try: return min(max(float(conf_value), 0.0), 1.0) except (ValueError, TypeError): return 0.0 def calculate_similarity_fixed(text1, text2): """Calculate similarity between two texts using difflib""" def preprocess(text): if not isinstance(text, str): text = str(text) return " ".join(text.strip().lower().split()) t1_processed = preprocess(text1) t2_processed = preprocess(text2) if not t1_processed and not t2_processed: return 1.0 if not t1_processed or not t2_processed: return 0.0 return difflib.SequenceMatcher(None, t1_processed, t2_processed).ratio() print("✅ Helper functions loaded successfully.") # --- CELL 2: STATISTICS TRACKING SYSTEM (ADD HERE) --- # Global statistics tracker STATS_TRACKER = { 'roman_samples': defaultdict(lambda: { 'total': 0, 'lid_correct': 0, 'xlit_correct': 0, 'trans_correct': 0, 'misdetections': Counter(), 'xlit_similarities': [], 'trans_similarities': [] }), 'native_samples': defaultdict(lambda: { 'total': 0, 'lid_correct': 0, 'trans_correct': 0, 'misdetections': Counter(), 'trans_similarities': [] }), 'overall_stats': {'roman_total': 0, 'native_total': 0} } def update_stats(detected_lang, script_type, is_lid_correct, xlit_similarity=0.0, trans_similarity=0.0, ground_truth_lang=None): """Update global statistics tracker""" lang_key = ground_truth_lang or detected_lang if script_type == "Romanized": stats = STATS_TRACKER['roman_samples'][lang_key] stats['total'] += 1 if is_lid_correct: stats['lid_correct'] += 1 if xlit_similarity > 0.7: stats['xlit_correct'] += 1 if trans_similarity > 0.6: stats['trans_correct'] += 1 stats['xlit_similarities'].append(xlit_similarity) stats['trans_similarities'].append(trans_similarity) STATS_TRACKER['overall_stats']['roman_total'] += 1 else: stats = STATS_TRACKER['native_samples'][lang_key] stats['total'] += 1 if is_lid_correct: stats['lid_correct'] += 1 if trans_similarity > 0.6: stats['trans_correct'] += 1 stats['trans_similarities'].append(trans_similarity) STATS_TRACKER['overall_stats']['native_total'] += 1 def get_stats_summary(): """Generate detailed statistics summary""" summary_lines = [] # Roman text statistics summary_lines.append("## 📊 Roman Text Analysis") for lang, stats in STATS_TRACKER['roman_samples'].items(): if stats['total'] > 0: lid_acc = (stats['lid_correct'] / stats['total']) * 100 xlit_acc = (stats['xlit_correct'] / stats['total']) * 100 trans_acc = (stats['trans_correct'] / stats['total']) * 100 avg_xlit_sim = sum(stats['xlit_similarities']) / len(stats['xlit_similarities']) * 100 if stats['xlit_similarities'] else 0 avg_trans_sim = sum(stats['trans_similarities']) / len(stats['trans_similarities']) * 100 if stats['trans_similarities'] else 0 summary_lines.append(f"**{lang}** (n={stats['total']})") summary_lines.append(f" - LID: {lid_acc:.1f}% | IndicXlit: {xlit_acc:.1f}% (avg sim: {avg_xlit_sim:.1f}%) | IndicTrans2: {trans_acc:.1f}% (avg sim: {avg_trans_sim:.1f}%)") # Native text statistics summary_lines.append("\n## 📊 Native Text Analysis") for lang, stats in STATS_TRACKER['native_samples'].items(): if stats['total'] > 0: lid_acc = (stats['lid_correct'] / stats['total']) * 100 trans_acc = (stats['trans_correct'] / stats['total']) * 100 avg_trans_sim = sum(stats['trans_similarities']) / len(stats['trans_similarities']) * 100 if stats['trans_similarities'] else 0 summary_lines.append(f"**{lang}** (n={stats['total']})") summary_lines.append(f" - LID: {lid_acc:.1f}% | IndicTrans2: {trans_acc:.1f}% (avg sim: {avg_trans_sim:.1f}%)") # Overall statistics roman_total = STATS_TRACKER['overall_stats']['roman_total'] native_total = STATS_TRACKER['overall_stats']['native_total'] summary_lines.append(f"\n## 📈 Overall Summary") summary_lines.append(f"**Total samples:** Roman: {roman_total}, Native: {native_total}") return "\n".join(summary_lines) def reset_stats(): """Reset all statistics""" global STATS_TRACKER STATS_TRACKER = { 'roman_samples': defaultdict(lambda: { 'total': 0, 'lid_correct': 0, 'xlit_correct': 0, 'trans_correct': 0, 'misdetections': Counter(), 'xlit_similarities': [], 'trans_similarities': [] }), 'native_samples': defaultdict(lambda: { 'total': 0, 'lid_correct': 0, 'trans_correct': 0, 'misdetections': Counter(), 'trans_similarities': [] }), 'overall_stats': {'roman_total': 0, 'native_total': 0} } return "✅ Statistics reset successfully!" print("✅ Statistics tracking system loaded successfully.") # --- CELL 3: ENHANCED PIPELINE FUNCTION --- def detect_and_translate_with_parallel_comparison(text, ground_truth_lang=None, reference_translation=None): """ Enhanced pipeline with parallel transliteration comparison and statistics tracking """ try: # 1. Language Detection with safe confidence parsing preds = lid.batch_predict([text], 1) item = preds[0] if isinstance(item, dict): detected_lang = item.get("lang", item.get("pred_lang", "")) score = parse_confidence_safely(item.get("score", 0.0)) else: _, detected_lang, raw_score, _ = item score = parse_confidence_safely(raw_score) is_romanized = detected_lang.endswith("_Latn") script_type = "Romanized" if is_romanized else "Native Script" # Check if LID is correct (if ground truth provided) is_lid_correct = True if ground_truth_lang: expected_prefix = ground_truth_lang.lower()[:3] is_lid_correct = expected_prefix in detected_lang.lower() # 2. Check mapping support if detected_lang not in MASTER_LANGUAGE_MAPPING: return { 'detected_lang': detected_lang, 'script_type': script_type, 'lid_confidence': f"{score:.3f}", 'method': "Unsupported", 'translation': f"Language '{detected_lang}' not supported", 'rule_based_result': "", 'rule_based_score': "0.0", 'indicxlit_result': "", 'indicxlit_score': "0.0", 'final_native': "", 'rule_vs_native_sim': "0.0", 'xlit_vs_native_sim': "0.0", 'trans_similarity': "0.0" } lang_info = MASTER_LANGUAGE_MAPPING[detected_lang] src_code = lang_info["indictrans_code"] xlit_code = lang_info["indicxlit_code"] script_fallback = lang_info["script"] # Initialize results rule_based_result = "" rule_based_score = 0.0 indicxlit_result = "" indicxlit_score = 0.0 native_text = text method = "IndicTrans2 Only" # 3. PARALLEL TRANSLITERATION COMPARISON (for Roman input) if is_romanized: # Method 1: Rule-based transliteration if script_fallback: rule_based_result = enhanced_transliterate_robust(text, script_fallback) rule_based_score = 0.8 # Fixed confidence for rule-based else: rule_based_result = text rule_based_score = 0.0 # Method 2: IndicXlit API if xlit_code: indicxlit_result, indicxlit_conf = indicxlit_transliterate(text, xlit_code) indicxlit_score = indicxlit_conf else: indicxlit_result = text indicxlit_score = 0.0 # Decision making for final translation if indicxlit_score > 0.5: native_text = indicxlit_result method = f"IndicXlit API (conf: {indicxlit_score:.2f}) + IndicTrans2" else: native_text = rule_based_result method = "Rule-Based Transliteration + IndicTrans2" # 4. Final Translation pre = ip.preprocess_batch([native_text], src_lang=src_code, tgt_lang="eng_Latn") inputs = tokenizer(pre, return_tensors="pt", padding=True).to(device) with torch.no_grad(): out = model.generate(**inputs, num_beams=5, max_length=256, early_stopping=True) dec = tokenizer.batch_decode(out, skip_special_tokens=True) post = ip.postprocess_batch(dec, lang=src_code) translation = post[0] # 5. Calculate similarities (if reference provided) rule_vs_native_sim = 0.0 xlit_vs_native_sim = 0.0 trans_similarity = 0.0 if is_romanized and ground_truth_lang: # Compare transliteration outputs with reference native text if rule_based_result: rule_vs_native_sim = calculate_similarity_fixed(rule_based_result, native_text) if indicxlit_result: xlit_vs_native_sim = calculate_similarity_fixed(indicxlit_result, native_text) if reference_translation: trans_similarity = calculate_similarity_fixed(translation, reference_translation) # 6. Update statistics update_stats( detected_lang, script_type, is_lid_correct, max(rule_vs_native_sim, xlit_vs_native_sim), trans_similarity, ground_truth_lang ) return { 'detected_lang': detected_lang, 'script_type': script_type, 'lid_confidence': f"{score:.3f}", 'method': method, 'translation': translation, 'rule_based_result': rule_based_result, 'rule_based_score': f"{rule_based_score:.3f}", 'indicxlit_result': indicxlit_result, 'indicxlit_score': f"{indicxlit_score:.3f}", 'final_native': native_text, 'rule_vs_native_sim': f"{rule_vs_native_sim:.3f}", 'xlit_vs_native_sim': f"{xlit_vs_native_sim:.3f}", 'trans_similarity': f"{trans_similarity:.3f}" } except Exception as e: import traceback error_msg = f"Pipeline error: {str(e)}" print(f"FATAL ERROR: {e}\n{traceback.format_exc()}") return { 'detected_lang': "Error", 'script_type': "Error", 'lid_confidence': "0.0", 'method': "Error", 'translation': error_msg, 'rule_based_result': "", 'rule_based_score': "0.0", 'indicxlit_result': "", 'indicxlit_score': "0.0", 'final_native': "", 'rule_vs_native_sim': "0.0", 'xlit_vs_native_sim': "0.0", 'trans_similarity': "0.0" } print("✅ Enhanced pipeline function loaded successfully.") # --- CELL 4: GRADIO INTERFACE (COMPLETE CODE) --- import gradio as gr def main_interface(input_text, ground_truth_lang="", reference_translation=""): """Main interface function leveraging enhanced pipeline with parallel transliteration comparison and stats tracking.""" if not input_text or not input_text.strip(): return ["Please enter some text"] + [""] * 12 # Call the enhanced pipeline function from Cell 3 result = detect_and_translate_with_parallel_comparison( input_text, ground_truth_lang if ground_truth_lang else None, reference_translation if reference_translation else None ) # Extract results for UI (13 outputs total) return [ result['detected_lang'], # 1. Detected Language result['script_type'], # 2. Script Type result['lid_confidence'], # 3. LID Confidence result['method'], # 4. Method Used result['translation'], # 5. English Translation result['rule_based_result'], # 6. Rule-Based Result result['rule_based_score'], # 7. Rule-Based Score result['indicxlit_result'], # 8. IndicXlit Result result['indicxlit_score'], # 9. IndicXlit Score result['final_native'], # 10. Final Native Text result['rule_vs_native_sim'], # 11. Rule vs Native Similarity result['xlit_vs_native_sim'], # 12. IndicXlit vs Native Similarity result['trans_similarity'] # 13. Translation Similarity ] def similarity_test(native_text, english_reference): """Similarity testing interface for Tab 3""" if not native_text or not english_reference: return "Please provide both inputs", "0.0" # Use the enhanced pipeline to translate result = detect_and_translate_with_parallel_comparison(native_text) translated_english = result['translation'] similarity = calculate_similarity_fixed(translated_english, english_reference) comparison = f"""**Native Input:** {native_text} **Model Output:** {translated_english} **Reference:** {english_reference} **Method:** {result['method']} **Detected Language:** {result['detected_lang']}""" return comparison, f"{similarity:.3f}" # Main Gradio App Layout with gr.Blocks(title="🇮🇳 Comprehensive Indian Language AI Testing Platform", theme=gr.themes.Soft()) as app: # Header gr.Markdown("# 🇮🇳 Comprehensive Indian Language AI Testing Platform") gr.Markdown("Advanced testing and analysis platform for Indian language detection, transliteration, and translation with parallel comparison and real-time statistics.") with gr.Tabs(): # ================================ # TAB 1: MAIN TRANSLATION & ANALYSIS # ================================ with gr.TabItem("🔍 Translation & Analysis"): gr.Markdown("### Input Section") with gr.Row(): with gr.Column(scale=2): input_text = gr.Textbox( label="📝 Input Text", placeholder="Enter text in any Indian language (Roman or Native script)", lines=4 ) with gr.Column(scale=1): ground_truth = gr.Textbox( label="🎯 Ground Truth Language (Optional)", placeholder="e.g., Hindi, Bengali, Tamil" ) reference_trans = gr.Textbox( label="📖 Reference Translation (Optional)", placeholder="Expected English translation" ) analyze_btn = gr.Button("🔍 Analyze & Translate", variant="primary", size="lg") gr.Markdown("---") gr.Markdown("### Detection & Translation Results") # Main Results Section with gr.Row(): detected_lang = gr.Textbox(label="🎯 Detected Language", interactive=False) script_type = gr.Textbox(label="📝 Script Type", interactive=False) lid_confidence = gr.Textbox(label="🎯 LID Confidence", interactive=False) method = gr.Textbox(label="⚙️ Method Used", interactive=False) translation = gr.Textbox(label="🌍 English Translation", interactive=False, lines=3) gr.Markdown("---") gr.Markdown("### 🔄 Parallel Transliteration Comparison") gr.Markdown("*Only shown for Romanized input - compares Rule-based vs IndicXlit API methods*") with gr.Row(): with gr.Column(): gr.Markdown("#### Method 1: Rule-Based") rule_result = gr.Textbox(label="Rule-Based Result", interactive=False) rule_score = gr.Textbox(label="Rule-Based Score", interactive=False) rule_sim = gr.Textbox(label="Rule vs Final Similarity", interactive=False) with gr.Column(): gr.Markdown("#### Method 2: IndicXlit API") xlit_result = gr.Textbox(label="IndicXlit Result", interactive=False) xlit_score = gr.Textbox(label="IndicXlit Confidence", interactive=False) xlit_sim = gr.Textbox(label="IndicXlit vs Final Similarity", interactive=False) with gr.Row(): final_native = gr.Textbox(label="🏆 Final Native Text Used for Translation", interactive=False, lines=2) trans_sim = gr.Textbox(label="🎯 Translation Similarity (vs Reference)", interactive=False) # Examples Section gr.Markdown("---") gr.Examples( examples=[ ["tum kaise ho", "Hindi", "How are you"], ["neenga epdi irukeenga", "Tamil", "How are you"], ["ami bhalo achi", "Bengali", "I am fine"], ["नमस्ते कैसे हैं आप", "Hindi", "Hello how are you"], ["আমি ভাল আছি", "Bengali", "I am fine"], ["dogri mein kaise likhte hain", "Dogri", "How to write in Dogri"] ], inputs=[input_text, ground_truth, reference_trans], label="📚 Try these examples (Input Text, Ground Truth, Reference Translation)" ) # ================================ # TAB 2: STATISTICS DASHBOARD # ================================ with gr.TabItem("📊 Statistics Dashboard"): gr.Markdown("### Real-time Performance Analytics") gr.Markdown("Track accuracy metrics across all models for both Roman and Native script inputs.") with gr.Row(): refresh_btn = gr.Button("🔄 Refresh Statistics", variant="secondary", size="lg") reset_btn = gr.Button("🗑️ Reset All Statistics", variant="primary", size="lg") stats_output = gr.Markdown( value="*No statistics available yet. Use the Translation & Analysis tab to generate data.*", label="Performance Statistics" ) reset_status = gr.Textbox(label="📋 Status", interactive=False) # ================================ # TAB 3: SIMILARITY TESTING # ================================ with gr.TabItem("🔍 Similarity Testing"): gr.Markdown("### Direct Translation Similarity Comparison") gr.Markdown("Compare model translation output directly with your reference translation using similarity scoring.") with gr.Row(): with gr.Column(): native_input = gr.Textbox( label="🔤 Native Script Text", placeholder="Enter text in native Indian script", lines=4 ) english_ref = gr.Textbox( label="📖 Reference English Translation", placeholder="Enter the expected English translation", lines=4 ) with gr.Column(): compare_btn = gr.Button("🔍 Compare Similarity", variant="primary", size="lg") gr.Markdown("### How it works:") gr.Markdown("1. Enter native script text") gr.Markdown("2. Enter expected English translation") gr.Markdown("3. Click Compare to see similarity score") gr.Markdown("4. Higher scores (closer to 1.0) indicate better translation quality") comparison_result = gr.Textbox( label="📊 Detailed Comparison Results", interactive=False, lines=6 ) similarity_score = gr.Textbox(label="🎯 Similarity Score (0.0 - 1.0)", interactive=False) # Examples for similarity testing gr.Examples( examples=[ ["नमस्ते", "Hello"], ["धन्यवाद", "Thank you"], ["আপনি কেমন আছেন?", "How are you?"], ["நீங்கள் எப்படி இருக்கிறீர்கள்?", "How are you?"] ], inputs=[native_input, english_ref], label="🔧 Try these similarity tests" ) # ================================ # CONNECT ALL FUNCTIONS TO BUTTONS # ================================ # Main analysis button analyze_btn.click( fn=main_interface, inputs=[input_text, ground_truth, reference_trans], outputs=[ detected_lang, script_type, lid_confidence, method, translation, rule_result, rule_score, xlit_result, xlit_score, final_native, rule_sim, xlit_sim, trans_sim ] ) # Statistics buttons refresh_btn.click( fn=get_stats_summary, outputs=[stats_output] ) reset_btn.click( fn=reset_stats, outputs=[reset_status] ) # Similarity testing button compare_btn.click( fn=similarity_test, inputs=[native_input, english_ref], outputs=[comparison_result, similarity_score] ) # Launch the app app.launch(share=True, debug=False) print("🎉 Comprehensive Indian Language AI Testing Platform launched!") print("📋 Features Available:") print(" ✅ Complete language detection and translation pipeline") print(" ✅ Parallel transliteration comparison (Rule-based vs IndicXlit)") print(" ✅ Real-time accuracy and performance statistics") print(" ✅ Advanced similarity testing with ground truth comparison") print(" ✅ Support for all 22 scheduled Indian languages") print(" ✅ Comprehensive error handling and robust processing") #old working functions def detect_and_translate_single_corrected(text): """ Corrected and robust pipeline function using the single MASTER_LANGUAGE_MAPPING. This version correctly handles all exceptions, unsupported languages, and ensures all confidence scores are valid. """ try: # 1. Language Detection (with safe confidence score parsing) preds = lid.batch_predict([text], 1) item = preds[0] if isinstance(item, dict): detected_lang = item.get("lang", item.get("pred_lang", "")) # Use the safe parsing function immediately score = parse_confidence_safely(item.get("score", 0.0)) else: _, detected_lang, raw_score, _ = item # Use the safe parsing function here as well score = parse_confidence_safely(raw_score) is_romanized = detected_lang.endswith("_Latn") script_type = "Romanized" if is_romanized else "Native Script" # 2. Unified Mapping Lookup (uses the complete master mapping) if detected_lang not in MASTER_LANGUAGE_MAPPING: return (detected_lang, script_type, f"{score:.3f}", "Unsupported", f"Language '{detected_lang}' is not defined in the master mapping.", "", "", "0.0", "") lang_info = MASTER_LANGUAGE_MAPPING[detected_lang] src_code_for_indictrans = lang_info["indictrans_code"] xlit_code_for_api = lang_info["indicxlit_code"] script_for_fallback = lang_info["script"] # 3. Transliteration Logic (robustly handles all cases) native_text = text method = "IndicTrans2 Only" # Default method for native inputs indic_result = "" indicxlit_result = "" indicxlit_conf = 0.0 if is_romanized: # Always run the rule-based method first as a reliable fallback if script_for_fallback: indic_result = enhanced_transliterate_robust(text, script_for_fallback) else: indic_result = text # If no rule-based method exists, the fallback is the original text # Attempt AI-based transliteration only if the language is supported if xlit_code_for_api: indicxlit_result, indicxlit_conf = indicxlit_transliterate(text, xlit_code_for_api) else: # This correctly handles Dogri, Santali, etc. indicxlit_result, indicxlit_conf = text, 0.0 print(f"INFO: IndicXlit not supported for {lang_info['name']}. Skipping API call.") # Decision Making: Prioritize the AI model when available if indicxlit_conf > 0.5: native_text = indicxlit_result method = "IndicXlit API + IndicTrans2" else: native_text = indic_result # Otherwise, use the rule-based result method = "Rule-Based Transliteration + IndicTrans2" # 4. Final Translation (receives verified, correct inputs) pre = ip.preprocess_batch([native_text], src_lang=src_code_for_indictrans, tgt_lang="eng_Latn") inputs = tokenizer(pre, return_tensors="pt", padding=True).to(device) with torch.no_grad(): out = model.generate(**inputs, num_beams=5, max_length=256, early_stopping=True) dec = tokenizer.batch_decode(out, skip_special_tokens=True) post = ip.postprocess_batch(dec, lang=src_code_for_indictrans) translation = post[0] return (detected_lang, script_type, f"{score:.3f}", method, translation, indic_result, indicxlit_result, f"{parse_confidence_safely(indicxlit_conf):.3f}", native_text) except Exception as e: import traceback print(f"FATAL ERROR in pipeline for input '{text}': {e}\n{traceback.format_exc()}") return ("Error", "Error", "0.0", "Pipeline Error", str(e), "", "", "0.0", "") #old working not properly import gradio as gr import pandas as pd from indic_transliteration import sanscript from indic_transliteration.sanscript import transliterate import requests from typing import List, Dict, Union, Optional import traceback # --- All Helper Functions - DEFINED FIRST --- def parse_confidence_safely(conf_value): """Safely converts and clamps confidence scores to the [0.0, 1.0] range.""" try: return min(max(float(conf_value), 0.0), 1.0) except (ValueError, TypeError): return 0.0 # NOTE: Ensure your other helper functions like 'enhanced_transliterate_robust' # and 'indicxlit_transliterate' are also defined here or in a cell above. # --- The Corrected Pipeline Function --- # This is the single, robust function that powers the app. def detect_and_translate_single_corrected(text): """ Corrected and robust pipeline function using the single MASTER_LANGUAGE_MAPPING. """ try: # 1. Language Detection (with safe confidence score parsing) preds = lid.batch_predict([text], 1) item = preds[0] if isinstance(item, dict): detected_lang = item.get("lang", item.get("pred_lang", "")) score = parse_confidence_safely(item.get("score", 0.0)) # Using the helper else: _, detected_lang, raw_score, _ = item score = parse_confidence_safely(raw_score) # Using the helper is_romanized = detected_lang.endswith("_Latn") script_type = "Romanized" if is_romanized else "Native Script" # 2. Unified Mapping Lookup if detected_lang not in MASTER_LANGUAGE_MAPPING: return (detected_lang, script_type, f"{score:.3f}", "Unsupported", f"Language '{detected_lang}' not in master mapping.", "", "", "0.0", "") lang_info = MASTER_LANGUAGE_MAPPING[detected_lang] src_code_for_indictrans = lang_info["indictrans_code"] xlit_code_for_api = lang_info["indicxlit_code"] script_for_fallback = lang_info["script"] # 3. Transliteration Logic native_text = text method = "IndicTrans2 Only" indic_result, indicxlit_result, indicxlit_conf = "", "", 0.0 if is_romanized: if script_for_fallback: indic_result = enhanced_transliterate_robust(text, script_for_fallback) else: indic_result = text if xlit_code_for_api: indicxlit_result, indicxlit_conf = indicxlit_transliterate(text, xlit_code_for_api) else: indicxlit_result, indicxlit_conf = text, 0.0 if indicxlit_conf > 0.5: native_text = indicxlit_result method = "IndicXlit API + IndicTrans2" else: native_text = indic_result method = "Rule-Based Fallback + IndicTrans2" # 4. Final Translation pre = ip.preprocess_batch([native_text], src_lang=src_code_for_indictrans, tgt_lang="eng_Latn") inputs = tokenizer(pre, return_tensors="pt", padding=True).to(device) with torch.no_grad(): out = model.generate(**inputs, num_beams=5, max_length=256, early_stopping=True) dec = tokenizer.batch_decode(out, skip_special_tokens=True) post = ip.postprocess_batch(dec, lang=src_code_for_indictrans) translation = post[0] return (detected_lang, script_type, f"{score:.3f}", method, translation, indic_result, indicxlit_result, f"{parse_confidence_safely(indicxlit_conf):.3f}", native_text) except Exception as e: print(f"FATAL ERROR in pipeline: {e}\n{traceback.format_exc()}") return ("Error", "Error", "0.0", "Pipeline Error", str(e), "", "", "0.0", "") # --- The Gradio Interface and App Layout --- def gradio_interface(input_text): """ Handles the input from the Gradio UI and calls the corrected backend pipeline. """ if not input_text or not input_text.strip(): return ("Please enter some text.", "", "0.0", "", "", "", "", "0.0", "") return detect_and_translate_single_corrected(input_text) with gr.Blocks(title="Robust Indian Language AI Pipeline") as app: gr.Markdown("# 🇮🇳 Robust Indian Language AI Pipeline") gr.Markdown("This application uses a verified, multi-stage pipeline for accurate detection, transliteration, and translation.") with gr.Row(): with gr.Column(scale=3): input_text = gr.Textbox(label="Input Text", placeholder="Enter text, e.g., 'tum kaise ho'", lines=4) with gr.Column(scale=1): translate_btn = gr.Button("🔍 Detect & Translate", variant="primary", scale=1) with gr.Row(): detected_lang = gr.Textbox(label="🎯 Detected Language", interactive=False) script_type = gr.Textbox(label="📝 Script Type", interactive=False) confidence = gr.Textbox(label="🎯 LID Confidence Score", interactive=False) method = gr.Textbox(label="⚙️ Method Used", interactive=False) translation_output = gr.Textbox(label="🌍 English Translation", interactive=False, lines=3) gr.Markdown("--- \n ## 🔄 Transliteration Comparison (for Romanized input)") with gr.Row(): indic_output = gr.Textbox(label="Method 1: Rule-Based Fallback", interactive=False) indicxlit_output = gr.Textbox(label="Method 2: IndicXlit AI Model", interactive=False) final_native_text = gr.Textbox(label="🏆 Final Native Text Used for Translation", interactive=False, lines=2) gr.Examples( examples=[ ["tum kaise ho"], ["neenga epdi irukeenga"], ["ami bhalo achi"], ["Mera naam kya hai"], ["तुम कैसे हो?"], ["நீங்கள் எப்படி இருக்கிறீர்கள்?"], ["Dillī bhārat kī rājadhānī hai"], # Romanized with diacritics ["dogri vich tusada swagat hai"], # Test unsupported transliteration ], inputs=input_text, label="📚 Try these examples:" ) # I have simplified the outputs list to match the 9 return values of the interface function outputs = [ detected_lang, script_type, confidence, method, translation_output, indic_output, indicxlit_output, # Removed indicxlit_confidence as it wasn't a separate component final_native_text ] # This click handler will fail because the number of outputs does not match the return tuple of the function # Let's fix the outputs list to match the 9-tuple returned by gradio_interface fixed_outputs = [ detected_lang, script_type, confidence, method, translation_output, indic_output, indicxlit_output, gr.Textbox(visible=False), # Placeholder for confidence score final_native_text ] # The above is not a good solution. The UI must match the function output. Let's fix the UI. # --- FINAL CORRECTED GRADIO APP --- with gr.Blocks(title="Robust Indian Language AI Pipeline") as app: gr.Markdown("# 🇮🇳 Robust Indian Language AI Pipeline") # ... (UI components as defined before) ... # The UI from your sample image is being used here for consistency. with gr.Row(): with gr.Column(scale=3): input_text = gr.Textbox(label="Input Text", placeholder="e.g., 'tum kaise ho'", lines=4) with gr.Column(scale=1): translate_btn = gr.Button("🔍 Detect & Translate", variant="primary") with gr.Row(): detected_lang = gr.Textbox(label="🎯 Detected Language", interactive=False) script_type = gr.Textbox(label="📝 Script Type", interactive=False) with gr.Row(): confidence = gr.Textbox(label="🎯 LID Confidence", interactive=False) method = gr.Textbox(label="⚙️ Method Used", interactive=False) translation_output = gr.Textbox(label="🌍 English Translation", interactive=False, lines=3) gr.Markdown("--- \n ## 🔄 Transliteration Comparison") with gr.Row(): indic_output = gr.Textbox(label="Rule-Based Result", interactive=False) indicxlit_output = gr.Textbox(label="IndicXlit API Result", interactive=False) # We add the missing component for IndicXlit confidence indicxlit_conf_output = gr.Textbox(label="🎯 IndicXlit Confidence", interactive=False) final_native_text = gr.Textbox(label="🏆 Final Native Text", interactive=False, lines=2) gr.Examples( examples=[["tum kaise ho"],["neenga epdi irukeenga"]], inputs=input_text ) # Now the outputs list matches the 9 return values of the function perfectly translate_btn.click( fn=gradio_interface, inputs=[input_text], outputs=[ detected_lang, script_type, confidence, method, translation_output, indic_output, indicxlit_output, indicxlit_conf_output, final_native_text ] ) app.launch(share=True, debug=False) from google.colab import files import os print("Please upload your Excel file again to ensure it's available.") uploaded = files.upload() # This will get the name of the file you just uploaded if uploaded: file_path = list(uploaded.keys())[0] print(f"\n✅ File '{file_path}' is ready for analysis.") else: file_path = None print(f"\n❌ Upload failed. Please try again.") import pandas as pd if file_path: try: xls = pd.ExcelFile(file_path) print("✅ Success! Available sheet names in your file are:") print(xls.sheet_names) except Exception as e: print(f"❌ Could not read the Excel file. Error: {e}") else: print("❌ Cannot check sheets because no file was uploaded.") import pandas as pd import difflib import re from google.colab import files # --- This block contains all the necessary, verified functions --- def parse_confidence_safely(conf_value): """Safely converts and clamps confidence scores to the [0.0, 1.0] range.""" try: return min(max(float(conf_value), 0.0), 1.0) except (ValueError, TypeError): return 0.0 def calculate_similarity_fixed(text1, text2): """A corrected similarity function that handles all scripts without deleting characters.""" def preprocess(text): if not isinstance(text, str): text = str(text) return " ".join(text.strip().lower().split()) t1_processed, t2_processed = preprocess(text1), preprocess(text2) if not t1_processed and not t2_processed: return 1.0 if not t1_processed or not t2_processed: return 0.0 return difflib.SequenceMatcher(None, t1_processed, t2_processed).ratio() def run_pipeline_analysis_from_excel(filepath: str, sheet_name: str): """Loads data from a specific Excel sheet and runs the full analysis pipeline.""" print(f"🔄 Loading data from '{filepath}', sheet: '{sheet_name}'...") try: df = pd.read_excel(filepath, sheet_name=sheet_name, engine='openpyxl') print(f"✅ Successfully loaded {len(df)} samples.") except Exception as e: print(f"❌ ERROR: Could not read the sheet named '{sheet_name}'. Please check the file and sheet name. Error: {e}") return None analysis_results = [] print("🚀 Starting pipeline analysis on each sample...") for index, row in df.iterrows(): true_language = row['Language'] roman_input = row['Roman_Input'] native_ground_truth = row['Native_Output'] english_ground_truth = row['English_Question'] (detected_lang, _, raw_lid_score, _, translated_english, _, roman_to_native_output, _, _) = detect_and_translate_single_corrected(roman_input) lid_confidence = parse_confidence_safely(raw_lid_score) roman_to_native_similarity = calculate_similarity_fixed(roman_to_native_output, native_ground_truth) native_to_english_similarity = calculate_similarity_fixed(translated_english, english_ground_truth) lang_map_info = MASTER_LANGUAGE_MAPPING.get(detected_lang, {}) xlit_mapping_used = f"{lang_map_info.get('indicxlit_code', 'N/A')}" it2_mapping_used = f"{lang_map_info.get('indictrans_code', 'N/A')}" analysis_results.append({ 'Language (Ground Truth)': true_language, 'IndicLID Confidence': lid_confidence, 'LID Mapping Used': detected_lang, 'Roman to Native Similarity': f"{roman_to_native_similarity:.2%}", 'IndicXlit Mapping Used': xlit_mapping_used, 'Native to English Similarity': f"{native_to_english_similarity:.2%}", 'IndicTrans2 Mapping Used': it2_mapping_used, }) print(f" Processed sample {index + 1}/{len(df)}: {true_language}") print("🎉 Analysis complete!") return pd.DataFrame(analysis_results) # --- Execute the Analysis with the Correct Uploaded File Name --- # The file path is now set to the name Colab assigned to your uploaded file. file_path_to_analyze = 'roman_native - Sheet_final (2).xlsx' sheet_name_to_analyze = 'Sheet1' # Run the main analysis function results_df = run_pipeline_analysis_from_excel(uploaded_file_path, sheet_name_to_analyze) # Display and Save the Final Report if results_df is not None: print("\n--- Analysis Report ---") display(results_df) report_filename = 'pipeline_analysis_report.xlsx' results_df.to_excel(report_filename, index=False) print(f"\n✅ Report saved to '{report_filename}'. You can now download it from the file browser on the left.") import pandas as pd import difflib import re from google.colab import files import os # --- This block contains all the necessary, verified functions and mappings --- # Make sure you have already run the cells that define: # 1. MASTER_LANGUAGE_MAPPING # 2. detect_and_translate_single_corrected() # 3. All the models (lid, tokenizer, model, ip) def parse_confidence_safely(conf_value): """Safely converts and clamps confidence scores to the [0.0, 1.0] range.""" try: return min(max(float(conf_value), 0.0), 1.0) except (ValueError, TypeError): return 0.0 def calculate_similarity_fixed(text1, text2): """A corrected similarity function that handles all scripts without deleting characters.""" def preprocess(text): if not isinstance(text, str): text = str(text) return " ".join(text.strip().lower().split()) t1_processed, t2_processed = preprocess(text1), preprocess(text2) if not t1_processed and not t2_processed: return 1.0 if not t1_processed or not t2_processed: return 0.0 return difflib.SequenceMatcher(None, t1_processed, t2_processed).ratio() def run_pipeline_analysis_from_excel(filepath: str, sheet_name: str): """Loads data from a specific Excel sheet and runs the full analysis pipeline.""" print(f"🔄 Loading data from '{filepath}', sheet: '{sheet_name}'...") try: df = pd.read_excel(filepath, sheet_name=sheet_name, engine='openpyxl') print(f"✅ Successfully loaded {len(df)} samples.") except FileNotFoundError: print(f"❌ ERROR: File '{filepath}' not found. Please re-upload it.") return None except Exception as e: print(f"❌ ERROR: Could not read the sheet named '{sheet_name}'. Error: {e}") return None analysis_results = [] print("🚀 Starting pipeline analysis on each sample...") for index, row in df.iterrows(): true_language = row['Language'] roman_input = row['Roman_Input'] native_ground_truth = row['Native_Output'] english_ground_truth = row['English_Question'] (detected_lang, _, raw_lid_score, _, translated_english, _, roman_to_native_output, _, _) = detect_and_translate_single_corrected(roman_input) lid_confidence = parse_confidence_safely(raw_lid_score) roman_to_native_similarity = calculate_similarity_fixed(roman_to_native_output, native_ground_truth) native_to_english_similarity = calculate_similarity_fixed(translated_english, english_ground_truth) lang_map_info = MASTER_LANGUAGE_MAPPING.get(detected_lang, {}) xlit_mapping_used = f"{lang_map_info.get('indicxlit_code', 'N/A')}" it2_mapping_used = f"{lang_map_info.get('indictrans_code', 'N/A')}" analysis_results.append({ 'Language (Ground Truth)': true_language, 'IndicLID Confidence': lid_confidence, 'LID Mapping Used': detected_lang, 'Roman to Native Similarity': f"{roman_to_native_similarity:.2%}", 'IndicXlit Mapping Used': xlit_mapping_used, 'Native to English Similarity': f"{native_to_english_similarity:.2%}", 'IndicTrans2 Mapping Used': it2_mapping_used, }) print(f" Processed sample {index + 1}/{len(df)}: {true_language}") print("🎉 Analysis complete!") return pd.DataFrame(analysis_results) # --- Execute the Analysis with the Correct Uploaded File Name and Sheet Name --- file_path_to_analyze = 'roman_native - Sheet_final (2).xlsx' sheet_name_to_analyze = 'Sheet1' # Run the main analysis function results_df = run_pipeline_analysis_from_excel(file_path_to_analyze, sheet_name_to_analyze) # Display and Save the Final Report if results_df is not None: print("\n--- Analysis Report ---") display(results_df) report_filename = 'pipeline_analysis_report.xlsx' results_df.to_excel(report_filename, index=False) print(f"\n✅ Report saved to '{report_filename}'. You can now download it from the file browser on the left.") import pandas as pd from collections import defaultdict, Counter import numpy as np def analyze_pipeline_report(filepath: str): """ Loads the pipeline analysis report and computes detailed accuracy metrics. Args: filepath (str): The path to the 'pipeline_analysis_report.xlsx' file. """ print(f"🔄 Loading analysis report from '{filepath}'...") try: df = pd.read_excel(filepath) print(f"✅ Report loaded successfully with {len(df)} entries.") except FileNotFoundError: print(f"❌ ERROR: Report file not found at '{filepath}'. Please ensure the file exists.") return except Exception as e: print(f"❌ An error occurred while reading the file: {e}") return # --- Data Aggregation --- language_stats = defaultdict(lambda: { 'total_samples': 0, 'lid_correct_count': 0, 'misdetections': Counter(), 'roman_native_sims': [], 'native_english_sims': [] }) for _, row in df.iterrows(): gt_lang_name = row['Language (Ground Truth)'] detected_lang_code = row['LID Mapping Used'] # We need to map the ground truth language name back to an expected LID code format. # This is a simplified assumption; a more robust way would be to have the expected code in the report. expected_lang_prefix = detected_lang_code.split('_')[0] if '_' in detected_lang_code else detected_lang_code # A simple check (can be improved with a reverse map if needed) is_correct_detection = gt_lang_name.lower() in detected_lang_code.lower() stats = language_stats[gt_lang_name] stats['total_samples'] += 1 if is_correct_detection: stats['lid_correct_count'] += 1 else: stats['misdetections'][detected_lang_code] += 1 # Safely convert similarity from string format like "95.24%" to float stats['roman_native_sims'].append(float(row['Roman to Native Similarity'].strip('%')) / 100.0) stats['native_english_sims'].append(float(row['Native to English Similarity'].strip('%')) / 100.0) # --- Metric Calculation --- report_data = [] for lang, stats in language_stats.items(): total = stats['total_samples'] if total == 0: continue lid_accuracy = stats['lid_correct_count'] / total avg_roman_native_sim = np.mean(stats['roman_native_sims']) avg_native_english_sim = np.mean(stats['native_english_sims']) report_data.append({ 'Language': lang, 'Samples': total, 'IndicLID Accuracy': f"{lid_accuracy:.2%}", 'Roman->Native Similarity': f"{avg_roman_native_sim:.2%}", 'Native->English Similarity': f"{avg_native_english_sim:.2%}", 'Top Misdetections': stats['misdetections'].most_common(2) }) report_df = pd.DataFrame(report_data).sort_values(by='Samples', ascending=False) # --- Overall Statistics --- total_samples = df.shape[0] overall_lid_accuracy = sum(s['lid_correct_count'] for s in language_stats.values()) / total_samples overall_roman_native_sim = np.mean([sim for s in language_stats.values() for sim in s['roman_native_sims']]) overall_native_english_sim = np.mean([sim for s in language_stats.values() for sim in s['native_english_sims']]) # Approximate pipeline accuracy: The chance a sample successfully passes all stages. overall_pipeline_accuracy = overall_lid_accuracy * overall_roman_native_sim * overall_native_english_sim # --- Display Results --- print("\n" + "="*50) print(" OVERALL PIPELINE PERFORMANCE") print("="*50) print(f"Total Samples Analyzed: {total_samples}") print("-" * 50) print(f"Model 1: IndicLID Accuracy: {overall_lid_accuracy:.2%}") print(f"Model 2: Roman->Native Sim.: {overall_roman_native_sim:.2%}") print(f"Model 3: Native->English Sim.: {overall_native_english_sim:.2%}") print("-" * 50) print(f"Approx. End-to-End Accuracy: {overall_pipeline_accuracy:.2%}") print("="*50) print("\n\n" + "="*80) print(" LANGUAGE-SPECIFIC BREAKDOWN") print("="*80) print(report_df.to_string(index=False)) # --- Execute the Analysis --- # Make sure your report file is named correctly report_file_path = 'pipeline_analysis_report.xlsx' analyze_pipeline_report(report_file_path) import pandas as pd from collections import defaultdict, Counter import numpy as np import os def create_final_detailed_report(input_filepath: str, output_filepath: str): """ Loads the pipeline report, performs a corrected and detailed analysis, and saves the results to a new, multi-sheet Excel file. """ print(f"🔄 Loading initial report from '{input_filepath}'...") try: df = pd.read_excel(input_filepath) print(f"✅ Report loaded successfully with {len(df)} entries.") except FileNotFoundError: print(f"❌ ERROR: File not found at '{input_filepath}'. Please ensure it exists.") return except Exception as e: print(f"❌ An error occurred while reading the file: {e}") return # --- Create the Correct Mapping for Verification --- # This maps the full language name to its 3-letter code prefix (e.g., "Hindi" -> "hin") lang_name_to_prefix = { lang_info['name']: code.split('_')[0] for code, lang_info in MASTER_LANGUAGE_MAPPING.items() } # --- Data Aggregation with Corrected Logic --- language_stats = defaultdict(lambda: { 'total_samples': 0, 'lid_correct_count': 0, 'misdetections': Counter(), 'roman_native_sims': [], 'native_english_sims': [] }) for _, row in df.iterrows(): gt_lang_name = row['Language (Ground Truth)'] detected_lang_code = row['LID Mapping Used'] # This is the corrected accuracy check expected_prefix = lang_name_to_prefix.get(gt_lang_name) is_correct_detection = expected_prefix is not None and expected_prefix in detected_lang_code stats = language_stats[gt_lang_name] stats['total_samples'] += 1 if is_correct_detection: stats['lid_correct_count'] += 1 else: stats['misdetections'][detected_lang_code] += 1 stats['roman_native_sims'].append(float(row['Roman to Native Similarity'].strip('%')) / 100.0) stats['native_english_sims'].append(float(row['Native to English Similarity'].strip('%')) / 100.0) # --- Metric Calculation --- total_samples = df.shape[0] overall_lid_accuracy = sum(s['lid_correct_count'] for s in language_stats.values()) / total_samples overall_roman_native_sim = np.mean([sim for s in language_stats.values() for sim in s['roman_native_sims']]) overall_native_english_sim = np.mean([sim for s in language_stats.values() for sim in s['native_english_sims']]) overall_pipeline_accuracy = overall_lid_accuracy * overall_roman_native_sim * overall_native_english_sim # --- Create DataFrames for Excel Output --- summary_df = pd.DataFrame({ 'Metric': ['Total Samples', 'IndicLID Accuracy', 'Roman->Native Similarity', 'Native->English Similarity', 'End-to-End Pipeline Accuracy'], 'Overall Result': [total_samples, f"{overall_lid_accuracy:.2%}", f"{overall_roman_native_sim:.2%}", f"{overall_native_english_sim:.2%}", f"{overall_pipeline_accuracy:.2%}"] }) details_data = [] for lang, stats in language_stats.items(): total = stats['total_samples'] if total == 0: continue details_data.append({ 'Language': lang, 'Samples': total, 'IndicLID Accuracy': f"{stats['lid_correct_count'] / total:.2%}", 'Roman->Native Similarity': f"{np.mean(stats['roman_native_sims']):.2%}", 'Native->English Similarity': f"{np.mean(stats['native_english_sims']):.2%}", 'Top Misdetections': str(stats['misdetections'].most_common(2)) }) details_df = pd.DataFrame(details_data).sort_values(by='Samples', ascending=False) # --- Write to a New Excel File with Two Sheets --- print(f"💾 Saving final detailed report to '{output_filepath}'...") with pd.ExcelWriter(output_filepath, engine='openpyxl') as writer: summary_df.to_excel(writer, sheet_name='Overall Summary', index=False) details_df.to_excel(writer, sheet_name='Per Language Details', index=False) print("\n🎉 Success! Your final, detailed, and correct analysis report has been generated.") # --- Execute the Final Analysis --- # Define the input and output filenames input_report_file = 'pipeline_analysis_report.xlsx' final_detailed_report_file = 'final_detailed_analysis.xlsx' # Run the function create_final_detailed_report(input_report_file, final_detailed_report_file)