import gradio as gr import pandas as pd from pathlib import Path import logging from inference import correct_text, get_model_name, load_model from evaluation import evaluate_all_models, get_best_models, load_dataset logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def load_models_from_file(filepath: str): """Load model URLs from text file""" with open(filepath, 'r') as f: models = [line.strip() for line in f if line.strip()] return models def load_cached_evaluation(): """Load cached evaluation results if they exist""" cache_file = Path("evaluation_cache.csv") if cache_file.exists(): return pd.read_csv(cache_file) return None def save_evaluation_cache(results_df: pd.DataFrame): """Save evaluation results to cache""" results_df.to_csv("evaluation_cache.csv", index=False) # Load models MODEL_FILE = Path("models.txt") # Use HuggingFace dataset instead of CSV HF_DATASET_ID = "SPEAK-PP/sinhala-spelling-correction" HF_DATASET_SPLIT = "test" if not MODEL_FILE.exists(): raise FileNotFoundError(f"Model file not found: {MODEL_FILE}") models = load_models_from_file(str(MODEL_FILE)) model_names = [f"{i+1}. {get_model_name(m)}" for i, m in enumerate(models)] # Correction function def correct_sinhala_text(input_text, model_choice): """Correct Sinhala text using selected model""" if not input_text.strip(): return "Please enter some text to correct." model_idx = int(model_choice.split('.')[0]) - 1 selected_model = models[model_idx] try: corrected = correct_text(input_text, selected_model) return corrected except Exception as e: return f"Error: {str(e)}" # Evaluation function def evaluate_models(): """Evaluate all models""" try: eval_results = evaluate_all_models( models, hf_dataset_id=HF_DATASET_ID, split=HF_DATASET_SPLIT ) save_evaluation_cache(eval_results) # Sort by accuracy eval_results = eval_results.sort_values('accuracy', ascending=False) # Format for display display_df = eval_results[['model_name', 'accuracy', 'wer', 'cer']].copy() display_df['accuracy'] = display_df['accuracy'].apply(lambda x: f"{x:.2f}%") display_df['wer'] = display_df['wer'].apply(lambda x: f"{x:.4f}") display_df['cer'] = display_df['cer'].apply(lambda x: f"{x:.4f}") # Get top 5 top_5 = eval_results.head(5) top_5_text = "TOP 5 MODELS BY ACCURACY:\n\n" for i, (idx, row) in enumerate(top_5.iterrows(), 1): top_5_text += f"{i}. {row['model_name']}: {row['accuracy']:.2f}%\n" # Stats stats_text = f""" EVALUATION STATISTICS: - Best Accuracy: {eval_results['accuracy'].max():.2f}% - Average Accuracy: {eval_results['accuracy'].mean():.2f}% - Best WER: {eval_results[eval_results['wer'] != float('inf')]['wer'].min():.4f} - Best CER: {eval_results[eval_results['cer'] != float('inf')]['cer'].min():.4f} """ return display_df.to_string(), top_5_text + stats_text except Exception as e: return f"Error: {str(e)}", f"Error during evaluation: {str(e)}" # Create Gradio interface with gr.Blocks(title="🇱🇰 Sinhala Spelling Corrector") as demo: gr.Markdown("# 🇱🇰 Sinhala Spelling Correction Demo") gr.Markdown(""" This application demonstrates spelling correction for Sinhala text using 21 pre-trained models. Select a model, input a Sinhala sentence with spelling errors, and see how it gets corrected! """) # Demo Tab with gr.Tab("🎯 Demo"): gr.Markdown("## Real-Time Spelling Correction") with gr.Row(): with gr.Column(): gr.Markdown("### Select Model") model_dropdown = gr.Dropdown( choices=model_names, value=model_names[0], label="Choose a model" ) with gr.Column(): gr.Markdown("### Input Text") input_text = gr.Textbox( label="Enter Sinhala text with spelling errors", placeholder="ඉකක් පැලැස් කල්ලා ස්භ්ජෙත් තෝරගන්නවනන් ...", lines=5 ) gr.Markdown("### Corrected Output") output_text = gr.Textbox( label="Corrected text", lines=5, interactive=False ) correct_button = gr.Button("✨ Correct Text", variant="primary") correct_button.click( correct_sinhala_text, inputs=[input_text, model_dropdown], outputs=output_text ) # Statistics Tab with gr.Tab("📊 Statistics"): gr.Markdown("## Model Performance Evaluation") gr.Markdown("Click the button below to evaluate all 21 models on the dataset.") eval_button = gr.Button("🔄 Evaluate All Models (10-15 min on GPU, 30+ min on CPU)", variant="primary") with gr.Row(): with gr.Column(): gr.Markdown("### Results Table") results_table = gr.Textbox( label="Evaluation Results", lines=15, interactive=False ) with gr.Column(): gr.Markdown("### Top Models & Stats") top_models = gr.Textbox( label="Top 5 Models", lines=15, interactive=False ) eval_button.click( evaluate_models, outputs=[results_table, top_models] ) # About Tab with gr.Tab("ℹ️ About"): gr.Markdown(""" ### 🎯 Purpose This application demonstrates spelling correction capabilities across multiple Sinhala language models trained on spelling correction and ASR post-processing tasks. ### 📊 Dataset - **Source:** YouTube Sinhala ASR predictions - **Size:** 599 test samples - **Format:** (prediction, reference) pairs ### 🤖 Models The application includes 21 fine-tuned models, including: - **T5 Models:** mt5-small, mT5-base variants - **mBART Models:** mBART-large-50 variants - **mBERT:** Multilingual BERT - **Custom Models:** LoRA-adapted models for ASR correction ### 📈 Evaluation Metrics - **Accuracy:** Exact match rate between corrected and reference text - **WER (Word Error Rate):** Levenshtein distance at word level - **CER (Character Error Rate):** Levenshtein distance at character level ### 🚀 How to Use 1. **Demo Tab:** Select a model and input Sinhala text to see real-time corrections 2. **Statistics Tab:** Evaluate all models on the dataset and compare performance 3. **Results:** View performance metrics and identify the best model for your use case ### 💡 Tips - Results are cached for faster subsequent views - First evaluation may take 5-10 minutes depending on hardware - GPU acceleration (if available) speeds up inference significantly ### 📝 Model Attribution All models are from the [SPEAK-PP](https://huggingface.co/SPEAK-PP) organization on Hugging Face. --- **Created for:** Sinhala Language Processing & ASR Correction Demo **Framework:** Gradio + Hugging Face Transformers **Deployment:** Hugging Face Spaces """) if __name__ == "__main__": demo.launch()