import streamlit as st import os import torch from transformers import pipeline from src.improved_extractor import ImprovedExtractor from src.rag_service import RAGService import re import json import time import pandas as pd from google import genai from dotenv import load_dotenv import config # Load environment variables load_dotenv() # --- Page Configuration --- st.set_page_config( page_title=config.APP_NAME, page_icon="⚖️", layout="wide", ) # Initialize Session State Globally with robust checks def init_session_state(): if "messages" not in st.session_state: st.session_state["messages"] = [] if "current_file" not in st.session_state: st.session_state["current_file"] = None if "file_data" not in st.session_state: st.session_state["file_data"] = {} if "models_loaded" not in st.session_state: st.session_state["models_loaded"] = False init_session_state() # --- Styling --- st.markdown(""" """, unsafe_allow_html=True) @st.cache_resource def get_classifier(): local_path = config.FINE_TUNED_MODEL_PATH model_to_load = "nlpaueb/legal-bert-base-uncased" if os.path.isdir(local_path) and os.path.exists(os.path.join(local_path, "config.json")): model_to_load = local_path print(f"✅ Found potential local model: {local_path}") try: return pipeline( "text-classification", model=model_to_load, device=0 if torch.cuda.is_available() else -1, model_kwargs={"low_cpu_mem_usage": True} ) except Exception as e: print(f"⚠️ Error loading model from {model_to_load}: {e}") # Emergency fallback so the app stays live return pipeline( "text-classification", model="nlpaueb/legal-bert-base-uncased", device=0 if torch.cuda.is_available() else -1, model_kwargs={"low_cpu_mem_usage": True} ) @st.cache_resource def get_gemini_client(): # Centralized client initialization if config.GEMINI_API_KEY: try: c = genai.Client(api_key=config.GEMINI_API_KEY) print("✅ Initialized Gemini Client") return c except Exception as e: print(f"❌ Gemini Client init failed: {e}") # Fallback to Vertex vertex_json = os.path.join(config.BASE_DIR, "vertex_config.json") if os.path.exists(vertex_json): try: with open(vertex_json, "r") as f: v_config = json.load(f) os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(vertex_json) c = genai.Client( vertexai=True, project=v_config.get("project_id"), location="us-central1" ) print("✅ Initialized Vertex AI Client") return c except Exception as e: print(f"⚠️ Vertex AI failed: {e}") return None # Removed BERT-based get_ner to save RAM. Using Gemini for Entity Extraction. @st.cache_resource def get_rag(): r = RAGService() try: rag_data_path = os.path.join(config.BASE_DIR, "data", "text") if os.path.exists(rag_data_path): r.load_documents(rag_data_path) except Exception as e: print(f"Error loading RAG: {e}") return r # --- UI Loader (Non-Cached) --- def load_models(progress_bar=None, status_text=None): """ Orchestrates the loading of all models with UI progress feedback. The individual load functions are cached, so this function run fast after the first time. """ # 1. Risk Classifier if status_text: status_text.text("Loading Risk Classifier (BERT)...") if progress_bar: progress_bar.progress(20) classifier = get_classifier() # 2. Gemini LLM (for Summarization, RAG & NER) if status_text: status_text.text("Connecting to Gemini AI...") if progress_bar: progress_bar.progress(50) client = get_gemini_client() # 3. RAG Service if status_text: status_text.text("Indexing Document Library...") if progress_bar: progress_bar.progress(90) rag_service = get_rag() return { "classifier": classifier, "client": client, "extractor": ImprovedExtractor(), "rag": rag_service } # --- Initial Setup & Startup Sequence --- def startup_sequence(): """ Shows a splash screen only on the first run of the session. """ if not st.session_state.get('models_loaded'): placeholder = st.empty() with placeholder.container(): st.markdown(f"""

⚖️ {config.APP_NAME}

Warming up the AI engine... This takes about 30-45 seconds on first boot.

""", unsafe_allow_html=True) progress_bar = st.progress(0) status_text = st.empty() # This triggers the individual cached loads load_models(progress_bar, status_text) progress_bar.progress(100) status_text.text("System ready!") st.session_state['models_loaded'] = True time.sleep(1) placeholder.empty() # Run startup startup_sequence() models = load_models() # Returns instantly if already cached classifier = models["classifier"] client = models["client"] extractor = models["extractor"] rag = models["rag"] def call_gemini(prompt, preferred_model=None): """Single helper for all Gemini calls. Retries with backoff and model fallback.""" if not client: return None models_to_try = [preferred_model] if preferred_model else config.MODEL_FALLBACKS for attempt in range(2): for model_id in models_to_try: try: response = client.models.generate_content(model=model_id, contents=prompt) if response and response.text: return response.text except Exception as e: err = str(e).upper() if "429" in err or "QUOTA" in err or "LIMIT" in err: if attempt < 1: wait = 10 * (attempt + 1) time.sleep(wait) break continue elif "404" in err or "NOT FOUND" in err: continue else: continue return None def get_summary(text): prompt = f""" Act as a Senior Legal Counsel with 20 years of experience in contract law. Review the provided legal contract and generate a high-level, professional executive summary. STRUCTURE YOUR RESPONSE AS FOLLOWS: 1. **Executive Overview**: High-level purpose of the agreement. 2. **Key Financial Terms**: Payment schedules, amounts, and late fees. 3. **Operational Obligations**: What must each party actually DO? 4. **Termination & Exit**: How do parties leave, and what are the notice periods? 5. **Critical Liability & Risk**: Indemnities, liability caps, and any "one-sided" clauses. 6. **Counsel's Recommendation**: A 2-3 sentence professional verdict on the contract's fairness. INSTRUCTIONS: - Use a professional, objective, and analytical tone. - If you find highly imbalanced or "predatory" clauses, mention them under "Critical Liability & Risk" in a factual, legal manner rather than using alarmist language. - Focus on specificities (dates, percentages, dollar amounts). - Ensure the summary is readable but dense with information. Contract Text: {text[:config.MAX_CHAR_LIMIT]} """ result = call_gemini(prompt) return result or "⚠️ Summary generation failed." def get_entities(text): """Smart sampling: first 20k + last 15k chars to capture preamble AND signature blocks.""" first_part = text[:20000] last_part = text[-15000:] if len(text) > 20000 else "" sample = first_part + "\n\n--- END OF DOCUMENT ---\n\n" + last_part prompt = f""" Act as a Legal Clerk. Extract the following core entities from the contract. CRITICAL ACCURACY RULES: 1. DO NOT extract generic placeholders like "[PROVIDER LEGAL NAME]", "[CUSTOMER]", or placeholders in curly brackets. 2. Examine BOTH the introductory paragraph AND the signature blocks at the end for ACTUAL company names. 3. If a field only contains a placeholder, write "NOT SPECIFIED (Generic Template Detected)". IDENTIFY: 1. Contracting Parties (Full legal names of all parties involved) 2. Effective Date (The start date of the agreement) 3. Governing Law (Which state/country's laws apply) 4. Total Contract Value (Specific monetary amount or fee structure) Contract Text: {sample} """ result = call_gemini(prompt) return result or "⚠️ Entity extraction failed." def check_unethical_clauses(text): """Scans full document (60k chars) for predatory clauses.""" prompt = f""" Act as a Senior Contract Auditor. Scan for predatory, hidden, or highly imbalanced clauses. Focus on: IP transfers, uncapped liability, sneaky auto-renewals, hidden exit fees. If found, respond: FLAGGED: [1-sentence explanation] If safe, respond: SAFE Contract Text: {text[:60000]} """ result = call_gemini(prompt) if result and "FLAGGED:" in result: return result.split("FLAGGED:")[1].strip() return None def get_risk(text): if not classifier: return "N/A", 0.0, "" try: cleaned = clean_text(text) # Use first 512 tokens as BERT limit result = classifier(cleaned[:512])[0] label_id = result['label'] mapping = { "LABEL_0": ("High Risk", "Critical issues found. Requires legal review."), "LABEL_1": ("Low Risk", "Standard safe clauses. Low legal overhead."), "LABEL_2": ("Medium Risk", "Minor deviations found. Proceed with caution.") } name, desc = mapping.get(label_id, (label_id, "")) return name, result['score'], desc except Exception as e: return f"Error: {e}", 0.0, "" def run_compliance_check(text, framework): """Runs compliance audit against the full document (60k chars).""" prompt = f""" Act as an expert compliance auditor. Check this contract against: '{framework}'. Evaluate 4-5 critical requirements. For each, give Pass (✅) or Fail (❌) with 1-sentence reason. Format as a clean Markdown list. Contract Text: {text[:60000]} """ result = call_gemini(prompt) return result or "⚠️ Compliance check failed." def clean_text(text): """ Cleans raw document text by removing noise like placeholders and extra whitespace. """ if not text: return "" # Remove long sequences of underscores (placeholders) text = re.sub(r'_{2,}', '', text) # Remove curly bracket placeholders e.g. {services/project name} text = re.sub(r'\{.*?\}', '', text) # Remove multiple newlines text = re.sub(r'\n{3,}', '\n\n', text) # Remove multiple spaces text = re.sub(r' +', ' ', text) # Remove "Page X" noise text = re.sub(r'Page \d+', '', text) return text.strip() def get_comparison_data(text): prompt = f""" Extract the following strictly as a JSON object with EXACTLY these keys: "Vendor_Name", "Total_Pricing", "Term_Duration", "Liability_Cap", "Termination_Notice" If a field is missing, use "Not Specified". Do NOT output any markdown blocks. ONLY output the raw JSON object. Contract text: {text[:15000]} """ result = call_gemini(prompt) if result: try: resp_text = result.strip() if resp_text.startswith("```json"): resp_text = resp_text[7:] if resp_text.endswith("```"): resp_text = resp_text[:-3] return json.loads(resp_text) except Exception: pass return None # Check for model health is_fine_tuned = os.path.isdir("legal_bert_finetuned_risk") # --- Sidebar --- with st.sidebar: st.image("https://cdn-icons-png.flaticon.com/512/2901/2901306.png", width=100) st.title("Admin Panel") st.info("Upload your legal contracts to begin automated analysis. Upload multiple files for vendor comparison.") uploaded_files = st.file_uploader("Upload PDF Contract(s)", type=["pdf"], accept_multiple_files=True) # --- Header --- st.title("⚖️ Legal Document & Risk Analyzer") st.markdown("Automated intelligence for contract review, risk mitigation, and semantic search.") if not is_fine_tuned: st.warning("⚠️ **Warning:** No fine-tuned model found in `./legal_bert_finetuned_risk`. The risk classification is currently using 'Base' weights and will be inaccurate. Please provide your trained model files for accurate assessment.") if uploaded_files: if len(uploaded_files) == 1: uploaded_file = uploaded_files[0] # Check if a new file was uploaded to reset the cache if st.session_state["current_file"] != uploaded_file.name: st.session_state["current_file"] = uploaded_file.name st.session_state["messages"] = [] # Reset chat st.session_state["file_data"] = {} # Reset analytics cache # Save uploaded file to temp path with open(uploaded_file.name, "wb") as f: f.write(uploaded_file.getbuffer()) # Text Extraction with st.spinner("Processing document..."): raw_text = extractor.extract_text(uploaded_file.name) cleaned_text = clean_text(raw_text) # Clean up temp file immediately so it doesn't clutter the folder try: os.remove(uploaded_file.name) except OSError: pass # Phase 1: Summary & Scam Scan col_sum, col_risk = st.columns([2, 1]) with col_sum: st.subheader("🤖 AI Summary") if "summary" not in st.session_state["file_data"]: with st.spinner("Analyzing document with Gemini 2.5 Pro..."): st.session_state["file_data"]["summary"] = get_summary(raw_text) st.markdown(st.session_state["file_data"]["summary"]) # Note: Scam/Ethical check is still performed but information is now integrated into the Risk and Summary sections if "scam_warning" not in st.session_state["file_data"]: time.sleep(2) # Rate limit protection with st.spinner("Auditing clauses..."): st.session_state["file_data"]["scam_warning"] = check_unethical_clauses(raw_text) with col_risk: st.subheader("⚖️ Risk Profile") if "risk" not in st.session_state["file_data"]: with st.spinner("Analyzing risk..."): st.session_state["file_data"]["risk"] = get_risk(raw_text) label, score, description = st.session_state["file_data"]["risk"] color = "#16a34a" # Low if "Medium" in label: color = "#f59e0b" if "High" in label: color = "#ef4444" st.markdown(f"""

{label}

""", unsafe_allow_html=True) if description: st.markdown(f"**Actionable Insight:** {description}") if not is_fine_tuned: st.caption("🚨 Results are uncalibrated (Base Model).") # Phase 2: Professional Entity Extraction st.divider() st.subheader("🔍 Key Legal Entities") if "entities" not in st.session_state["file_data"]: time.sleep(2) # Rate limit protection with st.spinner("Extracting parties with AI..."): entities_text = get_entities(raw_text) # Use raw_text for full context st.session_state["file_data"]["entities"] = entities_text if entities_text else "No entities detected." st.info(st.session_state["file_data"]["entities"]) # Phase 2.5: Automated Compliance Checklists st.divider() st.subheader("🛡️ Compliance & Audit") st.markdown("Run automated checks against strict regulatory and industry frameworks.") frameworks = [ "Select a framework to audit...", "General Data Protection Regulation (GDPR)", "Standard SaaS Agreement Best Practices", "Independent Contractor / Freelance Standard" ] selected_framework = st.selectbox("Select Compliance Framework:", frameworks) if selected_framework != "Select a framework to audit...": cache_key = f"compliance_{selected_framework}" if cache_key not in st.session_state["file_data"]: with st.spinner(f"Running {selected_framework} audit..."): if not client: st.warning("Gemini AI is not connected.") else: st.session_state["file_data"][cache_key] = run_compliance_check(cleaned_text, selected_framework) if st.session_state["file_data"].get(cache_key): st.info(st.session_state["file_data"][cache_key]) elif client: st.error("Audit failed to generate.") # Phase 3: Interactive Chatbot st.divider() st.subheader("💬 Chat with Document") if not client: st.warning("⚠️ **Gemini API Key Missing:** Interactive chat is disabled.") else: # Display chat messages from history on app rerun for message in st.session_state.messages: with st.chat_message(message["role"]): st.markdown(message["content"]) # Accept user input if prompt := st.chat_input("Ask a question about this contract (e.g., 'What are the termination conditions?')..."): # Add user message to chat history st.session_state.messages.append({"role": "user", "content": prompt}) # Display user message in chat message container with st.chat_message("user"): st.markdown(prompt) # Display assistant response in chat message container with st.chat_message("assistant"): message_placeholder = st.empty() with st.spinner("Analyzing document and generating answer..."): context = cleaned_text[:100000] # Safe limit for large documents # Build conversation history for the prompt history_text = "" for msg in st.session_state.messages[:-1]: # exclude the current prompt role_str = "User" if msg["role"] == "user" else "Assistant" history_text += f"{role_str}: {msg['content']}\n" full_prompt = f""" You are a helpful legal assistant. Answer the user's latest question based ONLY on the following contract text and the conversation history so far. If the answer is not in the text, say "I cannot find the answer to this in the document." Contract Text: {context} Conversation History: {history_text} Latest Question: {prompt} """ answer = call_gemini(full_prompt) if not answer: answer = "Failed to generate answer. Please try again." message_placeholder.markdown(answer) # Add assistant response to chat history st.session_state.messages.append({"role": "assistant", "content": answer}) else: # Multi-Document Vendor Comparison Mode st.header("📊 Multi-Document Vendor Comparison Matrix") st.markdown("Comparing key terms across multiple uploaded contracts.") comparison_results = [] # We use a progress bar to show extraction status progress_bar = st.progress(0) status_text = st.empty() for i, file in enumerate(uploaded_files): status_text.text(f"Extracting data from {file.name} ({i+1}/{len(uploaded_files)})...") # Save temp with open(file.name, "wb") as f: f.write(file.getbuffer()) # Extract raw_text = extractor.extract_text(file.name) cleaned_text = clean_text(raw_text) # Clean up temp file immediately try: os.remove(file.name) except OSError: pass # Get structured data data = get_comparison_data(cleaned_text) if data: data["Filename"] = file.name comparison_results.append(data) else: comparison_results.append({ "Filename": file.name, "Vendor_Name": "Extraction Failed", "Total_Pricing": "N/A", "Term_Duration": "N/A", "Liability_Cap": "N/A", "Termination_Notice": "N/A" }) progress_bar.progress((i + 1) / len(uploaded_files)) status_text.text("Extraction Complete!") if comparison_results: st.divider() df = pd.DataFrame(comparison_results) # Reorder columns to put Filename first cols = ["Filename", "Vendor_Name", "Total_Pricing", "Term_Duration", "Liability_Cap", "Termination_Notice"] df = df[[c for c in cols if c in df.columns]] st.dataframe(df, use_container_width=True, hide_index=True) st.success("Comparison Matrix generated successfully! You can download this table via the download button inside the table view.") else: st.warning("Please upload a PDF document in the sidebar to start analysis.") st.info("💡 **Tip:** You can use the Library Search below to query your repository of legal documents.") st.divider() st.subheader("📚 Global Library Search") lib_query = st.text_input("Identify patterns across your entire library:") if lib_query: with st.spinner("Searching library..."): relevant_chunks = rag.query(lib_query, top_k=3) answer = rag.generate_answer(lib_query, relevant_chunks, client) st.markdown("### 🤖 Synthesized Knowledge") st.success(answer) st.markdown("#### Document References") for i, res in enumerate(relevant_chunks): st.markdown(f"**{i+1}. From {res['metadata']['file']}:**") st.caption(res['text']) # --- Footer --- st.divider() st.caption("LegalAI Analyzer v1.2 | Powered by Legal-BERT & T5")