import os import gradio as gr import sys from sentence_transformers import SentenceTransformer import torch import torch.nn.functional as F from functools import lru_cache # Debug print: Check current working directory import os import subprocess import sys def install_private_repo(): github_agent_token = os.getenv('GITHUB_AGENT_TOKEN') if not github_agent_token: print("Error: GITHUB_AGENT_TOKEN environment variable is not set.") return False repo_url = f"git+https://{github_agent_token}@github.com/punekichikki/ideaLensAgent.git" try: #print(f"Attempting to install from private repo: {repo_url.replace(github_agent_token, '*******')}") # Install with verbose output result = subprocess.run( [sys.executable, "-m", "pip", "install", "-v", repo_url], capture_output=True, text=True ) if result.returncode != 0: print(f"Installation failed with error code: {result.returncode}") print(f"stdout: {result.stdout}") print(f"stderr: {result.stderr}") return False #print("Installation output:") #print(result.stdout) # Check installed packages print("\nInstalled packages:") pip_list = subprocess.run( [sys.executable, "-m", "pip", "list"], capture_output=True, text=True ) #print(pip_list.stdout) # Check Python path #print("\nPython path:") #print(sys.path) # Try to find the package location find_package = subprocess.run( [sys.executable, "-c", "import idealens_agents; print(idealens_agents.__file__)"], capture_output=True, text=True ) #print("\nPackage location attempt:") #print("stdout:", find_package.stdout) #print("stderr:", find_package.stderr) return True except Exception as e: print(f"Error: An unexpected error occurred: {str(e)}") traceback.print_exc() return False # Add debug information before installation #print("Current environment variables:", {k: v for k, v in os.environ.items() if 'TOKEN' in k}) #print("Python executable:", sys.executable) #print("Python version:", sys.version) #print("Current working directory:", os.getcwd()) #print("Directory contents:", os.listdir()) # Install private repo install_success = install_private_repo() if not install_success: print("Failed to install private repository. Exiting.") sys.exit(1) agents_dir = os.path.join(os.path.dirname(__file__), 'agents') sys.path.append(agents_dir) # Try importing with more detailed error handling from agents import GitHubAgent from agents import ArxivSearchAgent from agents import ProductHuntAgent from agents import RedditAgent from vertexai.generative_models import GenerativeModel import vertexai import asyncio import json import logging.config from typing import Dict, Any, Optional, Tuple import traceback from datetime import datetime import gc import jinja2 custom_css = """ .center-label { display: flex; flex-direction: column; /* Makes the label stack above the input*/ align-items: center; /* Horizontally center the contents*/ text-align: center; } .center-label .form { display: flex; flex-direction: column; align-items: center; width: 100%; } .center-label .label { text-align: center; width: 100%; } .equal-button { flex: 1; /* Makes the buttons share available space equally */ margin: 5px; background-color: #f0f0f0; font-size: 12px; } .loading-text textarea { text-align: center !important; font-weight: bold !important; color: #e67e22 !important; background-color: #f7f7f7 !important; } """ intro_text = """

Meet IdeaLens: the AI Agent for your product ideas

❓ Got an idea that could change the world? 🌍

💡 IdeaLens intelligently analyzes data to chart your idea's potential
🌟 It explores user perspectives and predicts market reactions
📊 It identifies competitors and uncovers technical resources to refine your concept
📚 By integrating cutting-edge academic insights, IdeaLens ensures thorough validation
🚀 With IdeaLens assisting you, success is just an idea away!

How IdeaLens Assists:

🧠 IdeaLens reveals hidden insights by connecting patterns across platforms like Reddit, ProductHunt, GitHub, and arXiv.
📋 It synthesizes competitive insights and surface critical technical resources to strengthen your vision
🔍 Want to go deeper? IdeaLens provides comprehensive strategic intelligence from each platform
⏳ In just 5 minutes, IdeaLens delivers focused, actionable guidance
🔥 Ready to let IdeaLens assist you?
👉 Enter your idea in the search prompt or try one of the curated examples below! 🎯✨
""" # Debug print: Initial imports complete print("Debug: Initial imports complete") # Create logs directory if it doesn't exist os.makedirs(os.path.join(os.path.dirname(__file__), 'logs'), exist_ok=True) print(f"Debug: Logs directory created or exists at {os.path.join(os.path.dirname(__file__), 'logs')}") print("Debug: Agents imported") # Load configuration print("Debug: Loading configuration...") with open('CONFIG.json') as f: CONFIG = json.load(f) print("Debug: Configuration loaded successfully") # Set up logging print("Debug: Setting up logging...") logging.config.fileConfig('logging.conf') logger = logging.getLogger('app') print("Debug: Logging setup complete") class QueryPreprocessor: def __init__(self): self._model = None print("Debug: QueryPreprocessor initialized") # Add print statement here @property @lru_cache() def model(self) -> SentenceTransformer: if self._model is None: print("Debug: Loading sentence transformer model...") self._model = SentenceTransformer('all-MiniLM-L6-v2') logger.info("Sentence transformer model initialized") print("Debug: Sentence transformer model loaded successfully") return self._model async def extract_core_concepts(self, text: str) -> str: """Extract core concepts from text, removing structural elements""" markers = ["Market analysis request:", "Target sector:", "Primary features:", "Business model category:", "Technical requirements:"] cleaned = text for marker in markers: cleaned = cleaned.replace(marker, "") return cleaned.strip() async def check_semantic_similarity(self, original: str, processed: str, threshold: float = 0.7) -> bool: try: # Extract core concepts from processed text processed_core = await self.extract_core_concepts(processed) # Run embedding computation in thread pool loop = asyncio.get_event_loop() embeddings = await loop.run_in_executor( None, lambda: ( self.model.encode(original, convert_to_tensor=True), self.model.encode(processed_core, convert_to_tensor=True) ) ) original_embedding, processed_embedding = embeddings # Calculate cosine similarity similarity = F.cosine_similarity( original_embedding.unsqueeze(0), processed_embedding.unsqueeze(0) ).item() logger.info(f"Original query: {original}") logger.info(f"Processed core concepts: {processed_core}") logger.info(f"Semantic similarity: {similarity:.3f}") return similarity > threshold except Exception as e: logger.error(f"Error in semantic similarity check: {str(e)}") return False def configure_environment(): start_time = datetime.now() print("Debug: Starting configure_environment") required_env_vars = [ 'GITHUB_TOKEN', 'PRODUCT_HUNT_TOKEN', 'REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET', 'REDDIT_USER_AGENT', 'GOOGLE_APPLICATION_CREDENTIALS_JSON', 'GITHUB_APPLICATION_CREDENTIALS_JSON', 'ARXIV_APPLICATION_CREDENTIALS_JSON', 'PRODUCT_HUNT_APPLICATION_CREDENTIALS_JSON', 'REDDIT_APPLICATION_CREDENTIALS_JSON', 'GITHUB_CLOUD_PROJECT', 'ARXIV_CLOUD_PROJECT', 'PRODUCTHUNT_CLOUD_PROJECT', 'REDDIT_CLOUD_PROJECT' ] print(f"Debug: Required environment variables: {required_env_vars}") missing_vars = [var for var in required_env_vars if not os.getenv(var)] if missing_vars: error_message = f"Missing required environment variables: {', '.join(missing_vars)}. " \ f"Please check .env.example for required variables." print(f"Debug: Error - {error_message}") raise EnvironmentError(error_message) print("Debug: All required environment variables are present") # Set up Google credentials from the JSON stored in env variable if 'GOOGLE_APPLICATION_CREDENTIALS_JSON' in os.environ: print("Debug: Found GOOGLE_APPLICATION_CREDENTIALS_JSON") creds_json = os.environ['GOOGLE_APPLICATION_CREDENTIALS_JSON'] with open('/tmp/google_credentials.json', 'w') as f: f.write(creds_json) os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/tmp/google_credentials.json' print("Debug: Google credentials file created and GOOGLE_APPLICATION_CREDENTIALS set") # Set up Github credentials from the JSON stored in env variable if 'GITHUB_APPLICATION_CREDENTIALS_JSON' in os.environ: print("Debug: Found GITHUB_APPLICATION_CREDENTIALS_JSON") creds_json = os.environ['GITHUB_APPLICATION_CREDENTIALS_JSON'] with open('/tmp/github_credentials.json', 'w') as f: f.write(creds_json) os.environ['GITHUB_APPLICATION_CREDENTIALS'] = '/tmp/github_credentials.json' print("Debug: Github credentials file created and GITHUB_APPLICATION_CREDENTIALS set") # Set up Arxiv credentials from the JSON stored in env variable if 'ARXIV_APPLICATION_CREDENTIALS_JSON' in os.environ: print("Debug: Found ARXIV_APPLICATION_CREDENTIALS_JSON") creds_json = os.environ['ARXIV_APPLICATION_CREDENTIALS_JSON'] with open('/tmp/arxiv_credentials.json', 'w') as f: f.write(creds_json) os.environ['ARXIV_APPLICATION_CREDENTIALS'] = '/tmp/arxiv_credentials.json' print("Debug: Arxiv credentials file created and ARXIV_APPLICATION_CREDENTIALS set") # Set up Product Hunt credentials from the JSON stored in env variable if 'PRODUCTHUNT_APPLICATION_CREDENTIALS_JSON' in os.environ: print("Debug: Found PRODUCTHUNT_APPLICATION_CREDENTIALS_JSON") creds_json = os.environ['PRODUCTHUNT_APPLICATION_CREDENTIALS_JSON'] with open('/tmp/producthunt_credentials.json', 'w') as f: f.write(creds_json) os.environ['PRODUCTHUNT_APPLICATION_CREDENTIALS'] = '/tmp/producthunt_credentials.json' print("Debug: Product Hunt credentials file created and PRODUCTHUNT_APPLICATION_CREDENTIALS set") # Set up Reddit credentials from the JSON stored in env variable if 'REDDIT_APPLICATION_CREDENTIALS_JSON' in os.environ: print("Debug: Found REDDIT_APPLICATION_CREDENTIALS_JSON") creds_json = os.environ['REDDIT_APPLICATION_CREDENTIALS_JSON'] with open('/tmp/reddit_credentials.json', 'w') as f: f.write(creds_json) os.environ['REDDIT_APPLICATION_CREDENTIALS'] = '/tmp/reddit_credentials.json' print("Debug: Reddit credentials file created and REDDIT_APPLICATION_CREDENTIALS set") end_time = datetime.now() print(f"Debug: Finished configure_environment, Time taken: {end_time - start_time}") def initialize_agents() -> Optional[Dict[str, Any]]: """Initialize all search agents with proper error handling.""" start_time = datetime.now() print("Debug: Starting initialize_agents") try: print("Debug: Initializing Vertex AI...") logger.info("Initializing agents...") vertexai.init(project=None, location="us-central1") print("Debug: Vertex AI initialized successfully") summary_model = GenerativeModel("gemini-pro") print("Debug: Summary model initialized") # Initialize agents print("Debug: Initializing agents...") github_agent = GitHubAgent( project_id=os.getenv('GITHUB_CLOUD_PROJECT'), credentials_path='/tmp/github_credentials.json' ) print("Debug: GitHub agent initialized") arxiv_agent = ArxivSearchAgent( project_id=os.getenv('ARXIV_CLOUD_PROJECT'), credentials_path='/tmp/arxiv_credentials.json' ) print("Debug: Arxiv agent initialized") producthunt_agent = ProductHuntAgent( project_id=os.getenv('PRODUCTHUNT_CLOUD_PROJECT'), credentials_path='/tmp/producthunt_credentials.json' ) print("Debug: Product Hunt agent initialized") reddit_agent = RedditAgent( project_id=os.getenv('REDDIT_CLOUD_PROJECT'), credentials_path='/tmp/reddit_credentials.json' ) print("Debug: Reddit agent initialized") agents_dict = { 'summary_model': summary_model, 'github_agent': {'agent': github_agent, 'model': summary_model,'config': CONFIG.get('github_settings', {})}, 'arxiv_agent': {'agent': arxiv_agent, 'model': summary_model}, 'producthunt_agent': { 'agent': producthunt_agent, 'model': summary_model, 'config': CONFIG.get('producthunt_settings', {}) # Pass ProductHunt specific config }, 'reddit_agent': {'agent': reddit_agent, 'model': summary_model} } print("Debug: Agents initialized successfully.") end_time = datetime.now() print(f"Debug: Finished initialize_agents, Time taken: {end_time - start_time}") return agents_dict except Exception as e: logger.error(f"Error initializing agents: {str(e)}") print(f"Debug: Error initializing agents: {str(e)}") logger.error(traceback.format_exc()) print(f"Debug: Traceback: {traceback.format_exc()}") return None async def fetch_with_timeout(coro: Any, timeout: int = 600) -> Any: """Wrapper for async operations with timeout.""" start_time = datetime.now() print(f"Debug: Starting fetch_with_timeout with timeout: {timeout}") try: result = await asyncio.wait_for(coro, timeout=timeout) print("Debug: fetch_with_timeout completed successfully") end_time = datetime.now() print(f"Debug: Finished fetch_with_timeout, Time taken: {end_time - start_time}") return result except asyncio.TimeoutError: logger.error(f"Task timed out: {coro}") print(f"Debug: Task timed out: {coro}") return "Task timed out" async def process_prompt(prompt: str, agents: Dict[str, Any], progress: Optional[gr.Progress] = None) -> Tuple[str, str, str, str, str]: try: print(f"\n=== APP.PY PROCESSING START ===") print(f"Original prompt received: {prompt}") if progress is not None: progress(0, "Starting preprocessing") # Initialize task tracking tasks_completed = 0 # Initialize here total_tasks = 4 # Total number of major tasks preprocessing_prompt = f""" Transform this business/app idea query into structured market research format. Original query: {prompt} Transform using these rules: 1. Begin with "Market analysis request:" 2. Include "Target sector:" 3. Specify "Primary features:" 4. Add "Business model category:" 5. End with "Technical requirements:" Format as a single paragraph without the rule headers. Use professional business language. """ print(f"Generated preprocessing prompt: {preprocessing_prompt}") # Initialize preprocessor preprocessor = QueryPreprocessor() try: # First pass - structure the query response = await fetch_with_timeout( asyncio.to_thread( lambda: agents['summary_model'].generate_content(preprocessing_prompt) ) ) structured_prompt = response.text.strip() print(f"After first pass structuring: {structured_prompt}") # Second pass - format for API efficiency format_prompt = f""" Convert this market analysis into a concise, direct query suitable for API processing. Keep all key details but remove unnecessary words. Query: {structured_prompt} """ format_response = await fetch_with_timeout( asyncio.to_thread( lambda: agents['summary_model'].generate_content(format_prompt) ) ) processed_prompt = format_response.text.strip() print(f"After second pass formatting: {processed_prompt}") # Check semantic similarity with core concept extraction is_similar = await preprocessor.check_semantic_similarity(prompt, processed_prompt) print(f"Semantic similarity check result: {is_similar}") if not processed_prompt or not is_similar: logger.warning("Query preprocessing validation failed, using original query") logger.info(f"Original: {prompt}") logger.info(f"Processed: {processed_prompt}") processed_prompt = prompt except Exception as e: logger.error(f"Query preprocessing failed: {str(e)}") processed_prompt = prompt logger.info(f"Original prompt: {prompt}") logger.info(f"Processed prompt: {processed_prompt}") # Sequential execution with individual error handling if progress is not None: progress(0.2, "Processing ProductHunt data") try: print("\n=== CALLING PRODUCTHUNT AGENT ===") producthunt_result = await fetch_with_timeout( agents['producthunt_agent']['agent'].process_search( agents['producthunt_agent']['model'], processed_prompt ), timeout=CONFIG['timeouts']['analysis'] ) print(f"ProductHunt result received: {'Empty' if not producthunt_result else 'Has content'}") producthunt_result = str(producthunt_result) tasks_completed += 1 await asyncio.sleep(5) gc.collect() except Exception as e: producthunt_result = f"ProductHunt Error: {str(e)}" logger.error(f"ProductHunt search failed: {str(e)}") print(f"ProductHunt search failed: {str(e)}") if progress is not None: progress(0.4, "Processing GitHub data") await asyncio.sleep(20) try: print("\n=== CALLING GITHUB AGENT ===") github_result = await fetch_with_timeout( agents['github_agent']['agent'].search_and_analyze( processed_prompt, CONFIG['github_settings']['search']['final_analysis_count'], agents['github_agent']['model'] ) ) print(f"GitHub result received: {'Empty' if not github_result else 'Has content'}") github_result = str(github_result) tasks_completed += 1 except Exception as e: github_result = f"GitHub Error: {str(e)}" logger.error(f"GitHub search failed: {str(e)}") print(f"GitHub search failed: {str(e)}") if progress is not None: progress(0.6, "Processing Arxiv data") try: arxiv_result = await fetch_with_timeout( agents['arxiv_agent']['agent'].search_and_analyze( processed_prompt, 5, agents['arxiv_agent']['model'] ) ) arxiv_result = str(arxiv_result) tasks_completed += 1 except Exception as e: arxiv_result = f"Arxiv Error: {str(e)}" logger.error(f"Arxiv search failed: {str(e)}") print(f"Arxiv search failed: {str(e)}") if progress is not None: progress(0.8, "Processing Reddit data") try: reddit_result = await fetch_with_timeout( agents['reddit_agent']['agent'].search_and_analyze( processed_prompt, agents['reddit_agent']['model'], num_posts=5 ), timeout=CONFIG['timeouts']['analysis'] ) reddit_result = str(reddit_result) tasks_completed += 1 except Exception as e: reddit_result = f"Reddit Error: {str(e)}" logger.error(f"Reddit search failed: {str(e)}") if progress is not None: progress(0.9, "Generating summary") template_env = jinja2.Environment( loader=jinja2.FileSystemLoader('templates') ) template = template_env.get_template('summary_template.txt') prompt_text = template.render( prompt=processed_prompt, github_data=github_result, arxiv_data=arxiv_result, producthunt_data=producthunt_result, reddit_data=reddit_result ) API_LIMIT_MESSAGE = """ IdeaLens has reached its API query limits but you may still be able to see results from individual platform sections below. Please try again in a few minutes. This temporary pause helps us maintain service quality and ensure fair access for all users. Thank you for your patience! """ try: max_retries = 3 retry_delay = 5 attempt = 0 while attempt < max_retries: try: loop = asyncio.get_event_loop() response = await asyncio.wait_for( loop.run_in_executor( None, lambda: agents['summary_model'].generate_content(prompt_text) ), timeout=CONFIG['timeouts']['analysis'] ) executive_summary = response.text break except asyncio.TimeoutError: print("Summary generation timeout") if attempt < max_retries - 1: print(f"Retry attempt {attempt + 1}/{max_retries}") await asyncio.sleep(retry_delay) retry_delay *= 2 attempt += 1 else: raise Exception("Summary generation timeout") except Exception as e: if "429" in str(e) or attempt < max_retries - 1: print(f"Rate limit or error, attempt {attempt + 1}/{max_retries}. Waiting {retry_delay} seconds.") await asyncio.sleep(retry_delay) retry_delay *= 2 attempt += 1 else: raise else: raise Exception("Max retries exceeded for summary generation") except Exception as e: error_type = str(e) print(f"Summary generation failed: {error_type}") logger.error(f"Summary generation failed: {error_type}") executive_summary = API_LIMIT_MESSAGE if progress is not None: progress(1, "Completed") return (executive_summary, arxiv_result, github_result, producthunt_result, reddit_result) except Exception as e: print(f"Process prompt error: {str(e)}") logger.error(f"Process prompt error: {str(e)}") API_LIMIT_MESSAGE = """ IdeaLens has reached its API query limits. Please try again in a few minutes. This temporary pause helps us maintain service quality and ensure fair access for all users. Thank you for your patience! """ return (API_LIMIT_MESSAGE, "", "", "", "") finally: gc.collect() def start_loading(prompt): """Show loader when search starts""" print("Start loading") if not prompt.strip(): return gr.update(visible=False) return gr.update(visible=True) def create_interface(agents: Dict[str, Any]) -> gr.Blocks: """Create and configure the Gradio interface.""" start_time = datetime.now() print("Debug: Starting create_interface") with gr.Blocks(css=custom_css) as interface: gr.HTML(intro_text) with gr.Row(): input_text = gr.Textbox( lines=2, placeholder="Enter your search prompt here", label="Search Prompt", elem_classes="center-label" ) print("Debug: Input textbox created") # Create the sample prompt buttons with gr.Row(equal_height=True): sample_prompts = [ "A crypto-backed decentralized marketplace for digital assets, enabling trustless peer-to-peer trading and licensing", "A music app that adjusts soundscapes based on relaxation or focus detected via EEG", "An AI app that turns real-world objects into interactive holographic tutorials using augmented reality and real-time object recognition", ] for prompt in sample_prompts: gr.Button(prompt, elem_classes="equal-button").click( lambda p=prompt: p, outputs=input_text ) with gr.Row(): start_button = gr.Button("Start Search", variant="primary") print("Debug: Start button created") with gr.Row(): error_box = gr.Textbox( label="Status/Error Messages", visible=False ) print("Debug: Error box created") loader = gr.Textbox( value="Processing your request... Please wait...", visible=False, label="Status", elem_classes="loading-text" ) with gr.Row(): executive_summary_output = gr.Markdown( label="Executive Summary", show_copy_button=True ) print("Debug: Executive summary textbox created") with gr.Column(visible=False) as output_column: outputs = {} buttons = {} for source in ['reddit', 'producthunt', 'github', 'arxiv']: outputs[source] = gr.Markdown( label=f"{source.title()} Results", visible=False, show_copy_button=True ) if source == "reddit": button_label = "Show User Perspectives (Reddit)" elif source == "producthunt": button_label = "Show Similar Products (Producthunt)" elif source == "arxiv": button_label = "Show Related Research (Arxiv)" elif source == "github": button_label = "Explore Related Code (Github)" else: button_label = f"Show Full {source.title()} Results" buttons[source] = gr.Button(button_label) # Configure button handlers async def handle_search(prompt: str) -> Tuple[str, str, str, str, str, gr.Textbox, gr.Column]: start_time = datetime.now() print(f"Debug: Starting handle_search with prompt: {prompt}") API_LIMIT_MESSAGE = """ IdeaLens has reached its API query limits. Please try again in a few minutes. This temporary pause helps us maintain service quality and ensure fair access for all users. Thank you for your patience! """ try: if not prompt.strip(): print("Debug: Prompt is empty") return "Please enter a search query", "", "", "", "", gr.update(visible=False), gr.update(visible=True) result = await process_prompt(prompt, agents, gr.Progress()) print("Debug: handle_search completed") # Check if any of the results contain error messages if any(isinstance(r, str) and "Error:" in r for r in result[1:]): raise Exception("One or more data sources failed") end_time = datetime.now() print(f"Debug: Finished handle_search, Time taken: {end_time - start_time}") return result[0], result[1], result[2], result[3], result[4], gr.update(visible=False), gr.update(visible=True) except Exception as e: print(f"Handle search error: {str(e)}") logger.error(f"Handle search error: {str(e)}") # Return API limit message for all error scenarios return ( API_LIMIT_MESSAGE, # executive summary "", # arxiv "", # github "", # producthunt "", # reddit gr.update(visible=False), # loader gr.update(visible=True) # output column ) # Wire up the start button handlers start_button.click( fn=start_loading, inputs=[input_text], outputs=[loader], queue=False # Execute immediately ).then( fn=handle_search, inputs=[input_text], outputs=[ executive_summary_output, outputs['arxiv'], outputs['github'], outputs['producthunt'], outputs['reddit'], loader, output_column ], api_name="search", queue=True # This will run after the loader is shown ) # Configure view buttons for source, button in buttons.items(): button.click( lambda: gr.update(visible=True), None, outputs[source] ) print(f"Debug: {source} button click handler configured") return interface def main() -> None: """Main application entry point.""" start_time = datetime.now() print("Debug: Starting main function") try: # Configure environment print("Debug: Configuring environment...") configure_environment() print("Debug: Environment configured successfully") # Initialize agents print("Debug: Initializing agents...") agents = initialize_agents() if not agents: print("Debug: Agent initialization failed") raise RuntimeError("Failed to initialize one or more agents") print("Debug: Agents initialized successfully") # Create and launch interface print("Debug: Creating interface...") interface = create_interface(agents) print("Debug: Interface created successfully") interface.launch(debug=True) print("Debug: Gradio interface launched") except Exception as e: logger.error(f"Application startup failed: {str(e)}") print(f"Debug: Application startup failed: {str(e)}") logger.error(traceback.format_exc()) print(f"Debug: Traceback: {traceback.format_exc()}") raise finally: end_time = datetime.now() print(f"Debug: Exiting main function, Time taken: {end_time - start_time}") if __name__ == "__main__": main()