Upload 10 files

CONFIG.json

@@ -0,0 +1,191 @@
+{
+  "app_settings": {
+    "title": "Multi-Source Research Assistant",
+    "description": "Comprehensive research analysis combining multiple data sources",
+    "version": "1.0.0",
+    "debug_mode": false
+  },
+  "rate_limits": {
+    "github_api": {
+      "calls_per_minute": 30,
+      "max_retries": 3,
+      "retry_delay": 3,
+      "search_rate_limit": 10,
+      "core_rate_limit": 5000,
+      "min_remaining_threshold": 5,
+      "wait_time_multiplier": 1.5
+    },
+    "arxiv_api": {
+      "calls_per_minute": 20,
+      "max_retries": 3,
+      "retry_delay": 10
+    },
+    "producthunt_api": {
+      "calls_per_minute": 25,
+      "max_retries": 3,
+      "retry_delay": 5
+    },
+    "reddit_api": {
+      "global_rate_limit": 60,
+      "model_rate_limit": 45,
+      "search_rate_limit": 45,
+      "content_rate_limit": 60,
+      "max_retries": 3,
+      "retry_delay": 5,
+      "backoff_multiplier": 2,
+      "max_backoff": 300,
+      "rate_limit_buffer": 0.1
+    }
+  },
+  "timeouts": {
+    "api_request": 300,
+    "analysis": 400,
+    "summary_generation": 300,
+    "total_operation": 800,
+    "github": {
+      "search": 30,
+      "repo_analysis": 60,
+      "batch_timeout": 180,
+      "query_preprocessing": 20
+    }
+  },
+  "batch_settings": {
+    "max_batch_size": 10,
+    "processing_delay": 1,
+    "max_items_per_request": 30,
+    "github": {
+      "max_search_results": 20,
+      "max_analyze_per_batch": 5,
+      "min_stars_threshold": 5,
+      "max_description_length": 1000,
+      "relevance_score_threshold": 0.6
+    }
+  },
+  "reddit_settings": {
+    "search": {
+      "max_results_per_subreddit": 5,
+      "posts_per_subreddit": 5,
+      "batch_size": 20,
+      "time_filter": "year",
+      "sort_options": ["relevance", "hot", "top", "new"],
+      "default_sort": "relevance",
+      "syntax": "cloudsearch",
+      "pagination_delay": 1
+    },
+    "resource_limits": {
+      "max_concurrent_searches": 4,
+      "max_retries_per_request": 3,
+      "request_timeout": 25
+    },
+    "content_processing": {
+      "max_title_length": 300,
+      "max_content_length": 40000,
+      "content_chunk_size": 2000
+    }
+  },
+  "producthunt_settings": {
+    "search": {
+      "max_results": 5,
+      "max_batch_size": 10,
+      "keyword_match_count": 20,
+      "weights": {
+        "name_weight": 1.0,
+        "tagline_weight": 3.0
+      }
+    },
+    "api": {
+      "max_retries": 3,
+      "retry_delay": 5,
+      "request_delay": 1
+    },
+    "semantic_analysis": {
+      "batch_size": 5,
+      "timeout": 60,
+      "min_score_threshold": 0.3
+    }
+  },
+  "cache": {
+    "enabled": true,
+    "max_size": 2000,
+    "ttl": 7200,
+    "cleanup_interval": 600
+  },
+  "memory_management": {
+    "gc_collect_frequency": 10,
+    "max_retries": 3,
+    "cleanup_on_error": true,
+    "batch_gc_threshold": 5
+  },
+  "github_settings": {
+    "search": {
+      "min_query_length": 3,
+      "max_query_length": 256,
+      "default_order": "desc",
+      "fallback_enabled": true,
+      "languages_priority": ["python", "javascript", "typescript", "java", "go"],
+      "exclude_archived": true,
+      "exclude_forks": true,
+      "initial_result_count": 20,
+      "relevance_scoring_count": 10,
+      "final_analysis_count": 5,
+      "default_sort": "best-match",
+      "preprocessing": {
+        "min_keyword_length": 3,
+        "max_keywords": 4,
+        "gemini_enhancement": {
+          "temperature": 0.1,
+          "top_k": 10,
+          "top_p": 0.5,
+          "max_retries": 3
+        },
+        "nlp_extraction": {
+          "min_relevance_score": 0.6,
+          "max_tokens_per_keyword": 20,
+          "pos_weights": {
+            "NOUN": 1.0,
+            "VERB": 0.8,
+            "ADJ": 0.6
+          }
+        }
+      }
+    },
+    "analysis": {
+      "relevance_weights": {
+        "stars": 2.0,
+        "recency": 1.5,
+        "description_similarity": 8.0,
+        "topics_match": 0.3
+      },
+      "max_analysis_retries": 2,
+      "max_concurrent_analyses": 3
+    }
+  },
+  "allowed_subreddits": [
+    "InternetIsBeautiful",
+    "Indiewebdev",
+    "SideProject",
+    "Productivity",
+    "HuggingFace",
+    "ArtificialInteligence",
+    "MachineLearning",
+    "deeplearning",
+    "webdev",
+    "Futurology",
+    "ProductHunters",
+    "coolgithubprojects",
+    "github"
+  ],
+  "error_handling": {
+    "max_retries": 3,
+    "retry_delay_base": 2,
+    "retry_delay_max": 10,
+    "fail_fast": false,
+    "retry_status_codes": [429, 500, 502, 503, 504]
+  },
+  "ui_settings": {
+    "theme": "default",
+    "max_input_length": 1000,
+    "refresh_interval": 5000,
+    "show_progress": true
+  }
+}

app.py

@@ -0,0 +1,841 @@
+import os
+import gradio as gr
+import sys
+from sentence_transformers import SentenceTransformer
+import torch
+import torch.nn.functional as F
+from functools import lru_cache
+# Debug print: Check current working directory
+import os
+import subprocess
+import sys
+def install_private_repo():
+    github_agent_token = os.getenv('GITHUB_AGENT_TOKEN')
+    if not github_agent_token:
+        print("Error: GITHUB_AGENT_TOKEN environment variable is not set.")
+        return False
+
+    repo_url = f"git+https://{github_agent_token}@github.com/punekichikki/ideaLensAgent.git"
+    
+    try:
+        #print(f"Attempting to install from private repo: {repo_url.replace(github_agent_token, '*******')}")
+        
+        # Install with verbose output
+        result = subprocess.run(
+            [sys.executable, "-m", "pip", "install", "-v", repo_url],
+            capture_output=True,
+            text=True
+        )
+        
+        if result.returncode != 0:
+            print(f"Installation failed with error code: {result.returncode}")
+            print(f"stdout: {result.stdout}")
+            print(f"stderr: {result.stderr}")
+            return False
+            
+        #print("Installation output:")
+        #print(result.stdout)
+        
+        # Check installed packages
+        print("\nInstalled packages:")
+        pip_list = subprocess.run(
+            [sys.executable, "-m", "pip", "list"],
+            capture_output=True,
+            text=True
+        )
+        #print(pip_list.stdout)
+        
+        # Check Python path
+        #print("\nPython path:")
+        #print(sys.path)
+        
+        # Try to find the package location
+        find_package = subprocess.run(
+            [sys.executable, "-c", "import idealens_agents; print(idealens_agents.__file__)"],
+            capture_output=True,
+            text=True
+        )
+        #print("\nPackage location attempt:")
+        #print("stdout:", find_package.stdout)
+        #print("stderr:", find_package.stderr)
+        
+        return True
+
+    except Exception as e:
+        print(f"Error: An unexpected error occurred: {str(e)}")
+        traceback.print_exc()
+        return False
+
+# Add debug information before installation
+#print("Current environment variables:", {k: v for k, v in os.environ.items() if 'TOKEN' in k})
+#print("Python executable:", sys.executable)
+#print("Python version:", sys.version)
+#print("Current working directory:", os.getcwd())
+#print("Directory contents:", os.listdir())
+
+# Install private repo
+install_success = install_private_repo()
+
+if not install_success:
+    print("Failed to install private repository. Exiting.")
+    sys.exit(1)
+agents_dir = os.path.join(os.path.dirname(__file__), 'agents')
+sys.path.append(agents_dir)
+# Try importing with more detailed error handling
+from agents import GitHubAgent
+from agents import ArxivSearchAgent
+from agents import ProductHuntAgent
+from agents import RedditAgent
+from vertexai.generative_models import GenerativeModel
+import vertexai
+import asyncio
+import json
+import logging.config
+from typing import Dict, Any, Optional, Tuple
+import traceback
+from datetime import datetime
+import gc
+import jinja2
+
+custom_css = """
+.center-label {
+    display: flex;
+    flex-direction: column; /* Makes the label stack above the input*/
+    align-items: center; /* Horizontally center the contents*/
+    text-align: center;
+}
+
+.center-label .form {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    width: 100%;
+}
+
+
+.center-label .label {
+    text-align: center;
+    width: 100%;
+}
+
+.equal-button {
+       flex: 1; /* Makes the buttons share available space equally */
+        margin: 5px;
+        background-color: #f0f0f0;
+        font-size: 12px;
+    }
+.loading-text textarea {
+    text-align: center !important;
+    font-weight: bold !important;
+    color: #e67e22 !important;
+    background-color: #f7f7f7 !important;
+}
+
+"""
+intro_text = """
+  <div style="padding: 20px; border: 1px solid #e0e0e0; border-radius: 8px; margin-bottom: 20px;">
+    <h2 style="text-align:center; margin-bottom: 10px;">Meet IdeaLens: the AI Agent for your product ideas</h2>
+    <div style="font-family: Arial, sans-serif; line-height: 1.6; color: #333; margin: 10px;">
+        <div style="display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 20px; position: relative;">
+            <div style="flex: 1; margin-right: 20px; padding-right: 20px; border-right: 1px solid #e0e0e0;">
+                <h2 style="margin: 0; font-size: 20px; font-weight: bold;">❓ Got an idea that could change the world? 🌍</h2>
+                <div style="margin-top: 10px; font-size: 13px;">
+                    💡 <span style="font-weight: bold; color: #0073e6;">IdeaLens</span> intelligently analyzes data to chart your idea's potential<br>
+                    🌟 It explores user perspectives and predicts market reactions<br>
+                    📊 It identifies competitors and uncovers technical resources to refine your concept<br>
+                    📚 By integrating cutting-edge academic insights, <span style="font-weight: bold; color: #0073e6;">IdeaLens</span> ensures thorough validation<br>
+                    🚀 With <span style="font-weight: bold; color: #0073e6;">IdeaLens</span> assisting you, success is just an idea away!
+                </div>
+            </div>
+            <div style="flex: 1; margin-left: 20px;">
+                <h2 style="margin: 0; font-size: 20px;">How IdeaLens Assists:</h2>
+                <div style="margin-top: 10px; font-size: 13px;">
+                    🧠 IdeaLens reveals hidden insights by connecting patterns across platforms like <span style="font-weight: bold; color: #0073e6;">Reddit</span>, 
+                    <span style="font-weight: bold; color: #0073e6;">ProductHunt</span>, 
+                    <span style="font-weight: bold; color: #0073e6;">GitHub</span>, and 
+                    <span style="font-weight: bold; color: #0073e6;">arXiv</span>.<br>
+                    📋 It synthesizes competitive insights and surface critical technical resources to strengthen your vision<br>
+                    🔍 Want to go deeper? IdeaLens provides comprehensive strategic intelligence from each platform<br>
+                    ⏳ In just <span style="font-weight: bold; color: #0073e6;">5 minutes</span>, IdeaLens delivers focused, actionable guidance
+                </div>
+            </div>
+        </div>
+    
+        <div style="font-weight: bold; margin-top: 20px;">
+            🔥 Ready to let IdeaLens assist you?<br>
+            👉 Enter your idea in the search prompt or try one of the curated examples below! 🎯✨
+        </div>
+    </div>
+</div>
+
+"""
+# Debug print: Initial imports complete
+print("Debug: Initial imports complete")
+
+# Create logs directory if it doesn't exist
+os.makedirs(os.path.join(os.path.dirname(__file__), 'logs'), exist_ok=True)
+print(f"Debug: Logs directory created or exists at {os.path.join(os.path.dirname(__file__), 'logs')}")
+
+
+
+print("Debug: Agents imported")
+
+# Load configuration
+print("Debug: Loading configuration...")
+with open('CONFIG.json') as f:
+    CONFIG = json.load(f)
+print("Debug: Configuration loaded successfully")
+
+# Set up logging
+print("Debug: Setting up logging...")
+logging.config.fileConfig('logging.conf')
+logger = logging.getLogger('app')
+print("Debug: Logging setup complete")
+
+
+class QueryPreprocessor:
+    def __init__(self):
+        self._model = None
+        print("Debug: QueryPreprocessor initialized") # Add print statement here
+
+    @property
+    @lru_cache()
+    def model(self) -> SentenceTransformer:
+
+        if self._model is None:
+            print("Debug: Loading sentence transformer model...")
+            self._model = SentenceTransformer('all-MiniLM-L6-v2')
+            logger.info("Sentence transformer model initialized")
+            print("Debug: Sentence transformer model loaded successfully")
+        return self._model
+        
+    async def extract_core_concepts(self, text: str) -> str:
+        """Extract core concepts from text, removing structural elements"""
+        markers = ["Market analysis request:", "Target sector:",
+                   "Primary features:", "Business model category:",
+                   "Technical requirements:"]
+        cleaned = text
+        for marker in markers:
+            cleaned = cleaned.replace(marker, "")
+        return cleaned.strip()
+
+    async def check_semantic_similarity(self, original: str, processed: str,
+                                        threshold: float = 0.7) -> bool:
+        try:
+            # Extract core concepts from processed text
+            processed_core = await self.extract_core_concepts(processed)
+
+            # Run embedding computation in thread pool
+            loop = asyncio.get_event_loop()
+            embeddings = await loop.run_in_executor(
+                None,
+                lambda: (
+                    self.model.encode(original, convert_to_tensor=True),
+                    self.model.encode(processed_core, convert_to_tensor=True)
+                )
+            )
+            original_embedding, processed_embedding = embeddings
+
+            # Calculate cosine similarity
+            similarity = F.cosine_similarity(
+                original_embedding.unsqueeze(0),
+                processed_embedding.unsqueeze(0)
+            ).item()
+
+            logger.info(f"Original query: {original}")
+            logger.info(f"Processed core concepts: {processed_core}")
+            logger.info(f"Semantic similarity: {similarity:.3f}")
+
+            return similarity > threshold
+
+        except Exception as e:
+            logger.error(f"Error in semantic similarity check: {str(e)}")
+            return False
+
+
+def configure_environment():
+    start_time = datetime.now()
+    print("Debug: Starting configure_environment")
+    required_env_vars = [
+        'GITHUB_TOKEN',
+        'PRODUCT_HUNT_TOKEN',
+        'REDDIT_CLIENT_ID',
+        'REDDIT_CLIENT_SECRET',
+        'REDDIT_USER_AGENT',
+        'GOOGLE_APPLICATION_CREDENTIALS_JSON',
+        'GITHUB_APPLICATION_CREDENTIALS_JSON',
+        'ARXIV_APPLICATION_CREDENTIALS_JSON',
+        'PRODUCT_HUNT_APPLICATION_CREDENTIALS_JSON',
+        'REDDIT_APPLICATION_CREDENTIALS_JSON',
+        'GITHUB_CLOUD_PROJECT',
+        'ARXIV_CLOUD_PROJECT',
+        'PRODUCTHUNT_CLOUD_PROJECT',
+        'REDDIT_CLOUD_PROJECT'
+    ]
+    print(f"Debug: Required environment variables: {required_env_vars}")
+
+    missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+    if missing_vars:
+        error_message = f"Missing required environment variables: {', '.join(missing_vars)}. " \
+                        f"Please check .env.example for required variables."
+        print(f"Debug: Error - {error_message}")
+        raise EnvironmentError(error_message)
+    print("Debug: All required environment variables are present")
+
+    # Set up Google credentials from the JSON stored in env variable
+    if 'GOOGLE_APPLICATION_CREDENTIALS_JSON' in os.environ:
+        print("Debug: Found GOOGLE_APPLICATION_CREDENTIALS_JSON")
+        creds_json = os.environ['GOOGLE_APPLICATION_CREDENTIALS_JSON']
+        with open('/tmp/google_credentials.json', 'w') as f:
+            f.write(creds_json)
+        os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/tmp/google_credentials.json'
+        print("Debug: Google credentials file created and GOOGLE_APPLICATION_CREDENTIALS set")
+
+    # Set up Github credentials from the JSON stored in env variable
+    if 'GITHUB_APPLICATION_CREDENTIALS_JSON' in os.environ:
+        print("Debug: Found GITHUB_APPLICATION_CREDENTIALS_JSON")
+        creds_json = os.environ['GITHUB_APPLICATION_CREDENTIALS_JSON']
+        with open('/tmp/github_credentials.json', 'w') as f:
+            f.write(creds_json)
+        os.environ['GITHUB_APPLICATION_CREDENTIALS'] = '/tmp/github_credentials.json'
+        print("Debug: Github credentials file created and GITHUB_APPLICATION_CREDENTIALS set")
+
+    # Set up Arxiv credentials from the JSON stored in env variable
+    if 'ARXIV_APPLICATION_CREDENTIALS_JSON' in os.environ:
+        print("Debug: Found ARXIV_APPLICATION_CREDENTIALS_JSON")
+        creds_json = os.environ['ARXIV_APPLICATION_CREDENTIALS_JSON']
+        with open('/tmp/arxiv_credentials.json', 'w') as f:
+            f.write(creds_json)
+        os.environ['ARXIV_APPLICATION_CREDENTIALS'] = '/tmp/arxiv_credentials.json'
+        print("Debug: Arxiv credentials file created and ARXIV_APPLICATION_CREDENTIALS set")
+
+    # Set up Product Hunt credentials from the JSON stored in env variable
+    if 'PRODUCTHUNT_APPLICATION_CREDENTIALS_JSON' in os.environ:
+        print("Debug: Found PRODUCTHUNT_APPLICATION_CREDENTIALS_JSON")
+        creds_json = os.environ['PRODUCTHUNT_APPLICATION_CREDENTIALS_JSON']
+        with open('/tmp/producthunt_credentials.json', 'w') as f:
+            f.write(creds_json)
+        os.environ['PRODUCTHUNT_APPLICATION_CREDENTIALS'] = '/tmp/producthunt_credentials.json'
+        print("Debug: Product Hunt credentials file created and PRODUCTHUNT_APPLICATION_CREDENTIALS set")
+
+    # Set up Reddit credentials from the JSON stored in env variable
+    if 'REDDIT_APPLICATION_CREDENTIALS_JSON' in os.environ:
+        print("Debug: Found REDDIT_APPLICATION_CREDENTIALS_JSON")
+        creds_json = os.environ['REDDIT_APPLICATION_CREDENTIALS_JSON']
+        with open('/tmp/reddit_credentials.json', 'w') as f:
+            f.write(creds_json)
+        os.environ['REDDIT_APPLICATION_CREDENTIALS'] = '/tmp/reddit_credentials.json'
+        print("Debug: Reddit credentials file created and REDDIT_APPLICATION_CREDENTIALS set")
+    end_time = datetime.now()
+    print(f"Debug: Finished configure_environment, Time taken: {end_time - start_time}")
+
+
+def initialize_agents() -> Optional[Dict[str, Any]]:
+    """Initialize all search agents with proper error handling."""
+    start_time = datetime.now()
+    print("Debug: Starting initialize_agents")
+    try:
+        print("Debug: Initializing Vertex AI...")
+        logger.info("Initializing agents...")
+        vertexai.init(project=None, location="us-central1")
+        print("Debug: Vertex AI initialized successfully")
+        summary_model = GenerativeModel("gemini-pro")
+        print("Debug: Summary model initialized")
+
+        # Initialize agents
+        print("Debug: Initializing agents...")
+        github_agent = GitHubAgent(
+            project_id=os.getenv('GITHUB_CLOUD_PROJECT'),
+            credentials_path='/tmp/github_credentials.json'
+        )
+        print("Debug: GitHub agent initialized")
+
+        arxiv_agent = ArxivSearchAgent(
+            project_id=os.getenv('ARXIV_CLOUD_PROJECT'),
+            credentials_path='/tmp/arxiv_credentials.json'
+        )
+        print("Debug: Arxiv agent initialized")
+
+        producthunt_agent = ProductHuntAgent(
+            project_id=os.getenv('PRODUCTHUNT_CLOUD_PROJECT'),
+            credentials_path='/tmp/producthunt_credentials.json'
+        )
+        print("Debug: Product Hunt agent initialized")
+
+        reddit_agent = RedditAgent(
+            project_id=os.getenv('REDDIT_CLOUD_PROJECT'),
+            credentials_path='/tmp/reddit_credentials.json'
+        )
+        print("Debug: Reddit agent initialized")
+
+        agents_dict = {
+            'summary_model': summary_model,
+            'github_agent': {'agent': github_agent, 'model': summary_model,'config': CONFIG.get('github_settings', {})},
+            'arxiv_agent': {'agent': arxiv_agent, 'model': summary_model},
+            'producthunt_agent': {
+                'agent': producthunt_agent, 
+                'model': summary_model,
+                'config': CONFIG.get('producthunt_settings', {})  # Pass ProductHunt specific config
+            },
+            'reddit_agent': {'agent': reddit_agent, 'model': summary_model}
+        }
+        print("Debug: Agents initialized successfully.")
+        end_time = datetime.now()
+        print(f"Debug: Finished initialize_agents, Time taken: {end_time - start_time}")
+        return agents_dict
+    except Exception as e:
+        logger.error(f"Error initializing agents: {str(e)}")
+        print(f"Debug: Error initializing agents: {str(e)}")
+        logger.error(traceback.format_exc())
+        print(f"Debug: Traceback: {traceback.format_exc()}")
+        return None
+
+
+async def fetch_with_timeout(coro: Any, timeout: int = 600) -> Any:
+    """Wrapper for async operations with timeout."""
+    start_time = datetime.now()
+    print(f"Debug: Starting fetch_with_timeout with timeout: {timeout}")
+    try:
+        result = await asyncio.wait_for(coro, timeout=timeout)
+        print("Debug: fetch_with_timeout completed successfully")
+        end_time = datetime.now()
+        print(f"Debug: Finished fetch_with_timeout, Time taken: {end_time - start_time}")
+        return result
+    except asyncio.TimeoutError:
+        logger.error(f"Task timed out: {coro}")
+        print(f"Debug: Task timed out: {coro}")
+        return "Task timed out"
+
+
+async def process_prompt(prompt: str, agents: Dict[str, Any],
+                         progress: Optional[gr.Progress] = None) -> Tuple[str, str, str, str, str]:
+    try:
+        print(f"\n=== APP.PY PROCESSING START ===")
+        print(f"Original prompt received: {prompt}")
+
+        if progress is not None:
+            progress(0, "Starting preprocessing")
+
+        # Initialize task tracking
+        tasks_completed = 0  # Initialize here
+        total_tasks = 4  # Total number of major tasks
+
+        preprocessing_prompt = f"""
+        Transform this business/app idea query into structured market research format.
+
+        Original query: {prompt}
+
+        Transform using these rules:
+        1. Begin with "Market analysis request:"
+        2. Include "Target sector:"
+        3. Specify "Primary features:"
+        4. Add "Business model category:"
+        5. End with "Technical requirements:"
+
+        Format as a single paragraph without the rule headers. Use professional business language.
+        """
+        print(f"Generated preprocessing prompt: {preprocessing_prompt}")
+
+        # Initialize preprocessor
+        preprocessor = QueryPreprocessor()
+
+        try:
+            # First pass - structure the query
+            response = await fetch_with_timeout(
+                asyncio.to_thread(
+                    lambda: agents['summary_model'].generate_content(preprocessing_prompt)
+                )
+            )
+            structured_prompt = response.text.strip()
+            print(f"After first pass structuring: {structured_prompt}")
+
+            # Second pass - format for API efficiency
+            format_prompt = f"""
+            Convert this market analysis into a concise, direct query suitable for API processing.
+            Keep all key details but remove unnecessary words.
+            Query: {structured_prompt}
+            """
+
+            format_response = await fetch_with_timeout(
+                asyncio.to_thread(
+                    lambda: agents['summary_model'].generate_content(format_prompt)
+                )
+            )
+            processed_prompt = format_response.text.strip()
+            print(f"After second pass formatting: {processed_prompt}")
+
+            # Check semantic similarity with core concept extraction
+            is_similar = await preprocessor.check_semantic_similarity(prompt, processed_prompt)
+            print(f"Semantic similarity check result: {is_similar}")
+
+            if not processed_prompt or not is_similar:
+                logger.warning("Query preprocessing validation failed, using original query")
+                logger.info(f"Original: {prompt}")
+                logger.info(f"Processed: {processed_prompt}")
+                processed_prompt = prompt
+
+        except Exception as e:
+            logger.error(f"Query preprocessing failed: {str(e)}")
+            processed_prompt = prompt
+
+        logger.info(f"Original prompt: {prompt}")
+        logger.info(f"Processed prompt: {processed_prompt}")
+
+        # Sequential execution with individual error handling
+        if progress is not None:
+            progress(0.2, "Processing ProductHunt data")
+
+        try:
+            print("\n=== CALLING PRODUCTHUNT AGENT ===")
+            producthunt_result = await fetch_with_timeout(
+                agents['producthunt_agent']['agent'].process_search(
+                    agents['producthunt_agent']['model'],
+                    processed_prompt
+                ),
+                timeout=CONFIG['timeouts']['analysis']
+            )
+            print(f"ProductHunt result received: {'Empty' if not producthunt_result else 'Has content'}")
+            producthunt_result = str(producthunt_result)
+            tasks_completed += 1
+            await asyncio.sleep(5)
+            gc.collect()     
+        except Exception as e:
+            producthunt_result = f"ProductHunt Error: {str(e)}"
+            logger.error(f"ProductHunt search failed: {str(e)}")
+            print(f"ProductHunt search failed: {str(e)}")
+
+        if progress is not None:
+            progress(0.4, "Processing GitHub data")
+        await asyncio.sleep(20)
+        try:
+            print("\n=== CALLING GITHUB AGENT ===")
+            github_result = await fetch_with_timeout(
+                agents['github_agent']['agent'].search_and_analyze(
+                    processed_prompt,
+                    CONFIG['github_settings']['search']['final_analysis_count'],
+                    agents['github_agent']['model']
+                )
+            )
+            print(f"GitHub result received: {'Empty' if not github_result else 'Has content'}")
+            github_result = str(github_result)
+            tasks_completed += 1
+        except Exception as e:
+            github_result = f"GitHub Error: {str(e)}"
+            logger.error(f"GitHub search failed: {str(e)}")
+            print(f"GitHub search failed: {str(e)}")
+
+        if progress is not None:
+            progress(0.6, "Processing Arxiv data")
+
+        try:
+            arxiv_result = await fetch_with_timeout(
+                agents['arxiv_agent']['agent'].search_and_analyze(
+                    processed_prompt, 5, agents['arxiv_agent']['model']
+                )
+            )
+            arxiv_result = str(arxiv_result)
+            tasks_completed += 1
+        except Exception as e:
+            arxiv_result = f"Arxiv Error: {str(e)}"
+            logger.error(f"Arxiv search failed: {str(e)}")
+            print(f"Arxiv search failed: {str(e)}")
+
+        if progress is not None:
+            progress(0.8, "Processing Reddit data")
+
+        try:
+            reddit_result = await fetch_with_timeout(
+                agents['reddit_agent']['agent'].search_and_analyze(
+                    processed_prompt, 
+                    agents['reddit_agent']['model'],
+                    num_posts=5
+                ),
+                timeout=CONFIG['timeouts']['analysis']
+            )
+            reddit_result = str(reddit_result)
+            tasks_completed += 1
+        except Exception as e:
+            reddit_result = f"Reddit Error: {str(e)}"
+            logger.error(f"Reddit search failed: {str(e)}")
+
+        if progress is not None:
+            progress(0.9, "Generating summary")
+
+        template_env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader('templates')
+        )
+        template = template_env.get_template('summary_template.txt')
+
+        prompt_text = template.render(
+            prompt=processed_prompt,
+            github_data=github_result,
+            arxiv_data=arxiv_result,
+            producthunt_data=producthunt_result,
+            reddit_data=reddit_result
+        )
+
+        API_LIMIT_MESSAGE = """
+        IdeaLens has reached its API query limits but you may still be able to see results from individual platform sections below. Please try again in a few minutes.
+        
+        This temporary pause helps us maintain service quality and ensure fair access for all users.
+        Thank you for your patience!
+        """
+
+        try:
+            max_retries = 3
+            retry_delay = 5
+            attempt = 0
+
+            while attempt < max_retries:
+                try:
+                    loop = asyncio.get_event_loop()
+                    response = await asyncio.wait_for(
+                        loop.run_in_executor(
+                            None,
+                            lambda: agents['summary_model'].generate_content(prompt_text)
+                        ),
+                        timeout=CONFIG['timeouts']['analysis']
+                    )
+                    executive_summary = response.text
+                    break
+
+                except asyncio.TimeoutError:
+                    print("Summary generation timeout")
+                    if attempt < max_retries - 1:
+                        print(f"Retry attempt {attempt + 1}/{max_retries}")
+                        await asyncio.sleep(retry_delay)
+                        retry_delay *= 2
+                        attempt += 1
+                    else:
+                        raise Exception("Summary generation timeout")
+
+                except Exception as e:
+                    if "429" in str(e) or attempt < max_retries - 1:
+                        print(f"Rate limit or error, attempt {attempt + 1}/{max_retries}. Waiting {retry_delay} seconds.")
+                        await asyncio.sleep(retry_delay)
+                        retry_delay *= 2
+                        attempt += 1
+                    else:
+                        raise
+
+            else:
+                raise Exception("Max retries exceeded for summary generation")
+
+        except Exception as e:
+            error_type = str(e)
+            print(f"Summary generation failed: {error_type}")
+            logger.error(f"Summary generation failed: {error_type}")
+            executive_summary = API_LIMIT_MESSAGE
+
+        if progress is not None:
+            progress(1, "Completed")
+
+        return (executive_summary, arxiv_result, github_result, producthunt_result, reddit_result)
+
+    except Exception as e:
+        print(f"Process prompt error: {str(e)}")
+        logger.error(f"Process prompt error: {str(e)}")
+        API_LIMIT_MESSAGE = """
+        IdeaLens has reached its API query limits. Please try again in a few minutes.
+        
+        This temporary pause helps us maintain service quality and ensure fair access for all users.
+        Thank you for your patience!
+        """
+        return (API_LIMIT_MESSAGE, "", "", "", "")
+    finally:
+        gc.collect()
+
+def start_loading(prompt):
+    """Show loader when search starts"""
+    print("Start loading")
+    if not prompt.strip():
+        return gr.update(visible=False)
+    return gr.update(visible=True)
+
+def create_interface(agents: Dict[str, Any]) -> gr.Blocks:
+    """Create and configure the Gradio interface."""
+    start_time = datetime.now()
+    print("Debug: Starting create_interface")
+    with gr.Blocks(css=custom_css) as interface:
+        gr.HTML(intro_text)
+        with gr.Row():
+            input_text = gr.Textbox(
+                lines=2,
+                placeholder="Enter your search prompt here",
+                label="Search Prompt",
+                elem_classes="center-label"
+            )
+            print("Debug: Input textbox created")
+            
+        # Create the sample prompt buttons
+        with gr.Row(equal_height=True):
+            sample_prompts = [
+                "A crypto-backed decentralized marketplace for digital assets, enabling trustless peer-to-peer trading and licensing",
+                "A music app that adjusts soundscapes based on relaxation or focus detected via EEG",
+                "An AI app that turns real-world objects into interactive holographic tutorials using augmented reality and real-time object recognition",
+            ]
+            for prompt in sample_prompts:
+                gr.Button(prompt, elem_classes="equal-button").click(
+                    lambda p=prompt: p,
+                    outputs=input_text
+                )
+
+        with gr.Row():
+            start_button = gr.Button("Start Search", variant="primary")
+            print("Debug: Start button created")
+
+        with gr.Row():
+            error_box = gr.Textbox(
+                label="Status/Error Messages",
+                visible=False
+            )
+            print("Debug: Error box created")
+            
+        loader = gr.Textbox(
+            value="Processing your request... Please wait...",
+            visible=False,
+            label="Status",
+            elem_classes="loading-text"
+        )
+        
+        with gr.Row():
+            executive_summary_output = gr.Markdown(
+                label="Executive Summary",
+                show_copy_button=True
+            )
+            print("Debug: Executive summary textbox created")
+
+        with gr.Column(visible=False) as output_column:
+            outputs = {}
+            buttons = {}
+            for source in ['reddit', 'producthunt', 'github', 'arxiv']:
+                outputs[source] = gr.Markdown(
+                    label=f"{source.title()} Results",
+                    visible=False,
+                    show_copy_button=True
+                )
+                if source == "reddit":
+                    button_label = "Show User Perspectives (Reddit)"
+                elif source == "producthunt":
+                    button_label = "Show Similar Products (Producthunt)"
+                elif source == "arxiv":
+                    button_label = "Show Related Research (Arxiv)"
+                elif source == "github":
+                    button_label = "Explore Related Code (Github)"
+                else:
+                    button_label = f"Show Full {source.title()} Results"
+                buttons[source] = gr.Button(button_label)
+
+        # Configure button handlers
+        async def handle_search(prompt: str) -> Tuple[str, str, str, str, str, gr.Textbox, gr.Column]:
+            start_time = datetime.now()
+            print(f"Debug: Starting handle_search with prompt: {prompt}")
+            
+            API_LIMIT_MESSAGE = """
+            IdeaLens has reached its API query limits. Please try again in a few minutes.
+            
+            This temporary pause helps us maintain service quality and ensure fair access for all users.
+            Thank you for your patience!
+            """
+            
+            try:
+                if not prompt.strip():
+                    print("Debug: Prompt is empty")
+                    return "Please enter a search query", "", "", "", "", gr.update(visible=False), gr.update(visible=True)
+                
+                result = await process_prompt(prompt, agents, gr.Progress())
+                print("Debug: handle_search completed")
+                
+                # Check if any of the results contain error messages
+                if any(isinstance(r, str) and "Error:" in r for r in result[1:]):
+                    raise Exception("One or more data sources failed")
+                
+                end_time = datetime.now()
+                print(f"Debug: Finished handle_search, Time taken: {end_time - start_time}")
+                return result[0], result[1], result[2], result[3], result[4], gr.update(visible=False), gr.update(visible=True)
+                
+            except Exception as e:
+                print(f"Handle search error: {str(e)}")
+                logger.error(f"Handle search error: {str(e)}")
+                # Return API limit message for all error scenarios
+                return (
+                    API_LIMIT_MESSAGE,  # executive summary
+                    "",  # arxiv
+                    "",  # github
+                    "",  # producthunt
+                    "",  # reddit
+                    gr.update(visible=False),  # loader
+                    gr.update(visible=True)  # output column
+                )
+        
+        # Wire up the start button handlers
+        start_button.click(
+            fn=start_loading,
+            inputs=[input_text],
+            outputs=[loader],
+            queue=False  # Execute immediately
+        ).then(
+            fn=handle_search,
+            inputs=[input_text],
+            outputs=[
+                executive_summary_output,
+                outputs['arxiv'],
+                outputs['github'],
+                outputs['producthunt'],
+                outputs['reddit'],
+                loader,
+                output_column
+            ],
+            api_name="search",
+            queue=True  # This will run after the loader is shown
+        )
+
+        # Configure view buttons
+        for source, button in buttons.items():
+            button.click(
+                lambda: gr.update(visible=True),
+                None,
+                outputs[source]
+            )
+            print(f"Debug: {source} button click handler configured")
+
+        return interface
+
+def main() -> None:
+    """Main application entry point."""
+    start_time = datetime.now()
+    print("Debug: Starting main function")
+    try:
+        # Configure environment
+        print("Debug: Configuring environment...")
+        configure_environment()
+        print("Debug: Environment configured successfully")
+
+        # Initialize agents
+        print("Debug: Initializing agents...")
+        agents = initialize_agents()
+        if not agents:
+            print("Debug: Agent initialization failed")
+            raise RuntimeError("Failed to initialize one or more agents")
+        print("Debug: Agents initialized successfully")
+
+        # Create and launch interface
+        print("Debug: Creating interface...")
+        interface = create_interface(agents)
+        print("Debug: Interface created successfully")
+        interface.launch(debug=True)
+        print("Debug: Gradio interface launched")
+
+    except Exception as e:
+        logger.error(f"Application startup failed: {str(e)}")
+        print(f"Debug: Application startup failed: {str(e)}")
+        logger.error(traceback.format_exc())
+        print(f"Debug: Traceback: {traceback.format_exc()}")
+        raise
+    finally:
+        end_time = datetime.now()
+        print(f"Debug: Exiting main function, Time taken: {end_time - start_time}")
+
+
+if __name__ == "__main__":
+    main()

dockerignore

@@ -0,0 +1,60 @@
+# Version control
+.git
+.gitignore
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+ENV/
+
+# Logs
+*.log
+logs/
+log/
+
+# Local development
+.env
+.env.local
+.env.*.local
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Test files
+tests/
+test/
+testing/
+
+# Documentation
+docs/
+*.md
+
+# Temporary files
+*.tmp
+*.temp
+.DS_Store
+Thumbs.db

env.example

@@ -0,0 +1,19 @@
+# GitHub Configuration
+GITHUB_TOKEN=your_github_token_here
+
+# ProductHunt Configuration
+PRODUCT_HUNT_TOKEN=your_producthunt_token_here
+
+# Reddit Configuration
+REDDIT_CLIENT_ID=your_reddit_client_id_here
+REDDIT_CLIENT_SECRET=your_reddit_client_secret_here
+REDDIT_USER_AGENT=your_reddit_user_agent_here
+
+# Google Cloud Configuration
+GOOGLE_APPLICATION_CREDENTIALS_JSON=your_google_credentials_json_here
+
+# Application Settings
+DEBUG=false
+LOG_LEVEL=INFO
+MAX_BATCH_SIZE=10
+REQUEST_TIMEOUT=30

gitignore

@@ -0,0 +1,48 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+ENV/
+
+# Logs
+*.log
+logs/
+log/
+
+# Local development
+.env
+.env.local
+.env.*.local
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Operating System
+.DS_Store
+Thumbs.db
+
+# Private folders
+agents/

logging.conf

@@ -0,0 +1,69 @@
+[loggers]
+keys=root,app,githubAgent,arxivAgent,producthuntAgent,redditAgent
+
+[handlers]
+keys=consoleHandler,fileHandler,errorFileHandler
+
+[formatters]
+keys=defaultFormatter,errorFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=root
+
+[logger_app]
+level=INFO
+handlers=fileHandler,errorFileHandler
+qualname=app
+propagate=0
+
+[logger_githubAgent]
+level=INFO
+handlers=fileHandler,errorFileHandler
+qualname=githubAgent
+propagate=0
+
+[logger_arxivAgent]
+level=INFO
+handlers=fileHandler,errorFileHandler
+qualname=arxivAgent
+propagate=0
+
+[logger_producthuntAgent]
+level=INFO
+handlers=fileHandler,errorFileHandler
+qualname=producthuntAgent
+propagate=0
+
+[logger_redditAgent]
+level=INFO
+handlers=fileHandler,errorFileHandler
+qualname=redditAgent
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=defaultFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=handlers.TimedRotatingFileHandler
+level=INFO
+formatter=defaultFormatter
+args=('logs/app.log', 'midnight', 1, 30)
+
+[handler_errorFileHandler]
+class=handlers.TimedRotatingFileHandler
+level=ERROR
+formatter=errorFormatter
+args=('logs/error.log', 'midnight', 1, 30)
+
+[formatter_defaultFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+
+[formatter_errorFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(pathname)s:%(lineno)d
+datefmt=%Y-%m-%d %H:%M:%S

packages.txt

@@ -0,0 +1,7 @@
+python3-dev
+build-essential
+git
+wget
+curl
+software-properties-common
+python3-pip

pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: debug-statements
+      - id: detect-private-key
+      - id: check-added-large-files
+
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+        language_version: python3.10
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-docstrings]

pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "multi-source-research-assistant"
+version = "1.0.0"
+description = "A comprehensive research tool combining multiple data sources"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    {name = "Your Name", email = "your.email@example.com"}
+]
+
+[tool.black]
+line-length = 88
+target-version = ['py310']
+include = '\.pyi?$'
+
+[tool.flake8]
+max-line-length = 88
+extend-ignore = "E203"
+exclude = [
+    ".git",
+    "__pycache__",
+    "build",
+    "dist"
+]
+
+[tool.pytest]
+testpaths = "tests"
+python_files = "test_*.py"
+addopts = "-v -s --cov=agents"

requirements.txt

@@ -0,0 +1,26 @@
+gradio
+praw
+google-generativeai
+vertexai
+google-cloud-aiplatform
+sentence-transformers
+beautifulsoup4
+pandas
+datasets
+google-auth
+spacy
+httpx
+jinja2
+tenacity
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.0/en_core_web_sm-3.7.0.tar.gz
+tqdm
+numpy
+scikit-learn
+feedparser
+requests
+google.cloud.core
+python-dotenv
+torch
+psutil
+nltk
+aiohttp