reranked top 15 + reranking all together

index_retriever.py

@@ -22,12 +22,12 @@ def create_query_engine(vector_index):
         vector_retriever = VectorIndexRetriever(
             index=vector_index, 
             similarity_top_k=30,
-            similarity_cutoff=0.85
+            similarity_cutoff=0.7
         )
         
         hybrid_retriever = QueryFusionRetriever(
             [vector_retriever, bm25_retriever],
-            similarity_top_k=15,
+            similarity_top_k=30,
             num_queries=1
         )
         
@@ -52,37 +52,25 @@ def create_query_engine(vector_index):
 def rerank_nodes(query, nodes, reranker, top_k=15):
     if not nodes or not reranker:
         return nodes[:top_k]
-    
+
     try:
         log_message(f"Переранжирую {len(nodes)} узлов")
-        
-        # Separate tables and images from text nodes
-        table_nodes = [node for node in nodes if node.metadata.get('type') == 'table']
-        image_nodes = [node for node in nodes if node.metadata.get('type') == 'image']
-        text_nodes = [node for node in nodes if node.metadata.get('type', 'text') == 'text']
-        
-        priority_nodes = table_nodes + image_nodes
-        
-        # Rerank only text nodes
-        if text_nodes:
-            pairs = []
-            for node in text_nodes:
-                pairs.append([query, node.text])
-            
-            scores = reranker.predict(pairs)
-            scored_nodes = list(zip(text_nodes, scores))
-            scored_nodes.sort(key=lambda x: x[1], reverse=True)
-            reranked_text_nodes = [node for node, score in scored_nodes]
-        else:
-            reranked_text_nodes = []
-        
-        # Combine: priority nodes first, then reranked text nodes
-        final_nodes = priority_nodes + reranked_text_nodes
-        result = final_nodes[:top_k]
-        
-        log_message(f"Возвращаю {len(priority_nodes)} приоритетных узлов и {len(result) - len(priority_nodes)} текстовых узлов")
+
+        # Prepare pairs for reranker: use node.text for all types
+        pairs = [[query, node.text] for node in nodes]
+
+        # Get scores for all nodes
+        scores = reranker.predict(pairs)
+        scored_nodes = list(zip(nodes, scores))
+
+        # Sort all nodes by score descending
+        scored_nodes.sort(key=lambda x: x[1], reverse=True)
+        reranked_nodes = [node for node, score in scored_nodes]
+
+        result = reranked_nodes[:top_k]
+        log_message(f"Возвращаю топ-{top_k} узлов после совместной переранжировки")
         return result
-        
+
     except Exception as e:
         log_message(f"Ошибка переранжировки: {str(e)}")
         return nodes[:top_k]

utils.py

@@ -105,94 +105,6 @@ def format_context_for_llm(nodes):
     
     return "\n".join(context_parts)
 
-def answer_question(question, query_engine, reranker, current_model, chunks_df=None):
-    if query_engine is None:
-        return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", ""
-    
-    try:
-        log_message(f"Получен вопрос: {question}")
-        start_time = time.time()
-        
-        # Извлечение узлов
-        retrieved_nodes = query_engine.retriever.retrieve(question)
-        log_message(f"Извлечено {len(retrieved_nodes)} узлов")
-        
-        # ДЕТАЛЬНОЕ ЛОГИРОВАНИЕ ИСТОЧНИКОВ
-        log_message("=== ДЕТАЛЬНАЯ ИНФОРМАЦИЯ О НАЙДЕННЫХ УЗЛАХ ===")
-        for i, node in enumerate(retrieved_nodes):
-            log_message(f"Узел {i+1}:")
-            log_message(f"  Документ: {node.metadata.get('document_id', 'unknown')}")
-            log_message(f"  Тип: {node.metadata.get('type', 'unknown')}")
-            log_message(f"  Раздел: {node.metadata.get('section_id', 'unknown')}")
-            log_message(f"  Текст (первые 400 символов): {node.text[:400]}...")
-            log_message(f"  Метаданные: {node.metadata}")
-        
-        # Переранжировка
-        reranked_nodes = rerank_nodes(question, retrieved_nodes, reranker, top_k=10)
-        
-        log_message("=== УЗЛЫ ПОСЛЕ ПЕРЕРАНЖИРОВКИ ===")
-        for i, node in enumerate(reranked_nodes):
-            log_message(f"Переранжированный узел {i+1}:")
-            log_message(f"  Документ: {node.metadata.get('document_id', 'unknown')}")
-            log_message(f"  Тип: {node.metadata.get('type', 'unknown')}")
-            log_message(f"  Раздел: {node.metadata.get('section_id', 'unknown')}")
-            log_message(f"  Полный текст: {node.text}")
-        
-        formatted_context = format_context_for_llm(reranked_nodes)
-        log_message(f"ПОЛНЫЙ КОНТЕКСТ ДЛЯ LLM:\n{formatted_context}")
-        
-        enhanced_question = f"""
-Контекст из базы данных:
-{formatted_context}
-
-Вопрос пользователя: {question}"""
-        
-        response = query_engine.query(enhanced_question)
-        
-        log_message(f"ОТВЕТ LLM: {response.response}")
-        
-        end_time = time.time()
-        processing_time = end_time - start_time
-        
-        log_message(f"Обработка завершена за {processing_time:.2f} секунд")
-        
-        sources_html = generate_sources_html(reranked_nodes, chunks_df)
-        
-        answer_with_time = f"""<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; margin-bottom: 10px;'>
-        <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
-        <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
-        <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-        Время обработки: {processing_time:.2f} секунд
-        </div>
-        </div>"""
-        
-        chunk_info = []
-        for node in reranked_nodes:
-            metadata = node.metadata if hasattr(node, 'metadata') else {}
-            chunk_info.append({
-                'document_id': metadata.get('document_id', 'unknown'),
-                'section_id': metadata.get('section_id', metadata.get('section', 'unknown')),
-                'section_path': metadata.get('section_path', ''),
-                'section_text': metadata.get('section_text', ''),
-                'level': metadata.get('level', ''),
-                'parent_section': metadata.get('parent_section', ''),
-                'parent_title': metadata.get('parent_title', ''),
-                'type': metadata.get('type', 'text'),
-                'table_number': metadata.get('table_number', ''),
-                'image_number': metadata.get('image_number', ''),
-                'chunk_size': len(node.text),
-                'chunk_text': node.text
-            })
-        from app import create_chunks_display_html
-        chunks_html = create_chunks_display_html(chunk_info)
-
-        return answer_with_time, sources_html, chunks_html
-        
-    except Exception as e:
-        log_message(f"Ошибка обработки вопроса: {str(e)}")
-        error_msg = f"<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Ошибка обработки вопроса: {str(e)}</div>"
-        return error_msg, ""
-
 
 def get_llm_model(model_name):
     try:
@@ -377,7 +289,7 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         start_time = time.time()
         
         retrieved_nodes = query_engine.retriever.retrieve(question)
-        reranked_nodes = rerank_nodes(question, retrieved_nodes, reranker, top_k=10)
+        reranked_nodes = rerank_nodes(question, retrieved_nodes, reranker, top_k=15)
         
         formatted_context = format_context_for_llm(reranked_nodes)