Removed duplicate logs throughout all files

chat_handler.py

@@ -1,154 +0,0 @@
-import time
-import logging
-
-logger = logging.getLogger(__name__)
-
-def log_message(message):
-    logger.info(message)
-    print(message, flush=True)
-
-class ChatHandler:
-    def __init__(self, index_retriever):
-        self.index_retriever = index_retriever
-        self.chat_history = []
-        
-    def format_section_path(self, metadata):
-        parts = []
-        
-        section_id = metadata.get('section_id')
-        if section_id and section_id != 'Unknown':
-            parts.append(section_id)
-            
-        subsection_id = metadata.get('subsection_id')
-        if subsection_id and subsection_id != 'Unknown':
-            parts.append(subsection_id)
-            
-        sub_subsection_id = metadata.get('sub_subsection_id')
-        if sub_subsection_id and sub_subsection_id != 'Unknown':
-            parts.append(sub_subsection_id)
-            
-        sub_sub_subsection_id = metadata.get('sub_sub_subsection_id')
-        if sub_sub_subsection_id and sub_sub_subsection_id != 'Unknown':
-            parts.append(sub_sub_subsection_id)
-        
-        return " → ".join(parts) if parts else "Основной раздел"
-
-    def generate_sources_html(self, nodes):
-        html = "<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; max-height: 400px; overflow-y: auto;'>"
-        html += "<h3 style='color: #63b3ed; margin-top: 0;'>Источники:</h3>"
-        
-        for i, node in enumerate(nodes):
-            metadata = node.metadata if hasattr(node, 'metadata') else {}
-            doc_type = metadata.get('type', 'text')
-            doc_id = metadata.get('document_id', 'unknown')
-            
-            html += f"<div style='margin-bottom: 15px; padding: 15px; border: 1px solid #4a5568; border-radius: 8px; background-color: #1a202c;'>"
-            
-            if doc_type == 'text':
-                document_name = metadata.get('document_name', doc_id)
-                
-                html += f"<h4 style='margin: 0 0 10px 0; color: #63b3ed;'>📄 {doc_id}</h4>"
-                html += f"<div style='color: #a0aec0; font-size: 13px; margin-bottom: 8px;'>{document_name}</div>"
-                
-            elif doc_type == 'table':
-                table_num = metadata.get('table_number', 'unknown')
-                section = metadata.get('section', '')
-                if table_num and table_num != 'unknown':
-                    if not table_num.startswith('№'):
-                        table_num = f"№{table_num}"
-                    html += f"<h4 style='margin: 0 0 10px 0; color: #68d391;'>📊 Таблица {table_num} - {doc_id}</h4>"
-                else:
-                    html += f"<h4 style='margin: 0 0 10px 0; color: #68d391;'>📊 Таблица - {doc_id}</h4>"
-                if section:
-                    html += f"<div style='color: #68d391; font-size: 14px;'>📍 {section}</div>"
-                
-            elif doc_type == 'image':
-                image_num = metadata.get('image_number', 'unknown')
-                section = metadata.get('section', '')
-                if image_num and image_num != 'unknown':
-                    if not str(image_num).startswith('№'):
-                        image_num = f"№{image_num}"
-                    html += f"<h4 style='margin: 0 0 10px 0; color: #fbb6ce;'>🖼️ Изображение {image_num} - {doc_id}</h4>"
-                else:
-                    html += f"<h4 style='margin: 0 0 10px 0; color: #fbb6ce;'>🖼️ Изображение - {doc_id}</h4>"
-                if section:
-                    html += f"<div style='color: #fbb6ce; font-size: 14px;'>📍 {section}</div>"
-            
-            html += "</div>"
-        
-        html += "</div>"
-        return html
-
-    def answer_question(self, question):
-        if not self.index_retriever.is_initialized():
-            return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Система не инициализирована</div>", ""
-        
-        try:
-            log_message(f"Получен вопрос: {question}")
-            current_model = self.index_retriever.get_current_model()
-            log_message(f"Используется модель: {current_model}")
-            start_time = time.time()
-            
-            retrieved_nodes = self.index_retriever.retrieve_nodes(question)
-            
-            if not retrieved_nodes:
-                return "<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Не удалось найти релевантные документы</div>", ""
-            
-            log_message(f"Отправляю запрос в LLM с {len(retrieved_nodes)} узлами")
-            response = self.index_retriever.query_engine.query(question)
-            
-            end_time = time.time()
-            processing_time = end_time - start_time
-            
-            log_message(f"Обработка завершена за {processing_time:.2f} секунд")
-            
-            self.chat_history.append({
-                "question": question,
-                "answer": response.response,
-                "model": current_model,
-                "processing_time": processing_time,
-                "nodes_count": len(retrieved_nodes)
-            })
-            
-            sources_html = self.generate_sources_html(retrieved_nodes)
-            
-            answer_with_time = f"""<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; margin-bottom: 10px;'>
-            <h3 style='color: #63b3ed; margin-top: 0;'>Ответ (Модель: {current_model}):</h3>
-            <div style='line-height: 1.6; font-size: 16px;'>{response.response}</div>
-            <div style='margin-top: 15px; padding-top: 10px; border-top: 1px solid #4a5568; font-size: 14px; color: #a0aec0;'>
-            Время обработки: {processing_time:.2f} секунд 
-            </div>
-            </div>"""
-            
-            return answer_with_time, sources_html
-            
-        except Exception as e:
-            log_message(f"Ошибка обработки вопроса: {str(e)}")
-            error_msg = f"<div style='background-color: #e53e3e; color: white; padding: 20px; border-radius: 10px;'>Ошибка обработки вопроса: {str(e)}</div>"
-            return error_msg, ""
-
-    def get_chat_history(self):
-        return self.chat_history
-    
-    def clear_history(self):
-        self.chat_history = []
-        log_message("История чата очищена")
-    
-    def get_history_html(self):
-        if not self.chat_history:
-            return "<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; text-align: center;'>История пуста</div>"
-        
-        html = "<div style='background-color: #2d3748; color: white; padding: 20px; border-radius: 10px; max-height: 500px; overflow-y: auto;'>"
-        html += "<h3 style='color: #63b3ed; margin-top: 0;'>История чата:</h3>"
-        
-        for i, entry in enumerate(reversed(self.chat_history[-10:])):
-            html += f"<div style='margin-bottom: 20px; padding: 15px; border: 1px solid #4a5568; border-radius: 8px; background-color: #1a202c;'>"
-            html += f"<div style='color: #68d391; font-weight: bold; margin-bottom: 8px;'>Вопрос {len(self.chat_history) - i}:</div>"
-            html += f"<div style='margin-bottom: 10px; font-size: 14px;'>{entry['question']}</div>"
-            html += f"<div style='color: #63b3ed; font-weight: bold; margin-bottom: 8px;'>Ответ ({entry['model']}):</div>"
-            html += f"<div style='margin-bottom: 10px; font-size: 14px; line-height: 1.4;'>{entry['answer'][:300]}{'...' if len(entry['answer']) > 300 else ''}</div>"
-            html += f"<div style='color: #a0aec0; font-size: 12px;'>Время: {entry['processing_time']:.2f}с</div>"
-            html += "</div>"
-        
-        html += "</div>"
-        return html

documents_prep.py

@@ -44,39 +44,27 @@ def process_documents_with_chunking(documents):
     all_chunked_docs = []
     chunk_info = []
     table_count = 0
+    table_added_count = 0
     image_count = 0
     text_chunks_count = 0
-    custom_processed_count = 0
     
     for doc in documents:
         doc_type = doc.metadata.get('type', 'text')
         
         if doc_type == 'table':
             table_count += 1
-            doc_id = doc.metadata.get('document_id', 'unknown')
-            table_num = doc.metadata.get('table_number', 'unknown')
-            from table_prep import get_custom_config
-            method_config = get_custom_config(doc_id, table_num)
+            doc_size = len(doc.text)
             
-            if method_config:
-                custom_processed_count += 1
-                all_chunked_docs.append(doc)
-                chunk_info.append({
-                    'document_id': doc_id,
-                    'section_id': doc.metadata.get('section_id', 'unknown'),
-                    'chunk_id': 0,
-                    'chunk_size': len(doc.text),
-                    'chunk_preview': doc.text[:200] + "..." if len(doc.text) > 200 else doc.text,
-                    'type': 'table',
-                    'table_number': table_num,
-                    'processing_method': method_config.get('method')
-                })
-                continue
+            # Log table size
+            table_num = doc.metadata.get('table_number', 'unknown')
+            doc_id = doc.metadata.get('document_id', 'unknown')
+            log_message(f"Таблица {table_num} в документе {doc_id}: размер {doc_size} символов")
             
-            doc_size = len(doc.text)
+            # Always add table regardless of size or custom config
             if doc_size > CHUNK_SIZE:
                 chunked_docs = chunk_document(doc)
                 all_chunked_docs.extend(chunked_docs)
+                table_added_count += len(chunked_docs)
                 
                 for i, chunk_doc in enumerate(chunked_docs):
                     chunk_info.append({
@@ -87,10 +75,11 @@ def process_documents_with_chunking(documents):
                         'chunk_preview': chunk_doc.text[:200] + "..." if len(chunk_doc.text) > 200 else chunk_doc.text,
                         'type': 'table',
                         'table_number': chunk_doc.metadata.get('table_number', 'unknown'),
-                        'processing_method': 'standard_chunked'
+                        'processing_method': 'chunked'
                     })
             else:
                 all_chunked_docs.append(doc)
+                table_added_count += 1
                 chunk_info.append({
                     'document_id': doc.metadata.get('document_id', 'unknown'),
                     'section_id': doc.metadata.get('section_id', 'unknown'),
@@ -158,7 +147,7 @@ def process_documents_with_chunking(documents):
                     'type': 'text'
                 })
         
-    log_message(f"Таблицы: {table_count} (кастомных: {custom_processed_count}), Изображения: {image_count}, Текстовые чанки: {text_chunks_count}, Итого: {len(all_chunked_docs)}")
+    log_message(f"Таблицы: всего {table_count}, добавлено {table_added_count}, Изображения: {image_count}, Текстовые чанки: {text_chunks_count}, Итого: {len(all_chunked_docs)}")
     
     return all_chunked_docs, chunk_info
 

utils.py

@@ -379,6 +379,20 @@ def answer_question(question, query_engine, reranker, current_model, chunks_df=N
         retrieved_nodes = query_engine.retriever.retrieve(question)
         reranked_nodes = rerank_nodes(question, retrieved_nodes, reranker, top_k=10)
         
+        # Add after reranking, before formatting context:
+        log_message(f"=== НАЙДЕННЫЕ ЧАНКИ ПОСЛЕ ПЕРЕРАНЖИРОВКИ ===")
+        log_message(f"Всего найдено релевантных чанков: {len(reranked_nodes)}")
+        for i, node in enumerate(reranked_nodes, 1):
+            log_message(f"Чанк {i}/{len(reranked_nodes)}:")
+            log_message(f"  Документ: {node.metadata.get('document_id', 'unknown')}")
+            log_message(f"  Тип: {node.metadata.get('type', 'unknown')}")
+            log_message(f"  Раздел: {node.metadata.get('section_id', 'unknown')}")
+            if node.metadata.get('type') == 'table':
+                log_message(f"  Таблица: {node.metadata.get('table_number', 'unknown')}")
+            log_message(f"  Размер: {len(node.text)} символов")
+            log_message(f"  Превью: {node.text[:150]}...")
+            log_message("-" * 50)
+
         formatted_context = format_context_for_llm(reranked_nodes)
         
         enhanced_question = f"""