simplest version

documents_prep.py

@@ -32,17 +32,20 @@ def chunk_text_documents(documents):
 
 
 def chunk_table_by_rows(table_data, doc_id, max_rows=30):
-    """Split large tables into row blocks"""
+    """Split large tables into row blocks - OPTIMIZED"""
     headers = table_data.get('headers', [])
     rows = table_data.get('data', [])
     table_num = table_data.get('table_number', 'unknown')
     table_title = table_data.get('table_title', '')
     section = table_data.get('section', '')
     
+    # Normalize table number
+    table_num_clean = str(table_num).replace('№', '').strip()
+    
     if not rows:
         return []
     
-    # Ensure table_data has document_id for format_table_content
+    # Ensure document_id is set
     if 'document_id' not in table_data:
         table_data['document_id'] = doc_id
     
@@ -54,10 +57,12 @@ def chunk_table_by_rows(table_data, doc_id, max_rows=30):
             metadata={
                 'type': 'table',
                 'document_id': doc_id,
-                'table_number': table_num,
+                'table_number': table_num_clean,
+                'table_number_original': table_num,
                 'table_title': table_title,
                 'section': section,
-                'total_rows': len(rows)
+                'total_rows': len(rows),
+                'is_complete_table': True
             }
         )]
     
@@ -69,7 +74,7 @@ def chunk_table_by_rows(table_data, doc_id, max_rows=30):
             table_data, 
             headers, 
             chunk_rows, 
-            chunk_info=f"Строки {i+1}-{i+len(chunk_rows)} из {len(rows)}"
+            chunk_info=f"Часть таблицы: строки {i+1}-{i+len(chunk_rows)} из {len(rows)} всего"
         )
         
         chunks.append(Document(
@@ -77,75 +82,83 @@ def chunk_table_by_rows(table_data, doc_id, max_rows=30):
             metadata={
                 'type': 'table',
                 'document_id': doc_id,
-                'table_number': table_num,
+                'table_number': table_num_clean,
+                'table_number_original': table_num,
                 'table_title': table_title,
                 'section': section,
                 'chunk_id': i // max_rows,
                 'row_start': i,
                 'row_end': i + len(chunk_rows),
-                'total_rows': len(rows)
+                'total_rows': len(rows),
+                'total_chunks': (len(rows) + max_rows - 1) // max_rows,
+                'is_complete_table': False
             }
         ))
     
-    log_message(f"  📊 Table {table_num} ({doc_id}): {len(rows)} rows → {len(chunks)} chunks")
+    log_message(f"  📊 Table {table_num_clean} ({doc_id}): {len(rows)} rows → {len(chunks)} chunks")
     return chunks
 
 
 def format_table_content(table_data, headers, rows, chunk_info=""):
-    """Format table for semantic search"""
+    """Format table for semantic search - OPTIMIZED"""
     doc_id = table_data.get('document_id', table_data.get('document', 'unknown'))
     table_num = table_data.get('table_number', 'unknown')
     table_title = table_data.get('table_title', '')
     section = table_data.get('section', '')
     
-    # Normalize table number
-    if table_num and table_num != 'unknown':
-        if not str(table_num).startswith('№'):
-            table_num = f"№{table_num}"
+    # Normalize table number - remove № prefix for consistent searching
+    table_num_clean = str(table_num).replace('№', '').strip()
     
-    content = f"=== ТАБЛИЦА ===\n"
+    # Create highly searchable header
+    content = f"ТАБЛИЦА {table_num_clean}\n"
     content += f"Документ: {doc_id}\n"
-    content += f"Таблица: {table_num}\n"
     if table_title:
-        content += f"Название: {table_title}\n"
+        content += f"Название таблицы: {table_title}\n"
     if section:
         content += f"Раздел: {section}\n"
     if chunk_info:
         content += f"{chunk_info}\n"
-    content += f"================\n\n"
+    content += f"{'='*60}\n\n"
     
-    # Add searchable description
-    content += f"Это таблица {table_num} из документа {doc_id}. "
+    # Add multiple search-friendly descriptions
+    content += f"Это таблица {table_num_clean} из документа {doc_id}. "
+    content += f"Таблица номер {table_num_clean}. "
     if table_title:
-        content += f"{table_title}. "
+        content += f"Таблица называется: {table_title}. "
+        content += f"Содержание таблицы: {table_title}. "
     if section:
-        content += f"Находится в разделе: {section}. "
-    content += f"\n\n"
+        content += f"Таблица находится в разделе {section}. "
+    
+    # Add searchable patterns
+    content += f"Если вы ищете таблицу {table_num_clean} в {doc_id}, это она. "
+    content += f"\n\nСОДЕРЖАНИЕ ТАБЛИЦЫ:\n"
     
-    # Headers
+    # Headers with emphasis
     if headers:
         header_str = ' | '.join(str(h) for h in headers)
-        content += f"Колонки: {header_str}\n\n"
+        content += f"\nКОЛОНКИ: {header_str}\n"
+        content += f"Заголовки столбцов: {header_str}\n\n"
     
-    # Rows
-    for row in rows:
+    # Row data with better formatting
+    content += "ДАННЫЕ:\n"
+    for idx, row in enumerate(rows, 1):
         if isinstance(row, dict):
             parts = [f"{k}: {v}" for k, v in row.items() 
-                    if v and str(v).strip() and str(v) != 'nan']
+                    if v and str(v).strip() and str(v).lower() != 'nan']
             if parts:
-                content += ' | '.join(parts) + "\n"
+                content += f"Строка {idx}: {' | '.join(parts)}\n"
         elif isinstance(row, list):
-            parts = [str(v) for v in row if v and str(v).strip() and str(v) != 'nan']
+            parts = [str(v) for v in row if v and str(v).strip() and str(v).lower() != 'nan']
             if parts:
-                content += ' | '.join(parts) + "\n"
+                content += f"Строка {idx}: {' | '.join(parts)}\n"
     
     return content
 
-
 def load_json_documents(repo_id, hf_token, json_dir):
     """Load text sections from JSON (including ZIPs)"""
     import zipfile
     import tempfile
+    import os
     
     log_message("Loading JSON documents...")
     
@@ -160,6 +173,7 @@ def load_json_documents(repo_id, hf_token, json_dir):
     # Load direct JSON files
     for file_path in json_files:
         try:
+            log_message(f"  Loading: {file_path}")
             local_path = hf_hub_download(
                 repo_id=repo_id,
                 filename=file_path,
@@ -169,13 +183,15 @@ def load_json_documents(repo_id, hf_token, json_dir):
             
             docs = extract_sections_from_json(local_path)
             documents.extend(docs)
+            log_message(f"    ✓ Extracted {len(docs)} sections")
             
         except Exception as e:
-            log_message(f"Error loading {file_path}: {e}")
+            log_message(f"    ✗ Error loading {file_path}: {e}")
     
     # Extract and load ZIP files
     for zip_path in zip_files:
         try:
+            log_message(f"  Processing ZIP: {zip_path}")
             local_zip = hf_hub_download(
                 repo_id=repo_id,
                 filename=zip_path,
@@ -184,23 +200,59 @@ def load_json_documents(repo_id, hf_token, json_dir):
             )
             
             with zipfile.ZipFile(local_zip, 'r') as zf:
-                for json_file in zf.namelist():
-                    if json_file.endswith('.json') and not json_file.startswith('__MACOSX'):
-                        with zf.open(json_file) as f:
-                            with tempfile.NamedTemporaryFile(delete=False, suffix='.json') as tmp:
-                                tmp.write(f.read())
-                                tmp_path = tmp.name
-                            
-                            docs = extract_sections_from_json(tmp_path)
-                            documents.extend(docs)
-                            
-                            import os
-                            os.unlink(tmp_path)
-                            
+                json_files_in_zip = [f for f in zf.namelist() 
+                                    if f.endswith('.json') 
+                                    and not f.startswith('__MACOSX')
+                                    and not f.startswith('.')]
+                
+                log_message(f"    Found {len(json_files_in_zip)} JSON files in ZIP")
+                
+                for json_file in json_files_in_zip:
+                    try:
+                        log_message(f"      Processing: {json_file}")
+                        
+                        # Read file content
+                        file_content = zf.read(json_file)
+                        
+                        # Try to detect encoding
+                        try:
+                            # First try UTF-8
+                            text_content = file_content.decode('utf-8')
+                        except UnicodeDecodeError:
+                            try:
+                                # Try UTF-8 with BOM
+                                text_content = file_content.decode('utf-8-sig')
+                            except UnicodeDecodeError:
+                                try:
+                                    # Try Windows-1251 (common for Cyrillic)
+                                    text_content = file_content.decode('windows-1251')
+                                except UnicodeDecodeError:
+                                    log_message(f"        ✗ Skipping: Cannot decode {json_file}")
+                                    continue
+                        
+                        # Write to temp file
+                        with tempfile.NamedTemporaryFile(mode='w', delete=False, 
+                                                        suffix='.json', encoding='utf-8') as tmp:
+                            tmp.write(text_content)
+                            tmp_path = tmp.name
+                        
+                        # Extract sections
+                        docs = extract_sections_from_json(tmp_path)
+                        documents.extend(docs)
+                        log_message(f"        ✓ Extracted {len(docs)} sections")
+                        
+                        # Clean up temp file
+                        os.unlink(tmp_path)
+                        
+                    except json.JSONDecodeError as e:
+                        log_message(f"        ✗ Invalid JSON in {json_file}: {e}")
+                    except Exception as e:
+                        log_message(f"        ✗ Error processing {json_file}: {e}")
+                        
         except Exception as e:
-            log_message(f"Error loading ZIP {zip_path}: {e}")
+            log_message(f"    ✗ Error with ZIP {zip_path}: {e}")
     
-    log_message(f"✓ Loaded {len(documents)} text sections")
+    log_message(f"✓ Loaded {len(documents)} text sections total")
     return documents
 
 def extract_sections_from_json(json_path):

utils.py

@@ -37,44 +37,86 @@ def format_sources(nodes):
     return "\n".join(set(sources))
 
 def answer_question(question, query_engine, reranker):
-    """Answer question using RAG"""
+    """Answer question using RAG - OPTIMIZED"""
     try:
         log_message(f"Query: {question}")
         
-        # Retrieve
+        # Retrieve more nodes initially
         retrieved = query_engine.retriever.retrieve(question)
         log_message(f"Retrieved {len(retrieved)} nodes")
         
-        # Rerank
-        reranked = rerank_nodes(question, retrieved, reranker, top_k=15)
+        # Log what was retrieved for debugging
+        doc_ids = [n.metadata.get('document_id', 'unknown') for n in retrieved]
+        table_nums = [n.metadata.get('table_number', '') for n in retrieved if n.metadata.get('type') == 'table']
+        log_message(f"Retrieved from documents: {set(doc_ids)}")
+        if table_nums:
+            log_message(f"Retrieved tables: {set(table_nums)}")
+        
+        # Rerank with more nodes
+        reranked = rerank_nodes(question, retrieved, reranker, top_k=20)
         log_message(f"Reranked to {len(reranked)} nodes")
         
-        # Format context
-        context = "\n\n".join([
-            f"[{n.metadata.get('document_id', 'unknown')}]\n{n.text}" 
-            for n in reranked
-        ])
+        # Log what survived reranking
+        doc_ids_reranked = [n.metadata.get('document_id', 'unknown') for n in reranked]
+        table_nums_reranked = [n.metadata.get('table_number', '') for n in reranked if n.metadata.get('type') == 'table']
+        log_message(f"After reranking - documents: {set(doc_ids_reranked)}")
+        if table_nums_reranked:
+            log_message(f"After reranking - tables: {set(table_nums_reranked)}")
+        
+        # Format context with clear source attribution
+        context_parts = []
+        for n in reranked:
+            meta = n.metadata
+            doc_id = meta.get('document_id', 'unknown')
+            doc_type = meta.get('type', 'text')
+            
+            if doc_type == 'table':
+                table_num = meta.get('table_number', 'unknown')
+                title = meta.get('table_title', '')
+                source_label = f"[{doc_id} - Таблица {table_num}]"
+                if title:
+                    source_label += f" {title}"
+            elif doc_type == 'image':
+                img_num = meta.get('image_number', 'unknown')
+                source_label = f"[{doc_id} - Рисунок {img_num}]"
+            else:
+                section = meta.get('section_id', '')
+                source_label = f"[{doc_id} - Раздел {section}]"
+            
+            context_parts.append(f"{source_label}\n{n.text}")
+        
+        context = "\n\n" + "="*60 + "\n\n".join(context_parts)
         
-        # Generate answer
-        prompt = f"""Контекст из базы данных:
+        # Improved prompt
+        prompt = f"""Ты эксперт по технической документации. Используй ТОЛЬКО предоставленный контекст для ответа.
+
+КОНТЕКСТ ИЗ БАЗЫ ДАННЫХ:
 {context}
 
-Вопрос: {question}
+ВОПРОС: {question}
+
+ИНСТРУКЦИИ:
+1. Отвечай ТОЛЬКО на основе предоставленного контекста
+2. Если спрашивают о конкретной таблице - найди её в контексте и передай её содержание
+3. ОБЯЗАТЕЛЬНО укажи источник: документ, номер таблицы/раздела
+4. Если нужной информации нет в контексте - четко скажи об этом
+5. Будь точным и конкретным
 
-Ответь на вопрос используя ТОЛЬКО информацию из контекста. Цитируй источники."""
+ОТВЕТ:"""
         
         response = query_engine.query(prompt)
-        
         sources = format_sources(reranked)
         
         return response.response, sources
         
     except Exception as e:
         log_message(f"Error: {e}")
+        import traceback
+        log_message(traceback.format_exc())
         return f"Ошибка: {e}", ""
 
-def rerank_nodes(query, nodes, reranker, top_k=15, min_score=0.5):
-    """Rerank nodes with diversity"""
+def rerank_nodes(query, nodes, reranker, top_k=20, min_score=0.3):
+    """Rerank nodes with diversity - MORE LENIENT"""
     if not nodes:
         return []
     
@@ -85,28 +127,40 @@ def rerank_nodes(query, nodes, reranker, top_k=15, min_score=0.5):
     # Sort by score
     scored = sorted(zip(nodes, scores), key=lambda x: x[1], reverse=True)
     
-    # Filter by threshold
+    # More lenient threshold
     filtered = [(n, s) for n, s in scored if s >= min_score]
     
     if not filtered:
-        # Fallback: take top 30% if nothing passes threshold
-        cutoff = max(scores) * 0.6
-        filtered = [(n, s) for n, s in scored if s >= cutoff]
+        # Fallback: take top 50% if nothing passes threshold
+        cutoff = max(scores) * 0.5
+        filtered = [(n, s) for n, s in scored if s >= cutoff][:top_k]
     
-    # Diversity selection
+    # Log top scores for debugging
+    log_message(f"Top 5 reranking scores: {[f'{s:.3f}' for _, s in scored[:5]]}")
+    
+    # Diversity selection - but prioritize tables if query mentions them
     selected = []
     seen_docs = set()
+    table_nodes = []
+    other_nodes = []
     
     for node, score in filtered:
+        if node.metadata.get('type') == 'table':
+            table_nodes.append((node, score))
+        else:
+            other_nodes.append((node, score))
+    
+    # If query mentions "таблица", prioritize table nodes
+    if 'таблиц' in query.lower():
+        combined = table_nodes + other_nodes
+    else:
+        combined = filtered
+    
+    for node, score in combined[:top_k]:
         if len(selected) >= top_k:
             break
-        
-        doc_id = node.metadata.get('document_id', 'unknown')
-        
-        # Prioritize diverse documents
-        if doc_id not in seen_docs or len(selected) < 5:
-            selected.append(node)
-            seen_docs.add(doc_id)
+        selected.append(node)
+        seen_docs.add(node.metadata.get('document_id', 'unknown'))
     
     log_message(f"Reranked: {len(filtered)} → {len(selected)} (from {len(seen_docs)} docs)")