import json
import zipfile
import pandas as pd
from huggingface_hub import hf_hub_download, list_repo_files
from llama_index.core import Document
from llama_index.core.text_splitter import SentenceSplitter
from my_logging import log_message

# Configuration
CHUNK_SIZE = 1024
CHUNK_OVERLAP = 256

def chunk_text_documents(documents):
    text_splitter = SentenceSplitter(
        chunk_size=CHUNK_SIZE,
        chunk_overlap=CHUNK_OVERLAP
    )
    
    chunked = []
    for doc in documents:
        chunks = text_splitter.get_nodes_from_documents([doc])
        for i, chunk in enumerate(chunks):
            chunk.metadata.update({
                'chunk_id': i,
                'total_chunks': len(chunks),
                'chunk_size': len(chunk.text)  # Add chunk size
            })
            chunked.append(chunk)
    
    # Log statistics
    if chunked:
        avg_size = sum(len(c.text) for c in chunked) / len(chunked)
        min_size = min(len(c.text) for c in chunked)
        max_size = max(len(c.text) for c in chunked)
        log_message(f"✓ Text: {len(documents)} docs → {len(chunked)} chunks")
        log_message(f"  Size stats: avg={avg_size:.0f}, min={min_size}, max={max_size} chars")
    
    return chunked


def chunk_table_by_rows(table_data, doc_id, max_rows=30):
    headers = table_data.get('headers', [])
    rows = table_data.get('data', [])
    table_num = table_data.get('table_number', 'unknown')
    table_title = table_data.get('table_title', '')
    section = table_data.get('section', '')
    
    # Enhanced table identification
    table_num_clean = str(table_num).strip()
    
    # Create unique table identifier with section context
    if 'приложени' in section.lower():
        # Extract appendix number
        import re
        appendix_match = re.search(r'приложени[еия]\s*(\d+|[а-яА-Я])', section.lower())
        if appendix_match:
            appendix_num = appendix_match.group(1).upper()
            table_identifier = f"{table_num_clean} (Приложение {appendix_num})"
        else:
            table_identifier = f"{table_num_clean} ({section[:30]})"
    else:
        table_identifier = table_num_clean
    
    if not rows:
        log_message(f"  ⚠️  Table {table_identifier} ({doc_id}): Empty table, skipping")
        return []
    
    log_message(f"  📊 Processing Table {table_identifier} ({doc_id}): {len(rows)} rows, {len(headers)} columns")
    
    # For small tables
    if len(rows) <= max_rows:
        content = format_table_content(table_data, headers, rows, table_identifier)
        chunk_size = len(content)
        
        metadata = {
            'type': 'table',
            'document_id': doc_id,
            'table_number': table_num_clean,
            'table_identifier': table_identifier,  # NEW: unique identifier
            'table_title': table_title,
            'section': section,
            'total_rows': len(rows),
            'chunk_size': chunk_size,
            'is_complete_table': True
        }
        
        log_message(f"    ✓ Single chunk created:")
        log_message(f"      Metadata: {metadata}")
        
        return [Document(text=content, metadata=metadata)]
    
    # For large tables with chunking
    chunks = []
    overlap = 3
    chunk_num = 0
    
    for i in range(0, len(rows), max_rows - overlap):
        chunk_rows = rows[i:min(i+max_rows, len(rows))]
        
        chunk_info = f"Часть {chunk_num+1}: строки {i+1}-{i+len(chunk_rows)} из {len(rows)}"
        
        content = format_table_content(
            table_data, 
            headers, 
            chunk_rows, 
            table_identifier,
            chunk_info=chunk_info
        )
        
        chunk_size = len(content)
        
        metadata = {
            'type': 'table',
            'document_id': doc_id,
            'table_number': table_num_clean,
            'table_identifier': table_identifier,  # NEW
            'table_title': table_title,
            'section': section,
            'chunk_id': chunk_num,
            'row_start': i,
            'row_end': i + len(chunk_rows),
            'total_rows': len(rows),
            'chunk_size': chunk_size,
            'total_chunks': ((len(rows) - overlap) // (max_rows - overlap)) + 1,
            'is_complete_table': False
        }
        
        chunks.append(Document(text=content, metadata=metadata))
        
        log_message(f"    Chunk {chunk_num+1} created:")
        log_message(f"      Rows: {i}-{i+len(chunk_rows)}, Size: {chunk_size} chars")
        log_message(f"      Metadata: {metadata}")
        
        chunk_num += 1
    
    log_message(f"  ✓ Table {table_identifier} ({doc_id}): {len(rows)} rows → {len(chunks)} chunks")
    
    return chunks


def format_table_content(table_data, headers, rows, table_identifier, chunk_info=""):
    doc_id = table_data.get('document_id', table_data.get('document', 'unknown'))
    table_num = table_data.get('table_number', 'unknown')
    table_title = table_data.get('table_title', '')
    section = table_data.get('section', '')
    
    # Use enhanced identifier
    content = f"ДОКУМЕНТ: {doc_id}\n"
    content += f"ТАБЛИЦА: {table_identifier}\n"
    content += f"ПОЛНОЕ НАЗВАНИЕ: {table_identifier}\n"
    content += f"НОМЕР ТАБЛИЦЫ: {table_num}\n"
    if table_title:
        content += f"НАЗВАНИЕ: {table_title}\n"
    if section:
        content += f"РАЗДЕЛ: {section}\n"
    content += f"{'='*70}\n\n"
    
    # Enhanced search keywords
    content += f"Это таблица {table_identifier} из документа {doc_id}. "
    content += f"Идентификатор таблицы: {table_identifier}. "
    content += f"Номер: {table_num}. "
    content += f"Документ: {doc_id}. "
    
    if section:
        content += f"Находится в разделе: {section}. "
        if 'приложени' in section.lower():
            content += f"Таблица из приложения. "
    
    if table_title:
        content += f"Название таблицы: {table_title}. "
        content += f"Таблица о: {table_title}. "
    
    content += f"Поиск: таблица {table_identifier} {doc_id}. "
    
    if chunk_info:
        content += f"\n{chunk_info}\n"
    
    content += f"\n\nСОДЕРЖИМОЕ ТАБЛИЦЫ {table_identifier}:\n"
    content += f"="*70 + "\n\n"
    
    if headers:
        header_str = ' | '.join(str(h) for h in headers)
        content += f"ЗАГОЛОВКИ СТОЛБЦОВ:\n{header_str}\n\n"
    
    content += f"ДАННЫЕ ТАБЛИЦЫ:\n"
    for idx, row in enumerate(rows, 1):
        if isinstance(row, dict):
            parts = [f"{k}: {v}" for k, v in row.items() 
                    if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
            if parts:
                content += f"{idx}. {' | '.join(parts)}\n"
        elif isinstance(row, list):
            parts = [str(v) for v in row if v and str(v).strip() and str(v).lower() not in ['nan', 'none', '']]
            if parts:
                content += f"{idx}. {' | '.join(parts)}\n"
    
    content += f"\n{'='*70}\n"
    content += f"КОНЕЦ ТАБЛИЦЫ {table_identifier} ИЗ {doc_id}\n"
    
    return content

def load_json_documents(repo_id, hf_token, json_dir):
    import zipfile
    import tempfile
    import os
    
    log_message("Loading JSON documents...")
    
    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
    json_files = [f for f in files if f.startswith(json_dir) and f.endswith('.json')]
    zip_files = [f for f in files if f.startswith(json_dir) and f.endswith('.zip')]
    
    log_message(f"Found {len(json_files)} JSON files and {len(zip_files)} ZIP files")
    
    documents = []
    stats = {'success': 0, 'failed': 0, 'empty': 0}
    
    for file_path in json_files:
        try:
            log_message(f"  Loading: {file_path}")
            local_path = hf_hub_download(
                repo_id=repo_id,
                filename=file_path,
                repo_type="dataset",
                token=hf_token
            )
            
            docs = extract_sections_from_json(local_path)
            if docs:
                documents.extend(docs)
                stats['success'] += 1
                log_message(f"    ✓ Extracted {len(docs)} sections")
            else:
                stats['empty'] += 1
                log_message(f"    ⚠ No sections found")
            
        except Exception as e:
            stats['failed'] += 1
            log_message(f"    ✗ Error: {e}")
    
    for zip_path in zip_files:
        try:
            log_message(f"  Processing ZIP: {zip_path}")
            local_zip = hf_hub_download(
                repo_id=repo_id,
                filename=zip_path,
                repo_type="dataset",
                token=hf_token
            )
            
            with zipfile.ZipFile(local_zip, 'r') as zf:
                json_files_in_zip = [f for f in zf.namelist() 
                                    if f.endswith('.json') 
                                    and not f.startswith('__MACOSX')
                                    and not f.startswith('.')
                                    and not '._' in f]
                
                log_message(f"    Found {len(json_files_in_zip)} JSON files in ZIP")
                
                for json_file in json_files_in_zip:
                    try:
                        file_content = zf.read(json_file)
                        
                        # Skip if file is too small
                        if len(file_content) < 10:
                            log_message(f"      ✗ Skipping: {json_file} (file too small)")
                            stats['failed'] += 1
                            continue
                        
                        # Try UTF-8 first (most common)
                        try:
                            text_content = file_content.decode('utf-8')
                        except UnicodeDecodeError:
                            try:
                                text_content = file_content.decode('utf-8-sig')
                            except UnicodeDecodeError:
                                try:
                                    # Try UTF-16 (the issue you're seeing)
                                    text_content = file_content.decode('utf-16')
                                except UnicodeDecodeError:
                                    try:
                                        text_content = file_content.decode('windows-1251')
                                    except UnicodeDecodeError:
                                        log_message(f"      ✗ Skipping: {json_file} (encoding failed)")
                                        stats['failed'] += 1
                                        continue
                        
                        # Validate JSON structure
                        if not text_content.strip().startswith('{') and not text_content.strip().startswith('['):
                            log_message(f"      ✗ Skipping: {json_file} (not valid JSON)")
                            stats['failed'] += 1
                            continue
                        
                        with tempfile.NamedTemporaryFile(mode='w', delete=False, 
                                                        suffix='.json', encoding='utf-8') as tmp:
                            tmp.write(text_content)
                            tmp_path = tmp.name
                        
                        docs = extract_sections_from_json(tmp_path)
                        if docs:
                            documents.extend(docs)
                            stats['success'] += 1
                            log_message(f"      ✓ {json_file}: {len(docs)} sections")
                        else:
                            stats['empty'] += 1
                            log_message(f"      ⚠ {json_file}: No sections")
                        
                        os.unlink(tmp_path)
                        
                    except json.JSONDecodeError as e:
                        stats['failed'] += 1
                        log_message(f"      ✗ {json_file}: Invalid JSON")
                    except Exception as e:
                        stats['failed'] += 1
                        log_message(f"      ✗ {json_file}: {str(e)[:100]}")
                        
        except Exception as e:
            log_message(f"    ✗ Error with ZIP: {e}")
    
    log_message(f"="*60)
    log_message(f"JSON Loading Stats:")
    log_message(f"  Success: {stats['success']}")
    log_message(f"  Empty: {stats['empty']}")
    log_message(f"  Failed: {stats['failed']}")
    log_message(f"  Total sections: {len(documents)}")
    log_message(f"="*60)
    
    return documents

def extract_sections_from_json(json_path):
    """Extract sections from a single JSON file"""
    documents = []
    
    try:
        with open(json_path, 'r', encoding='utf-8') as f:
            data = json.load(f)
        
        doc_id = data.get('document_metadata', {}).get('document_id', 'unknown')
        
        # Extract all section levels
        for section in data.get('sections', []):
            if section.get('section_text', '').strip():
                documents.append(Document(
                    text=section['section_text'],
                    metadata={
                        'type': 'text',
                        'document_id': doc_id,
                        'section_id': section.get('section_id', '')
                    }
                ))
            
            # Subsections
            for subsection in section.get('subsections', []):
                if subsection.get('subsection_text', '').strip():
                    documents.append(Document(
                        text=subsection['subsection_text'],
                        metadata={
                            'type': 'text',
                            'document_id': doc_id,
                            'section_id': subsection.get('subsection_id', '')
                        }
                    ))
                
                # Sub-subsections
                for sub_sub in subsection.get('sub_subsections', []):
                    if sub_sub.get('sub_subsection_text', '').strip():
                        documents.append(Document(
                            text=sub_sub['sub_subsection_text'],
                            metadata={
                                'type': 'text',
                                'document_id': doc_id,
                                'section_id': sub_sub.get('sub_subsection_id', '')
                            }
                        ))
    
    except Exception as e:
        log_message(f"Error extracting from {json_path}: {e}")
    
    return documents


def load_table_documents(repo_id, hf_token, table_dir):
    """Load and chunk tables"""
    log_message("Loading tables...")
    
    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
    table_files = [f for f in files if f.startswith(table_dir) and f.endswith('.json')]
    
    all_chunks = []
    for file_path in table_files:
        try:
            local_path = hf_hub_download(
                repo_id=repo_id,
                filename=file_path,
                repo_type="dataset",
                token=hf_token
            )
            
            with open(local_path, 'r', encoding='utf-8') as f:
                data = json.load(f)
            
            # Extract file-level document_id
            file_doc_id = data.get('document_id', data.get('document', 'unknown'))
            
            for sheet in data.get('sheets', []):
                # Use sheet-level document_id if available, otherwise use file-level
                sheet_doc_id = sheet.get('document_id', sheet.get('document', file_doc_id))
                
                # CRITICAL: Pass document_id to chunk function
                chunks = chunk_table_by_rows(sheet, sheet_doc_id)
                all_chunks.extend(chunks)
                
        except Exception as e:
            log_message(f"Error loading {file_path}: {e}")
    
    log_message(f"✓ Loaded {len(all_chunks)} table chunks")
    return all_chunks


def load_image_documents(repo_id, hf_token, image_dir):
    """Load image descriptions"""
    log_message("Loading images...")
    
    files = list_repo_files(repo_id=repo_id, repo_type="dataset", token=hf_token)
    csv_files = [f for f in files if f.startswith(image_dir) and f.endswith('.csv')]
    
    documents = []
    for file_path in csv_files:
        try:
            local_path = hf_hub_download(
                repo_id=repo_id,
                filename=file_path,
                repo_type="dataset",
                token=hf_token
            )
            
            df = pd.read_csv(local_path)
            
            for _, row in df.iterrows():
                content = f"Документ: {row.get('Обозначение документа', 'unknown')}\n"
                content += f"Рисунок: {row.get('№ Изображения', 'unknown')}\n"
                content += f"Название: {row.get('Название изображения', '')}\n"
                content += f"Описание: {row.get('Описание изображение', '')}\n"
                content += f"Раздел: {row.get('Раздел документа', '')}\n"
                
                chunk_size = len(content)
                
                documents.append(Document(
                    text=content,
                    metadata={
                        'type': 'image',
                        'document_id': str(row.get('Обозначение документа', 'unknown')),
                        'image_number': str(row.get('№ Изображения', 'unknown')),
                        'section': str(row.get('Раздел документа', '')),
                        'chunk_size': chunk_size
                    }
                ))
        except Exception as e:
            log_message(f"Error loading {file_path}: {e}")
    
    if documents:
        avg_size = sum(d.metadata['chunk_size'] for d in documents) / len(documents)
        log_message(f"✓ Loaded {len(documents)} images (avg size: {avg_size:.0f} chars)")
    
    return documents


def load_all_documents(repo_id, hf_token, json_dir, table_dir, image_dir):
    """Main loader - combines all document types"""
    log_message("="*60)
    log_message("STARTING DOCUMENT LOADING")
    log_message("="*60)
    
    # Load text sections
    text_docs = load_json_documents(repo_id, hf_token, json_dir)
    text_chunks = chunk_text_documents(text_docs)
    
    # Load tables (already chunked)
    table_chunks = load_table_documents(repo_id, hf_token, table_dir)
    
    # Load images (no chunking needed)
    image_docs = load_image_documents(repo_id, hf_token, image_dir)
    
    all_docs = text_chunks + table_chunks + image_docs
    
    log_message("="*60)
    log_message(f"TOTAL DOCUMENTS: {len(all_docs)}")
    log_message(f"  Text chunks: {len(text_chunks)}")
    log_message(f"  Table chunks: {len(table_chunks)}")
    log_message(f"  Images: {len(image_docs)}")
    log_message("="*60)
    
    return all_docs