App.py

import os
import json
import requests
import subprocess
import shutil
import asyncio
import threading
import time
import hashlib
import zipfile
from typing import Dict, List, Set, Optional
from fastapi import FastAPI, BackgroundTasks, HTTPException, Form
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from huggingface_hub import HfApi, list_repo_files, HfHubHTTPError

# ==== CONFIGURATION ====
HF_TOKEN = os.getenv("HF_TOKEN", "")
SOURCE_REPO_ID = os.getenv("SOURCE_REPO", "Fred808/BG1") # Source for RARs
DEST_REPO_ID_RAR = os.getenv("DEST_REPO_RAR", "") # Destination for extracted RAR contents (set to empty string if not needed)
DEST_REPO_ID_VIDEO = os.getenv("DEST_REPO_VIDEO", "Fred808/BG3") # Destination for zipped video frames

DOWNLOAD_FOLDER = "downloads"
EXTRACT_FOLDER = "extracted_tmp"
VIDEO_FRAMES_EXTRACT_FOLDER = "video_frames_tmp"
ZIPPED_FRAMES_FOLDER = "zipped_frames"

DOWNLOAD_STATE_FILE = "download_progress.json"
PROCESS_STATE_FILE = "process_progress.json"
UPLOADED_FOLDERS_FILE = "uploaded_folders.json"  # Track uploaded folder hashes for BG2
PROCESSED_VIDEO_COURSES_FILE = "processed_video_courses.json" # Track processed video course folders for BG3
FAILED_FILES_LOG = "failed_files.txt"

CHUNK_SIZE = 3  # Smaller chunks for Space environment
PROCESSING_DELAY = 2  # Delay between processing files (seconds)
VIDEO_FRAME_FPS = 3 # Frames per second to extract from videos

os.makedirs(DOWNLOAD_FOLDER, exist_ok=True)
os.makedirs(EXTRACT_FOLDER, exist_ok=True)
os.makedirs(VIDEO_FRAMES_EXTRACT_FOLDER, exist_ok=True)
os.makedirs(ZIPPED_FRAMES_FOLDER, exist_ok=True)

api = HfApi(token=HF_TOKEN)

# Global state
processing_status = {
    "is_running": False,
    "current_file": None,
    "total_files": 0,
    "processed_files": 0,
    "failed_files": 0,
    "uploaded_rar_folders": 0,
    "uploaded_video_courses": 0,
    "last_update": None,
    "logs": []
}

app = FastAPI(title="RAR & Video Processing Service", description="Automated RAR extraction and video frame upload service")

# Add CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

def log_message(message: str):
    """Add message to logs and print it"""
    timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
    log_entry = f"[{timestamp}] {message}"
    print(log_entry)
    processing_status["logs"].append(log_entry)
    processing_status["last_update"] = timestamp
    # Keep only last 100 log entries
    if len(processing_status["logs"]) > 100:
        processing_status["logs"] = processing_status["logs"][-100:]

def log_failed_file(filename: str, error_msg: str):
    """Log failed files to a separate file for later review"""
    with open(FAILED_FILES_LOG, "a") as f:
        f.write(f"{filename}: {error_msg}\n")
    log_message(f"❌ Failed: {filename} - {error_msg}")

def get_folder_hash(folder_name: str) -> str:
    """Generate a hash for the folder name to use as a unique identifier"""
    return hashlib.md5(folder_name.encode()).hexdigest()

def load_uploaded_folders() -> Set[str]:
    """Load set of uploaded folder hashes for BG2"""
    if os.path.exists(UPLOADED_FOLDERS_FILE):
        try:
            with open(UPLOADED_FOLDERS_FILE, "r") as f:
                data = json.load(f)
                return set(data.get("uploaded_folder_hashes", []))
        except json.JSONDecodeError:
            log_message(f"⚠️ Warning: Could not decode {UPLOADED_FOLDERS_FILE}. Starting with empty set.")
            return set()
    return set()

def save_uploaded_folders(uploaded_set: Set[str]):
    """Save set of uploaded folder hashes for BG2"""
    with open(UPLOADED_FOLDERS_FILE, "w") as f:
        json.dump({"uploaded_folder_hashes": list(uploaded_set)}, f)

def load_processed_video_courses() -> Set[str]:
    """Loads the set of processed video course folder names for BG3."""
    if os.path.exists(PROCESSED_VIDEO_COURSES_FILE):
        try:
            with open(PROCESSED_VIDEO_COURSES_FILE, "r") as f:
                return set(json.load(f))
        except json.JSONDecodeError:
            log_message(f"⚠️ Warning: Could not decode {PROCESSED_VIDEO_COURSES_FILE}. Starting with empty set.")
            return set()
    return set()

def save_processed_video_courses(processed_set: set):
    """Saves the set of processed video course folder names for BG3 to a file."""
    with open(PROCESSED_VIDEO_COURSES_FILE, "w") as f:
        json.dump(list(processed_set), f)

def load_download_state() -> int:
    """Load download progress state"""
    if os.path.exists(DOWNLOAD_STATE_FILE):
        try:
            with open(DOWNLOAD_STATE_FILE, "r") as f:
                return json.load(f).get("next_download_index", 0)
        except json.JSONDecodeError:
            log_message(f"⚠️ Warning: Could not decode {DOWNLOAD_STATE_FILE}. Starting download from index 0.")
            return 0
    return 0

def save_download_state(next_index: int):
    """Save download progress state"""
    with open(DOWNLOAD_STATE_FILE, "w") as f:
        json.dump({"next_download_index": next_index}, f)

def load_processed_files_state() -> set:
    """Load processed files from the state file"""
    if os.path.exists(PROCESS_STATE_FILE):
        try:
            with open(PROCESS_STATE_FILE, "r") as f:
                data = json.load(f)
                return set(data.get("processed_rars", []))
        except json.JSONDecodeError:
            log_message(f"⚠️ Warning: Could not decode {PROCESS_STATE_FILE}. Starting with empty processed list.")
            return set()
    return set()

def save_processed_files_state(processed_set: set):
    """Save processed files to the state file"""
    with open(PROCESS_STATE_FILE, "w") as f:
        json.dump({"processed_rars": list(processed_set)}, f)

def download_rar_files(start_index: int, chunk_size: int) -> tuple:
    """Downloads a batch of RAR files from the source dataset"""
    try:
        all_files = list_repo_files(repo_id=SOURCE_REPO_ID, repo_type="dataset", token=HF_TOKEN)
        # Filter for .rar files and exclude the specific one
        rar_files_in_repo = sorted([f for f in all_files if f.endswith(".rar") and "ZBrush/3DConceptArtist_TheUltimateZbrushGuide_DownloadPirate.com.rar" not in f])

        end_index = start_index + chunk_size
        files_to_download_metadata = rar_files_in_repo[start_index:end_index]

        if not files_to_download_metadata:
            log_message("✅ No more RAR files to download.")
            return [], start_index

        log_message(f"📥 Downloading RAR files {start_index + 1} to {end_index} from {SOURCE_REPO_ID}")

        downloaded_paths = []
        for file_path_in_repo in files_to_download_metadata:
            filename = os.path.basename(file_path_in_repo)
            dest_path = os.path.join(DOWNLOAD_FOLDER, filename)
            file_url = f"https://huggingface.co/datasets/{SOURCE_REPO_ID}/resolve/main/{file_path_in_repo}"
            headers = {"Authorization": f"Bearer {HF_TOKEN}"}

            if os.path.exists(dest_path):
                log_message(f"⏩ Already exists, skipping: {filename}")
                downloaded_paths.append(dest_path)
                continue

            log_message(f"🔽 Downloading: {file_path_in_repo}")
            try:
                with requests.get(file_url, headers=headers, stream=True) as r:
                    r.raise_for_status()
                    with open(dest_path, "wb") as f:
                        for chunk in r.iter_content(chunk_size=8192):
                            f.write(chunk)
                log_message(f"✅ Downloaded: {filename}")
                downloaded_paths.append(dest_path)
            except Exception as e:
                log_message(f"❌ Failed to download {file_path_in_repo}: {e}")
                log_failed_file(file_path_in_repo, f"Download failed: {e}")

        return downloaded_paths, end_index
    except Exception as e:
        log_message(f"❌ Error in download_rar_files: {e}")
        return [], start_index

def extract_frames(video_path: str, output_folder: str, fps: int) -> bool:
    """Extracts frames from a video using ffmpeg."""
    os.makedirs(output_folder, exist_ok=True)
    
    # Ensure ffmpeg is available
    if shutil.which("ffmpeg") is None:
        log_message("❌ ffmpeg not found. Please install ffmpeg.")
        return False

    # ffmpeg command to extract frames at specified FPS
    command = [
        "ffmpeg",
        "-i", video_path,
        "-vf", f"fps={fps}",
        os.path.join(output_folder, "frame_%04d.png")
    ]

    log_message(f"🖼️ Extracting frames from {os.path.basename(video_path)} to {output_folder} at {fps} FPS...")
    try:
        subprocess.run(command, check=True, capture_output=True, text=True)
        log_message(f"✅ Successfully extracted frames from {os.path.basename(video_path)}")
        return True
    except subprocess.CalledProcessError as e:
        log_message(f"❌ Error extracting frames from {os.path.basename(video_path)}: {e.stderr}")
        return False

def zip_folder(folder_path: str, output_zip_path: str) -> bool:
    """Zips the contents of a folder."""
    log_message(f" compressing {folder_path} to {output_zip_path}...")
    try:
        with zipfile.ZipFile(output_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
            for root, _, files in os.walk(folder_path):
                for file in files:
                    file_path = os.path.join(root, file)
                    arcname = os.path.relpath(file_path, folder_path)
                    zipf.write(file_path, arcname)
        log_message(f"✅ Successfully zipped {folder_path}")
        return True
    except Exception as e:
        log_message(f"❌ Error zipping {folder_path}: {e}")
        return False

def upload_file_to_hf(local_path: str, path_in_repo: str, repo_id: str, max_retries: int = 5, initial_delay: int = 5) -> bool:
    """Uploads a single file to Hugging Face Hub with retry logic and exponential backoff."""
    log_message(f"⬆️ Uploading {os.path.basename(local_path)} to {repo_id}/{path_in_repo}")
    
    for attempt in range(max_retries):
        try:
            api.upload_file(
                path_or_fileobj=local_path,
                path_in_repo=path_in_repo,
                repo_id=repo_id,
                repo_type="dataset"
            )
            log_message(f"✅ Uploaded: {os.path.basename(local_path)}")
            return True
        except HfHubHTTPError as e:
            if e.response.status_code == 429 and attempt < max_retries - 1: # Too Many Requests
                delay = initial_delay * (2 ** attempt)
                log_message(f"⚠️ Rate limit hit. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})")
                time.sleep(delay)
            else:
                log_message(f"❌ Hugging Face Hub error uploading {os.path.basename(local_path)}: {e}")
                return False
        except Exception as e:
            log_message(f"❌ Error uploading {os.path.basename(local_path)}: {e}")
            return False
    log_message(f"❌ Failed to upload {os.path.basename(local_path)} after {max_retries} attempts.")
    return False

def process_video_frames_and_upload(extracted_rar_folder: str, processed_video_courses_set: Set[str]) -> bool:
    """Scans an extracted RAR folder for MP4s, extracts frames, zips, and uploads to BG3."""
    video_processed_successfully = False
    
    # Use the top-level folder name of the extracted RAR as the course folder name
    course_folder_name = os.path.basename(extracted_rar_folder)

    if course_folder_name in processed_video_courses_set:
        log_message(f"⏩ Video frames for course \'{course_folder_name}\' already processed. Skipping.")
        return True

    log_message(f"🎬 Processing videos in extracted RAR folder: {course_folder_name}")
    
    mp4_files_found = []
    for root, _, files in os.walk(extracted_rar_folder):
        for file in files:
            if file.lower().endswith(".mp4"):
                mp4_files_found.append(os.path.join(root, file))

    if not mp4_files_found:
        log_message(f"⚠️ No MP4 files found in {course_folder_name}. Skipping video frame extraction.")
        return False # Indicate no video processing was done

    course_video_extract_dir = os.path.join(VIDEO_FRAMES_EXTRACT_FOLDER, course_folder_name)
    os.makedirs(course_video_extract_dir, exist_ok=True)

    for mp4_path in mp4_files_found:
        video_basename = os.path.splitext(os.path.basename(mp4_path))[0]
        # Create a unique output folder for frames from this video within the course's frame directory
        video_output_folder = os.path.join(course_video_extract_dir, video_basename)
        
        if not extract_frames(mp4_path, video_output_folder, VIDEO_FRAME_FPS):
            log_message(f"❌ Failed to extract frames from {os.path.basename(mp4_path)}. Continuing with other videos.")
            # Clean up partially extracted frames for this video
            if os.path.exists(video_output_folder):
                shutil.rmtree(video_output_folder)

    # Check if any frames were extracted for the entire course folder
    if not os.listdir(course_video_extract_dir):
        log_message(f"⚠️ No frames extracted for any video in {course_folder_name}. Skipping zipping and upload to BG3.")
        shutil.rmtree(course_video_extract_dir)
        return False

    course_zip_path = os.path.join(ZIPPED_FRAMES_FOLDER, f"{course_folder_name}_frames.zip")
    if zip_folder(course_video_extract_dir, course_zip_path):
        path_in_repo = f"{course_folder_name}_frames.zip"
        if upload_file_to_hf(course_zip_path, path_in_repo, DEST_REPO_ID_VIDEO):
            log_message(f"✅ Successfully processed video frames and uploaded {course_folder_name}_frames.zip to BG3")
            processed_video_courses_set.add(course_folder_name) # Mark as processed
            save_processed_video_courses(processed_video_courses_set) # Save state
            video_processed_successfully = True
        else:
            log_message(f"❌ Failed to upload zipped video frames for {course_folder_name} to BG3")
    else:
        log_message(f"❌ Failed to zip video frames for {course_folder_name}")

    # Cleanup local video frame files
    log_message(f"🧹 Cleaning up local video frame files for {course_folder_name}")
    if os.path.exists(course_video_extract_dir):
        shutil.rmtree(course_video_extract_dir)
    if os.path.exists(course_zip_path):
        os.remove(course_zip_path)

    return video_processed_successfully

def extract_and_upload_rar(rar_path: str, processed_rars_set: set, uploaded_folders_set: Set[str], processed_video_courses_set: Set[str]) -> bool:
    """Extracts a single RAR file, uploads its contents to BG2 (if DEST_REPO_ID_RAR is set), and then processes videos for BG3"""
    filename = os.path.basename(rar_path)
    processing_status["current_file"] = filename

    folder_name = filename.replace(".rar", "")
    current_extract_folder = os.path.join(EXTRACT_FOLDER, f"{folder_name}_extracted")

    # Check if RAR is already processed (uploaded to BG2 or video frames processed)
    if filename in processed_rars_set and (not DEST_REPO_ID_RAR or get_folder_hash(folder_name) in uploaded_folders_set) and folder_name in processed_video_courses_set:
        log_message(f"⏩ {filename} already fully processed, skipping.")
        return True

    # Generate folder name and hash for tracking
    folder_hash = get_folder_hash(folder_name)
    
    # If BG2 upload is enabled and folder already uploaded to BG2, skip RAR extraction/upload to BG2
    if DEST_REPO_ID_RAR and folder_hash in uploaded_folders_set:
        log_message(f"🔒 Folder \'{folder_name}\' already uploaded to BG2 (hash: {folder_hash[:8]}...), skipping RAR upload.")
        processed_rars_set.add(filename)
        save_processed_files_state(processed_rars_set)
        # Proceed to video processing if not already done
        if not os.path.exists(current_extract_folder): # If extracted folder doesn't exist, we can't process videos
            log_message(f"⚠️ Extracted folder {current_extract_folder} not found for video processing. Skipping.")
            return False
        process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)
        return True

    log_message(f"📦 Attempting to extract: {filename}")
    os.makedirs(current_extract_folder, exist_ok=True)

    try:
        if shutil.which("unrar") is None:
            raise RuntimeError("unrar command not found. Please install unrar.")

        # Use -idq to suppress query messages and -o+ to overwrite without prompting
        unrar_command = ["unrar", "x", "-o+", rar_path, current_extract_folder]
        log_message(f"Running command: {' '.join(unrar_command)}")

        result = subprocess.run(
            unrar_command,
            check=True,
            capture_output=True,
            text=True,
            encoding='utf-8'
        )

        extracted_contents = os.listdir(current_extract_folder)
        if not extracted_contents:
            raise Exception("Extraction completed but no files were produced in the target directory.")
        log_message(f"Successfully extracted {len(extracted_contents)} items")

        # Upload extracted files to BG2 if DEST_REPO_ID_RAR is set
        if DEST_REPO_ID_RAR:
            upload_count = 0
            for root, _, files in os.walk(current_extract_folder):
                for file in files:
                    local_path = os.path.join(root, file)
                    # Construct path in repo relative to the extracted content's root
                    path_in_repo = os.path.join(folder_name, os.path.relpath(local_path, current_extract_folder))
                    log_message(f"⬆️ Uploading to BG2: {path_in_repo}")
                    
                    try:
                        upload_file_to_hf(
                            local_path=local_path,
                            path_in_repo=path_in_repo,
                            repo_id=DEST_REPO_ID_RAR
                        )
                        upload_count += 1
                    except Exception as upload_error:
                        log_message(f"❌ Failed to upload {path_in_repo} to BG2: {upload_error}")
                        raise upload_error
            
            log_message(f"✅ Successfully uploaded {upload_count} files from {filename} to BG2")
            
            # Mark folder as uploaded to BG2 using hash
            uploaded_folders_set.add(folder_hash)
            save_uploaded_folders(uploaded_folders_set)
            processing_status["uploaded_rar_folders"] = len(uploaded_folders_set)
            
            log_message(f"🔒 Folder \'{folder_name}\' locked in BG2 repo (hash: {folder_hash[:8]}...)")
        else:
            log_message("Skipping upload to BG2 as DEST_REPO_ID_RAR is not set.")
        
        # Now process video frames from the extracted content and upload to BG3
        process_video_frames_and_upload(current_extract_folder, processed_video_courses_set)

        return True

    except subprocess.CalledProcessError as e:
        error_msg = f"RAR extraction failed (exit {e.returncode}): {e.stderr.strip()}"
        log_failed_file(filename, error_msg)
        processing_status["failed_files"] += 1
        return False
    except Exception as e:
        error_msg = f"Unexpected error during processing {filename}: {str(e)}"
        log_failed_file(filename, error_msg)
        processing_status["failed_files"] += 1
        return False
    finally:
        # Always cleanup the extraction folder after processing (success or failure)
        if os.path.exists(current_extract_folder):
            log_message(f"🧹 Cleaning up extracted RAR files in {current_extract_folder}")
            try:
                shutil.rmtree(current_extract_folder)
                log_message(f"✅ Cleaned up RAR extraction folder")
            except Exception as e:
                log_message(f"⚠️ Could not clean up RAR extraction folder {current_extract_folder}: {e}")

def continuous_processing(start_download_index: Optional[int] = None):
    """Main processing loop that runs continuously, with an optional starting download index"""
    processing_status["is_running"] = True
    log_message("🚀 Starting continuous RAR and Video processing...")
    
    try:
        # Load uploaded folders tracking for BG2
        uploaded_folders = load_uploaded_folders()
        processing_status["uploaded_rar_folders"] = len(uploaded_folders)

        # Load processed video courses tracking for BG3
        processed_video_courses = load_processed_video_courses()
        processing_status["uploaded_video_courses"] = len(processed_video_courses)
        
        if start_download_index is not None:
            log_message(f"Starting download from index: {start_download_index}")
            save_download_state(start_download_index) # Set download state to start from this index
        else:
            log_message("Starting download from saved state or beginning.")

        while processing_status["is_running"]:
            # 1. Download a batch of RAR files
            download_start_index = load_download_state()
            downloaded_rar_paths, next_download_index = download_rar_files(download_start_index, CHUNK_SIZE)
            save_download_state(next_download_index)

            if not downloaded_rar_paths:
                # Check if there are any local RAR files to process
                all_local_rars = sorted([os.path.join(DOWNLOAD_FOLDER, f) for f in os.listdir(DOWNLOAD_FOLDER) if f.endswith(".rar")])
                processed_rars = load_processed_files_state()
                unprocessed_rars = [rar for rar in all_local_rars if os.path.basename(rar) not in processed_rars]
                
                if not unprocessed_rars:
                    log_message("✅ No more RAR files to download. Stopping...")
                    break
                else:
                    log_message(f"📋 Found {len(unprocessed_rars)} unprocessed local RAR files")
            
            # 2. Process all available RAR files (downloaded + existing)
            all_local_rars = sorted([os.path.join(DOWNLOAD_FOLDER, f) for f in os.listdir(DOWNLOAD_FOLDER) if f.endswith(".rar")])
            processed_rars = load_processed_files_state()
            processing_status["total_files"] = len(all_local_rars)
            processing_status["processed_files"] = len(processed_rars)

            for rar_file_path in all_local_rars:
                if not processing_status["is_running"]:
                    break
                    
                filename = os.path.basename(rar_file_path)
                if filename not in processed_rars:
                    success = extract_and_upload_rar(rar_file_path, processed_rars, uploaded_folders, processed_video_courses)
                    if success:
                        processed_rars.add(filename)
                        save_processed_files_state(processed_rars)
                        processing_status["processed_files"] += 1
                        
                        # Delete the RAR file after successful processing
                        log_message(f"🗑️ Deleting processed RAR: {filename}")
                        try:
                            os.remove(rar_file_path)
                            log_message(f"✅ Deleted RAR file: {filename}")
                        except Exception as e:
                            log_message(f"⚠️ Could not delete {rar_file_path}: {e}")
                    
                    # Add delay between processing files
                    time.sleep(PROCESSING_DELAY)

            # If no new files were downloaded and all local files are processed, we're done
            if not downloaded_rar_paths:
                break
                
    except Exception as e:
        log_message(f"❌ Error in continuous processing: {e}")
    finally:
        processing_status["is_running"] = False
        processing_status["current_file"] = None
        log_message("🏁 Processing stopped")

@app.get("/", response_class=HTMLResponse)
async def root():
    """Serve the main HTML interface"""
    html_content = """
    <!DOCTYPE html>
    <html>
    <head>
        <title>RAR & Video Processing Service</title>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <style>
            body { font-family: Arial, sans-serif; margin: 20px; background-color: #f5f5f5; }
            .container { max-width: 1200px; margin: 0 auto; background: white; padding: 20px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
            .status-card { background: #e3f2fd; padding: 15px; border-radius: 5px; margin: 10px 0; }
            .logs { background: #f5f5f5; padding: 15px; border-radius: 5px; height: 400px; overflow-y: auto; font-family: monospace; font-size: 12px; }
            .button { background: #2196F3; color: white; padding: 10px 20px; border: none; border-radius: 5px; cursor: pointer; margin: 5px; }
            .button:hover { background: #1976D2; }
            .button:disabled { background: #ccc; cursor: not-allowed; }
            .stop-button { background: #f44336; }
            .stop-button:hover { background: #d32f2f; }
            .stats { display: flex; gap: 20px; margin: 20px 0; }
            .stat-item { background: #f0f0f0; padding: 10px; border-radius: 5px; text-align: center; flex: 1; }
            .start-form { margin-top: 20px; padding: 15px; border: 1px solid #ddd; border-radius: 5px; background: #f9f9f9; }
            .start-form input[type="number"] { width: calc(100% - 120px); padding: 8px; margin-right: 10px; border: 1px solid #ccc; border-radius: 4px; }
            .start-form button { padding: 8px 15px; background: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer; }
            .start-form button:hover { background: #45a049; }
        </style>
    </head>
    <body>
        <div class="container">
            <h1>🔄 RAR & Video Processing Service</h1>
            <p>Automated extraction and upload of RAR files from BG1 to BG2 dataset, and video frame extraction/upload to BG3 dataset</p>
            
            <div class="status-card">
                <h3>Status: <span id="status">Stopped</span></h3>
                <p>Current File: <span id="current-file">None</span></p>
                <p>Last Update: <span id="last-update">Never</span></p>
            </div>
            
            <div class="stats">
                <div class="stat-item">
                    <h4>Total Files (RARs)</h4>
                    <span id="total-files">0</span>
                </div>
                <div class="stat-item">
                    <h4>Processed (RARs)</h4>
                    <span id="processed-files">0</span>
                </div>
                <div class="stat-item">
                    <h4>Uploaded Folders (BG2)</h4>
                    <span id="uploaded-rar-folders">0</span>
                </div>
                <div class="stat-item">
                    <h4>Uploaded Video Courses (BG3)</h4>
                    <span id="uploaded-video-courses">0</span>
                </div>
                <div class="stat-item">
                    <h4>Failed</h4>
                    <span id="failed-files">0</span>
                </div>
            </div>
            
            <div class="start-form">
                <h3>Start Processing from Specific Download Index</h3>
                <input type="number" id="start-index-input" placeholder="Enter start index (e.g., 0)" value="0">
                <button onclick="startProcessingWithIndex()">Start from Index</button>
            </div>

            <div>
                <button class="button" onclick="startProcessing()" id="start-btn">Start Processing (from last saved index)</button>
                <button class="button stop-button" onclick="stopProcessing()" id="stop-btn" disabled>Stop Processing</button>
                <button class="button" onclick="refreshStatus()">Refresh Status</button>
            </div>
            
            <h3>Logs</h3>
            <div class="logs" id="logs">Loading...</div>
        </div>
        
        <script>
            async function startProcessing() {
                try {
                    const response = await fetch(\"/start\", { method: \"POST\" });
                    const result = await response.json();
                    alert(result.message);
                    refreshStatus();
                } catch (error) {
                    alert(\"Error starting processing: \" + error.message);
                }
            }

            async function startProcessingWithIndex() {
                const index = document.getElementById(\"start-index-input\").value;
                if (index === "" || isNaN(index)) {
                    alert(\"Please enter a valid number for the start index.\");
                    return;
                }
                try {
                    const response = await fetch(\"/start_from_index\", {
                        method: \"POST\",
                        headers: { \"Content-Type\": \"application/x-www-form-urlencoded\" },
                        body: `start_index=${parseInt(index)}`
                    });
                    const result = await response.json();
                    alert(result.message);
                    refreshStatus();
                } catch (error) {
                    alert(\"Error starting processing from index: \" + error.message);
                }
            }
            
            async function stopProcessing() {
                try {
                    const response = await fetch(\"/stop\", { method: \"POST\" });
                    const result = await response.json();
                    alert(result.message);
                    refreshStatus();
                } catch (error) {
                    alert(\"Error stopping processing: \" + error.message);
                }
            }
            
            async function refreshStatus() {
                try {
                    const response = await fetch(\"/status\");
                    const status = await response.json();
                    
                    document.getElementById(\"status\").textContent = status.is_running ? \"Running\" : \"Stopped\";
                    document.getElementById(\"current-file\").textContent = status.current_file || \"None\";
                    document.getElementById(\"last-update\").textContent = status.last_update || \"Never\";
                    document.getElementById(\"total-files\").textContent = status.total_files;
                    document.getElementById(\"processed-files\").textContent = status.processed_files;
                    document.getElementById(\"uploaded-rar-folders\").textContent = status.uploaded_rar_folders;
                    document.getElementById(\"uploaded-video-courses\").textContent = status.uploaded_video_courses;
                    document.getElementById(\"failed-files\").textContent = status.failed_files;
                    
                    document.getElementById(\"start-btn\").disabled = status.is_running;
                    document.getElementById(\"stop-btn\").disabled = !status.is_running;
                    
                    const logsDiv = document.getElementById(\"logs\");
                    logsDiv.innerHTML = status.logs.join(\"<br>\");
                    logsDiv.scrollTop = logsDiv.scrollHeight;
                } catch (error) {
                    console.error(\"Error refreshing status:\", error);
                }
            }
            
            // Auto-refresh every 5 seconds
            setInterval(refreshStatus, 5000);
            
            // Initial load
            refreshStatus();
        </script>
    </body>
    </html>
    """
    return HTMLResponse(content=html_content)

@app.get("/status")
async def get_status():
    """Get current processing status"""
    return JSONResponse(content=processing_status)

@app.post("/start")
async def start_processing(background_tasks: BackgroundTasks):
    """Start the processing in background"""
    if processing_status["is_running"]:
        return {"message": "Processing is already running"}
    
    background_tasks.add_task(continuous_processing)
    return {"message": "Processing started"}

@app.post("/start_from_index")
async def start_processing_from_index(background_tasks: BackgroundTasks, start_index: int = Form(...)):
    """Start the processing from a specific download index in background"""
    if processing_status["is_running"]:
        return {"message": "Processing is already running"}
    
    if start_index < 0:
        return {"message": "Start index cannot be negative."}

    background_tasks.add_task(continuous_processing, start_download_index=start_index)
    return {"message": f"Processing started from download index: {start_index}"}

@app.post("/stop")
async def stop_processing():
    """Stop the processing"""
    if not processing_status["is_running"]:
        return {"message": "Processing is not running"}
    
    processing_status["is_running"] = False
    return {"message": "Processing stop requested"}

@app.get("/logs")
async def get_logs():
    """Get processing logs"""
    return {"logs": processing_status["logs"]}

@app.get("/uploaded-folders")
async def get_uploaded_folders():
    """Get list of uploaded folder hashes for BG2"""
    uploaded_folders = load_uploaded_folders()
    return {"uploaded_folder_count": len(uploaded_folders), "folder_hashes": list(uploaded_folders)}

@app.get("/processed-video-courses")
async def get_processed_video_courses():
    """Get list of processed video course folder names for BG3"""
    processed_video_courses = load_processed_video_courses()
    return {"processed_video_course_count": len(processed_video_courses), "course_names": list(processed_video_courses)}

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)