services/ai-service/test_gguf_loading.py

"""
Test script to debug GGUF model loading
"""
import sys
import os

# Add the src directory to the path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))

def test_gguf_loading():
    """Test GGUF model loading with the exact parameters from the user's request"""
    
    model_name = "microsoft/Phi-3-mini-4k-instruct-gguf/Phi-3-mini-4k-instruct-q4.gguf"
    model_type = "gguf"
    
    print(f"Testing GGUF model loading...")
    print(f"Model name: {model_name}")
    print(f"Model type: {model_type}")
    print("-" * 80)
    
    try:
        from ai_med_extract.utils.unified_model_manager import unified_model_manager, GenerationConfig
        
        print("\n1. Getting model from unified_model_manager...")
        model = unified_model_manager.get_model(
            name=model_name,
            model_type=model_type,
            lazy=False  # Force immediate loading
        )
        
        print(f"   Model object created: {model}")
        print(f"   Model status: {model.status}")
        print(f"   Model filename: {model.filename if hasattr(model, 'filename') else 'N/A'}")
        
        if model.status.value == "error":
            print(f"   ERROR: {model._error_message}")
            return False
        
        print("\n2. Testing generation...")
        test_prompt = "Hello, how are you?"
        config = GenerationConfig(max_tokens=50, temperature=0.7)
        
        result = model.generate(test_prompt, config)
        print(f"   Generation successful!")
        print(f"   Result: {result[:100]}...")
        
        return True
        
    except Exception as e:
        print(f"\nERROR during testing:")
        print(f"   Type: {type(e).__name__}")
        print(f"   Message: {str(e)}")
        import traceback
        traceback.print_exc()
        return False

if __name__ == "__main__":
    success = test_gguf_loading()
    sys.exit(0 if success else 1)