feat: setup secrets management, logging infrastructure, and code quality tools

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.4
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-added-large-files
+        args: ['--maxkb=50000']
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: check-json
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

Makefile

@@ -0,0 +1,25 @@
+.PHONY: lint format test test-all clean verify run-dev
+
+lint:
+	ruff check backend/
+	black --check backend/
+	mypy backend/core/
+
+format:
+	ruff check --fix backend/
+	black backend/
+
+test:
+	pytest -m "unit" --tb=short
+
+test-all:
+	pytest --tb=short
+
+clean:
+	del /s /q __pycache__ .pytest_cache .ruff_cache
+
+verify:
+	python verify_env.py
+
+run-dev:
+	uvicorn backend.main:app --reload --port 8000

backend/core/config.py

@@ -0,0 +1,110 @@
+import os
+from pathlib import Path
+from typing import Literal
+from pydantic import field_validator, computed_field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+class Settings(BaseSettings):
+    """Core application settings populated from environment variables."""
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore"
+    )
+
+    # === Application ===
+    ENVIRONMENT: Literal["development", "production", "test"] = "development"
+    SECRET_KEY: str
+    DEBUG: bool = False
+    VERSION: str = "1.0.0"
+    ALLOWED_ORIGINS: list[str] = ["http://localhost:3000"]
+    FRONTEND_URL: str = "http://localhost:3000"
+    BACKEND_URL: str = "http://localhost:8000"
+
+    # === Database ===
+    DATABASE_URL: str = "sqlite+aiosqlite:///./medsight.db"
+
+    # === JWT ===
+    JWT_SECRET_KEY: str
+    JWT_SECRET_KEY_OLD: str | None = None  # Used for key rotation
+    JWT_ALGORITHM: str = "HS256"
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
+    REFRESH_TOKEN_EXPIRE_DAYS: int = 7
+
+    # === Google OAuth ===
+    GOOGLE_CLIENT_ID: str = ""
+    GOOGLE_CLIENT_SECRET: str = ""
+    GOOGLE_REDIRECT_URI: str = ""
+
+    # === HuggingFace ===
+    HF_TOKEN: str = ""
+
+    # === ML Config ===
+    MODEL_CACHE_DIR: Path = Path("C:/hf_cache")
+    TEMP_DIR: Path = Path("./backend/temp")
+    MAX_UPLOAD_SIZE_MB: int = 10
+    GPU_VRAM_BUDGET_MB: int = 3500
+
+    # === Rate Limiting ===
+    RATE_LIMIT_ANALYZE: str = "10/hour"
+    RATE_LIMIT_CHAT: str = "50/hour"
+    RATE_LIMIT_AUTH: str = "5/minute"
+
+    # === Logging ===
+    LOG_LEVEL: str = "DEBUG"
+    LOG_DIR: Path = Path("./backend/logs")
+
+    @field_validator("SECRET_KEY")
+    @classmethod
+    def secret_key_must_be_strong(cls, v: str) -> str:
+        if len(v) < 32:
+            raise ValueError("SECRET_KEY must be at least 32 characters")
+        return v
+
+    @field_validator("JWT_SECRET_KEY")
+    @classmethod
+    def jwt_key_must_be_strong(cls, v: str) -> str:
+        if len(v) < 32:
+            raise ValueError("JWT_SECRET_KEY must be at least 32 characters")
+        return v
+
+    @computed_field
+    @property
+    def is_production(self) -> bool:
+        return self.ENVIRONMENT == "production"
+
+    @computed_field
+    @property
+    def max_upload_bytes(self) -> int:
+        return self.MAX_UPLOAD_SIZE_MB * 1024 * 1024
+
+    def __repr__(self) -> str:
+        # NEVER show secrets in repr to prevent accidental logging
+        return f"Settings(environment={self.ENVIRONMENT}, debug={self.DEBUG}, version={self.VERSION})"
+
+# Singleton — import this everywhere
+settings = Settings()
+
+def startup_validation():
+    """Validates critical infrastructure at startup."""
+    errors = []
+    
+    try:
+        settings.TEMP_DIR.mkdir(parents=True, exist_ok=True)
+        settings.LOG_DIR.mkdir(parents=True, exist_ok=True)
+        settings.MODEL_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    except PermissionError:
+        errors.append("Lack permissions to create required directories (temp, logs, cache).")
+
+    if not settings.DATABASE_URL:
+        errors.append("DATABASE_URL is not set.")
+
+    if settings.is_production:
+        if not settings.GOOGLE_CLIENT_ID:
+            errors.append("GOOGLE_CLIENT_ID is required in production.")
+        if settings.DEBUG:
+            errors.append("DEBUG mode must be False in production.")
+
+    if errors:
+        raise RuntimeError("Startup Validation Failed:\n" + "\n".join(errors))

backend/core/logging_config.py

@@ -0,0 +1,131 @@
+import json
+import logging
+from logging.handlers import RotatingFileHandler
+from datetime import datetime, UTC
+from contextvars import ContextVar
+from uuid import uuid4
+
+from backend.core.config import settings
+
+# Context var for async-safe request ID tracking
+_request_id_var: ContextVar[str] = ContextVar("request_id", default="")
+
+def get_request_id() -> str:
+    return _request_id_var.get()
+
+class MaskingFilter(logging.Filter):
+    SENSITIVE_FIELDS = {"password", "token", "secret", "key", "authorization", "cookie", "api_key"}
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Mask sensitive data in extra dict if present
+        if hasattr(record, "extra") and isinstance(record.extra, dict):
+            for k in record.extra.keys():
+                if any(sens in k.lower() for sens in self.SENSITIVE_FIELDS):
+                    record.extra[k] = "***MASKED***"
+        return True
+
+class JSONFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        log_dict = {
+            "timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "request_id": get_request_id(),
+            "module": record.module,
+            "function": record.funcName,
+            "line": record.lineno,
+            "environment": settings.ENVIRONMENT
+        }
+        if record.exc_info:
+            log_dict["exception"] = self.formatException(record.exc_info)[:500]
+        
+        # Merge extra fields
+        for key, value in record.__dict__.items():
+            if key not in ["args", "asctime", "created", "exc_info", "exc_text", "filename", "funcName", "levelname", "levelno", "lineno", "module", "msecs", "message", "msg", "name", "pathname", "process", "processName", "relativeCreated", "stack_info", "thread", "threadName"]:
+                log_dict[key] = value
+
+        return json.dumps(log_dict)
+
+class ColoredConsoleFormatter(logging.Formatter):
+    COLORS = {
+        'DEBUG': '\033[94m', 'INFO': '\033[92m', 'WARNING': '\033[93m', 
+        'ERROR': '\033[91m', 'CRITICAL': '\033[95m'
+    }
+    RESET = '\033[0m'
+
+    def format(self, record: logging.LogRecord) -> str:
+        color = self.COLORS.get(record.levelname, self.RESET)
+        time_str = datetime.fromtimestamp(record.created).strftime('%H:%M:%S')
+        req_id = get_request_id()
+        req_str = f" [{req_id[:8]}]" if req_id else ""
+        return f"{time_str} {color}[{record.levelname}]{self.RESET} {record.module}:{record.lineno}{req_str} — {record.getMessage()}"
+
+def setup_logging():
+    settings.LOG_DIR.mkdir(parents=True, exist_ok=True)
+    root_logger = logging.getLogger()
+    root_logger.setLevel(settings.LOG_LEVEL)
+    
+    # Clear existing handlers
+    root_logger.handlers.clear()
+
+    # Filters
+    masking_filter = MaskingFilter()
+
+    # Handlers
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(ColoredConsoleFormatter() if not settings.is_production else JSONFormatter())
+    console_handler.addFilter(masking_filter)
+
+    app_file = RotatingFileHandler(settings.LOG_DIR / "app.log", maxBytes=10*1024*1024, backupCount=5)
+    app_file.setFormatter(JSONFormatter())
+    app_file.addFilter(masking_filter)
+
+    err_file = RotatingFileHandler(settings.LOG_DIR / "error.log", maxBytes=10*1024*1024, backupCount=5)
+    err_file.setLevel(logging.ERROR)
+    err_file.setFormatter(JSONFormatter())
+    err_file.addFilter(masking_filter)
+
+    ml_file = RotatingFileHandler(settings.LOG_DIR / "ml.log", maxBytes=10*1024*1024, backupCount=5)
+    ml_file.setFormatter(JSONFormatter())
+    ml_file.addFilter(lambda r: "ml" in r.name)
+
+    access_file = RotatingFileHandler(settings.LOG_DIR / "access.log", maxBytes=10*1024*1024, backupCount=5)
+    access_file.setFormatter(JSONFormatter())
+    access_file.addFilter(lambda r: "access" in r.name)
+
+    root_logger.addHandler(console_handler)
+    root_logger.addHandler(app_file)
+    root_logger.addHandler(err_file)
+    logging.getLogger("ml").addHandler(ml_file)
+    logging.getLogger("access").addHandler(access_file)
+
+    # Suppress noise
+    logging.getLogger("uvicorn.access").propagate = False
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("chromadb").setLevel(logging.WARNING)
+
+class MLLogger:
+    def __init__(self):
+        self.logger = logging.getLogger("ml")
+
+    def log_model_load(self, name: str, device: str, load_time_ms: int, vram_delta_mb: int = None):
+        self.logger.info("model_loaded", extra={
+            "model_name": name, "device": device, "load_time_ms": load_time_ms, "vram_delta_mb": vram_delta_mb
+        })
+
+    def log_inference(self, name: str, inference_time_ms: int, input_summary: dict, output_summary: dict):
+        self.logger.info("inference_complete", extra={
+            "model_name": name, "inference_time_ms": inference_time_ms, "input": input_summary, "output": output_summary
+        })
+
+    def log_checkpoint(self, epoch: int, loss: float, val_loss: float, path: str):
+        self.logger.info("checkpoint_saved", extra={"epoch": epoch, "loss": loss, "val_loss": val_loss, "path": path})
+
+    def log_oom(self, model_name: str, batch_size: int, vram_available_mb: int):
+        self.logger.error("cuda_oom", extra={"model_name": model_name, "batch_size": batch_size, "vram_available_mb": vram_available_mb})
+
+    def log_pipeline_step(self, step_name: str, status: str, duration_ms: int, session_id: str):
+        self.logger.info("pipeline_step", extra={"step": step_name, "status": status, "duration_ms": duration_ms, "session_id": session_id})
+
+ml_logger = MLLogger()

backend/pyproject.toml

@@ -0,0 +1,47 @@
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+# Select critical rule sets: Errors, Warnings, Pyflakes, isort, PEP8-naming, pyupgrade, flake8-bugbear, flake8-builtins, flake8-comprehensions, pytest-style, Ruff-specific
+select = ["E", "W", "F", "I", "N", "UP", "B", "A", "C4", "PT", "RUF"]
+
+ignore = [
+    "B008",  # Do not perform function calls in default args (Required for FastAPI Depends)
+    "A003",  # Shadowing Python built-in (id is fine for DB models)
+    "N818",  # Exception names should end in Error (We use custom hierarchy names)
+]
+
+[tool.ruff.per-file-ignores]
+"tests/*" = ["S101"]         # Assert is fine in tests
+"db/migrations/*" = ["E501"] # Migration auto-generations can be long
+
+[tool.black]
+line-length = 100
+target-version = ["py310"]
+include = '\.pyi?$'
+exclude = "migrations"
+
+[tool.mypy]
+python_version = "3.10"
+strict = false
+warn_return_any = true
+warn_unused_imports = true
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = ["torch.*", "transformers.*", "chromadb.*", "cv2.*", "PIL.*"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+    "unit: fast isolated tests",
+    "integration: tests using real DB or filesystem",
+    "ml: tests that load ML models",
+    "slow: tests taking more than 5 seconds",
+    "e2e: full end-to-end tests"
+]
+filterwarnings = ["ignore::DeprecationWarning", "ignore::PendingDeprecationWarning"]
+log_cli = true
+log_cli_level = "INFO"

frontend/.eslintrc.json

@@ -0,0 +1,8 @@
+{
+  "extends": "next/core-web-vitals",
+  "rules": {
+    "no-unused-vars": "error",
+    "no-console": "warn",
+    "react-hooks/exhaustive-deps": "warn"
+  }
+}

frontend/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "tabWidth": 2,
+  "singleQuote": true,
+  "semi": true,
+  "trailingComma": "es5",
+  "printWidth": 100
+}