""" comparacao_bench.py Avaliação do sistema de com comparação directa à Table 5 (Protocol-3) do paper DF40 — linha EFS (FF). Gera gráficos com fundo branco prontos para relatório académico. """ import json import numpy as np import pandas as pd import matplotlib import matplotlib.pyplot as plt import matplotlib.patches as mpatches from pathlib import Path from sklearn.linear_model import LogisticRegression from sklearn.metrics import roc_auc_score, accuracy_score, roc_curve, auc from sklearn.model_selection import StratifiedKFold, train_test_split matplotlib.rcParams.update({ "font.family": "DejaVu Sans", "axes.spines.top": False, "axes.spines.right": False, }) CSV_PATH = Path("estudo_redes/Teste_Redes.csv") RESULTS_DIR = Path("results") RESULTS_DIR.mkdir(exist_ok=True) # ══════════════════════════════════════════════════════════════════════ # MAPEAMENTO: prefixo do ficheiro → nome canónico da Table 5 # ══════════════════════════════════════════════════════════════════════ GENERATOR_MAP = { # Coincidências directas com Table 5 "midjourney": "MidJourney-6", "whichfaceisrea": "Whichisreal", "stargan": "StarGAN", "starganv2": "StarGAN2", "styleclip": "StyleCLIP", "e4e": "e4e", "collabdif": "CollabDiff", # EFS adicionais (não na Table 5 — reportados em separado) "vqgan": "VQGAN", "stylegan2": "StyleGAN2", "stylegan3": "StyleGAN3", "styleganxl": "StyleGAN-XL", "sd2.1": "SD-v2.1", "ddim": "DDIM", "rddm": "RDDM", "pixart": "PixArt-α", "dit": "DiT", "sit": "SiT", } # Geradores que coincidem com Table 5 (os 7 disponíveis) TABLE5_GENS = [ "MidJourney-6", "Whichisreal", "StarGAN", "StarGAN2", "StyleCLIP", "e4e", "CollabDiff", ] # Valores da Table 5, Protocol-3, linha EFS (FF) — paper DF40 LITERATURA_PER_GEN = { "MidJourney-6": {"Xception":0.472, "CLIP":0.534, "SRM":0.338, "SPSL":0.427, "RECCE":0.442, "RFM":0.551}, "Whichisreal": {"Xception":0.772, "CLIP":0.828, "SRM":0.794, "SPSL":0.694, "RECCE":0.753, "RFM":0.623}, "StarGAN": {"Xception":0.777, "CLIP":0.946, "SRM":0.769, "SPSL":0.699, "RECCE":0.769, "RFM":0.730}, "StarGAN2": {"Xception":0.677, "CLIP":0.823, "SRM":0.703, "SPSL":0.723, "RECCE":0.724, "RFM":0.636}, "StyleCLIP": {"Xception":0.984, "CLIP":0.929, "SRM":0.982, "SPSL":0.922, "RECCE":0.964, "RFM":0.966}, "e4e": {"Xception":0.611, "CLIP":0.923, "SRM":0.509, "SPSL":0.602, "RECCE":0.643, "RFM":0.665}, "CollabDiff": {"Xception":0.997, "CLIP":0.983, "SRM":0.997, "SPSL":0.967, "RECCE":0.979, "RFM":0.979}, } # ══════════════════════════════════════════════════════════════════════ # CARREGAR CSV # ══════════════════════════════════════════════════════════════════════ df = pd.read_csv(CSV_PATH) X = df[["prob_swin", "prob_df40", "z_score"]].values y = df["label"].values print(f"Dataset: {(y==0).sum()} reais | {(y==1).sum()} fakes | total {len(y)}") # ══════════════════════════════════════════════════════════════════════ # 1. SPLIT 80/20 # ══════════════════════════════════════════════════════════════════════ X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.20, stratify=y, random_state=42 ) clf = LogisticRegression(class_weight="balanced", solver="liblinear", max_iter=1000) clf.fit(X_train, y_train) y_probs_test = clf.predict_proba(X_test)[:, 1] auc_test = roc_auc_score(y_test, y_probs_test) acc_test = accuracy_score(y_test, (y_probs_test >= 0.5).astype(int)) print(f"\n── Split 80/20 ──") print(f"AUC (teste) : {auc_test:.4f}") print(f"Accuracy (teste): {acc_test*100:.2f}%") print(f"Train/Test : {len(X_train)} / {len(X_test)}") # ══════════════════════════════════════════════════════════════════════ # 2. 5-FOLD CROSS-VALIDATION # ══════════════════════════════════════════════════════════════════════ skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) aucs_cv, accs_cv = [], [] for train_idx, val_idx in skf.split(X, y): clf_cv = LogisticRegression(class_weight="balanced", solver="liblinear", max_iter=1000) clf_cv.fit(X[train_idx], y[train_idx]) probs = clf_cv.predict_proba(X[val_idx])[:, 1] aucs_cv.append(roc_auc_score(y[val_idx], probs)) accs_cv.append(accuracy_score(y[val_idx], (probs >= 0.5).astype(int))) auc_cv_mean = np.mean(aucs_cv) auc_cv_std = np.std(aucs_cv) print(f"\n── 5-Fold Cross-Validation ──") for i, (a, acc) in enumerate(zip(aucs_cv, accs_cv)): print(f" Fold {i+1}: AUC={a:.4f} Acc={acc*100:.2f}%") print(f" Média : AUC={auc_cv_mean:.4f} ± {auc_cv_std:.4f}") # ══════════════════════════════════════════════════════════════════════ # 3. AUC POR GERADOR (extraído do prefixo do nome do ficheiro) # ══════════════════════════════════════════════════════════════════════ clf_full = LogisticRegression(class_weight="balanced", solver="liblinear", max_iter=1000) clf_full.fit(X, y) df["prob_pred"] = clf_full.predict_proba(X)[:, 1] df["generator_raw"] = df["img_name"].apply( lambda n: str(n).split("_")[0].lower() if pd.notna(n) else "unknown" ) df["generator"] = df["generator_raw"].map(GENERATOR_MAP).fillna(df["generator_raw"]) reais_probs = df.loc[df["label"] == 0, "prob_pred"].values our_per_gen = {} print(f"\n── AUC por Gerador ──") for raw, gen_name in GENERATOR_MAP.items(): mask = (df["label"] == 1) & (df["generator_raw"] == raw) if mask.sum() < 5: continue fake_probs = df.loc[mask, "prob_pred"].values y_bin = np.array([0]*len(reais_probs) + [1]*len(fake_probs)) p_bin = np.concatenate([reais_probs, fake_probs]) auc_gen = roc_auc_score(y_bin, p_bin) our_per_gen[gen_name] = auc_gen marker = "← Table 5" if gen_name in TABLE5_GENS else "" print(f" {gen_name:<18} AUC: {auc_gen:.4f} (n={mask.sum()}) {marker}") # Médias separadas our_t5 = [our_per_gen[g] for g in TABLE5_GENS if g in our_per_gen] our_add = [v for k, v in our_per_gen.items() if k not in TABLE5_GENS] avg_t5 = np.mean(our_t5) if our_t5 else float("nan") avg_add = np.mean(our_add) if our_add else float("nan") print(f"\n Avg. (7 gen. Table 5) : {avg_t5:.4f}") print(f" Avg. (adic. não T5) : {avg_add:.4f}") # ══════════════════════════════════════════════════════════════════════ # 4. TABELA IMPRESSA — comparação por gerador com Table 5 # ══════════════════════════════════════════════════════════════════════ # ── Tabela: linhas = modelos, colunas = geradores + Avg ────────────── modelos_ordem = ["Xception", "CLIP", "SRM", "SPSL", "RECCE", "RFM", "Ours"] # Cabeçalho — nomes curtos para caber gen_labels = ["MidJou.", "Whichis.", "StarGAN", "StarGAN2", "StyleCLIP", "e4e", "CollabD.", "Avg."] col_w = 10 header = f"{'Model':<12}" + "".join(f"{g:>{col_w}}" for g in gen_labels) print(f"\n{'='*len(header)}") print("PROTOCOL-3 DF40 Table 5 — EFS Test") print(f"{'='*len(header)}") print(header) print(f"{'-'*len(header)}") for modelo in modelos_ordem: if modelo == "Ours": vals = [our_per_gen.get(g, float("nan")) for g in TABLE5_GENS] else: vals = [LITERATURA_PER_GEN.get(g, {}).get(modelo, float("nan")) for g in TABLE5_GENS] avg = np.nanmean(vals) row = f"{'Ours':<12}" if modelo == "Ours" else f"{modelo:<12}" row += "".join(f"{v:>{col_w}.3f}" for v in vals) row += f"{avg:>{col_w}.3f}" print(row) print(f"{'='*len(header)}") # ══════════════════════════════════════════════════════════════════════ # 5. GRÁFICOS — FUNDO BRANCO, PRONTOS PARA RELATÓRIO # ══════════════════════════════════════════════════════════════════════ AZUL = "#1d4ed8" # nosso sistema CINZA = "#94a3b8" # literatura VERDE = "#16a34a" # CV LIGH = "#e2e8f0" # fundo de barras fig = plt.figure(figsize=(18, 12), facecolor="white") fig.suptitle( "Comparação Indicativa — DFFD vs DF40 EFS+FE " "(backbones SwinV2/CLIP congelados) vs. DF40 (Protocol-3)", fontsize=9, fontweight="bold", color="#0f172a", y=0.99 ) gs = fig.add_gridspec(2, 2, hspace=0.42, wspace=0.35, left=0.08, right=0.97, top=0.93, bottom=0.07) ax_bar = fig.add_subplot(gs[0, 0]) # barras horizontais avg ax_roc = fig.add_subplot(gs[0, 1]) # curva ROC ax_gen = fig.add_subplot(gs[1, :]) # por gerador (grouped bars) # ───────────────────────────────────────────────────────────────────── # Gráfico 1 — Barras horizontais: Avg. Table 5 vs Nosso sistema # ───────────────────────────────────────────────────────────────────── avg_lit_per_modelo = { m: np.mean([LITERATURA_PER_GEN[g][m] for g in TABLE5_GENS]) for m in modelos_ordem[:-1] # exclui "Ours" } avg_lit_per_modelo["Ours \n(80/20 test)"] = auc_test avg_lit_per_modelo["Ours \n(5-fold CV)"] = auc_cv_mean labels_bar = list(avg_lit_per_modelo.keys()) vals_bar = list(avg_lit_per_modelo.values()) cores_bar = [CINZA] * len(modelos_ordem[:-1]) + [AZUL, VERDE] bars = ax_bar.barh(labels_bar, vals_bar, color=cores_bar, edgecolor="white", linewidth=0.6, height=0.6) ax_bar.axvline(0.5, color="#cbd5e1", linestyle="--", lw=1.2) ax_bar.set_xlim(0.35, 1.02) ax_bar.set_xlabel("AUC-ROC (média — 7 geradores)", fontsize=9, color="#334155") ax_bar.set_title("Avg. AUC por Modelo\n(Table 5, Protocol-3, EFS Test)", fontsize=10, fontweight="bold", color="#0f172a", pad=8) ax_bar.set_facecolor("white") ax_bar.tick_params(axis="y", labelsize=8.5) ax_bar.tick_params(axis="x", labelsize=8) ax_bar.xaxis.grid(True, color="#e2e8f0", linewidth=0.8, zorder=0) ax_bar.set_axisbelow(True) for bar, val, cor in zip(bars, vals_bar, cores_bar): bold = cor in [AZUL, VERDE] ax_bar.text(val + 0.006, bar.get_y() + bar.get_height() / 2, f"{val:.3f}", va="center", fontsize=8.5, fontweight="bold" if bold else "normal", color=AZUL if cor == AZUL else (VERDE if cor == VERDE else "#334155")) # ───────────────────────────────────────────────────────────────────── # Gráfico 2 — Curva ROC (conjunto de teste 20%) # ───────────────────────────────────────────────────────────────────── fpr, tpr, _ = roc_curve(y_test, y_probs_test) ax_roc.plot([0, 1], [0, 1], "--", color="#cbd5e1", lw=1.2, zorder=1) ax_roc.plot(fpr, tpr, color=AZUL, lw=2.5, zorder=3, label=f"Nosso Sistema (AUC = {auc_test:.3f})") # Referência textual ao melhor baseline do paper (não como linha no gráfico) ax_roc.text(0.38, 0.12, f"Melhor baseline DF40 (CLIP): AUC = 0.802", fontsize=8, color="#475569", bbox=dict(boxstyle="round,pad=0.3", facecolor="#f1f5f9", edgecolor="#cbd5e1", linewidth=0.8)) ax_roc.set_xlim(-0.02, 1.02) ax_roc.set_ylim(-0.02, 1.05) ax_roc.set_xlabel("Taxa de Falsos Positivos (FPR)", fontsize=9, color="#334155") ax_roc.set_ylabel("Taxa de Verdadeiros Positivos (TPR)", fontsize=9, color="#334155") ax_roc.set_title("Curva ROC — Conjunto de Teste (20%)", fontsize=10, fontweight="bold", color="#0f172a", pad=8) ax_roc.legend(fontsize=8.5, loc="lower right", framealpha=0.9, edgecolor="#e2e8f0") ax_roc.set_facecolor("white") ax_roc.tick_params(labelsize=8) ax_roc.grid(color="#e2e8f0", linewidth=0.8, zorder=0) ax_roc.set_axisbelow(True) # ───────────────────────────────────────────────────────────────────── # Gráfico 3 — AUC por gerador: grouped bars (todos os modelos + nosso) # ───────────────────────────────────────────────────────────────────── modelos_plot = ["Xception", "CLIP", "SRM", "RECCE", "RFM", "Ours"] palette = { "Xception": "#94a3b8", "CLIP": "#64748b", "SRM": "#a8a29e", "RECCE": "#78716c", "RFM": "#b0b8c8", "Ours": AZUL, } n_gen = len(TABLE5_GENS) n_mod = len(modelos_plot) x = np.arange(n_gen) width = 0.13 offsets = np.linspace(-(n_mod-1)/2, (n_mod-1)/2, n_mod) * width for i, modelo in enumerate(modelos_plot): vals = [] for gen in TABLE5_GENS: if modelo == "Ours": vals.append(our_per_gen.get(gen, 0.0)) else: vals.append(LITERATURA_PER_GEN.get(gen, {}).get(modelo, 0.0)) cor = palette[modelo] ec = AZUL if modelo == "Ours" else "white" lw = 1.5 if modelo == "Ours" else 0.5 zorder = 4 if modelo == "Ours" else 2 ax_gen.bar(x + offsets[i], vals, width, label=modelo, color=cor, edgecolor=ec, linewidth=lw, zorder=zorder) ax_gen.set_xticks(x) ax_gen.set_xticklabels(TABLE5_GENS, fontsize=9, rotation=15, ha="right") ax_gen.set_ylabel("AUC-ROC", fontsize=9, color="#334155") ax_gen.set_ylim(0.25, 1.08) ax_gen.set_title( "AUC por Gerador — Comparação com Baselines (DF40 Table 5, Protocol-3, EFS Test)\n" "Os 7 geradores disponíveis que coincidem com os da tabela original", fontsize=10, fontweight="bold", color="#0f172a", pad=8 ) ax_gen.set_facecolor("white") ax_gen.tick_params(axis="x", labelsize=8.5) ax_gen.tick_params(axis="y", labelsize=8) ax_gen.yaxis.grid(True, color="#e2e8f0", linewidth=0.8, zorder=0) ax_gen.set_axisbelow(True) # Linha da média do nosso sistema (só nos T5) ax_gen.axhline(avg_t5, color=AZUL, linestyle="--", lw=1.5, alpha=0.7, label=f"Média Ours = {avg_t5:.3f}") ax_gen.axhline( np.mean([np.mean(list(LITERATURA_PER_GEN[g].values())) for g in TABLE5_GENS]), color="#64748b", linestyle=":", lw=1.2, alpha=0.6, label="Média CLIP (best baseline)" ) handles, lbls = ax_gen.get_legend_handles_labels() ax_gen.legend(handles, lbls, fontsize=8, ncol=4, loc="upper left", framealpha=0.92, edgecolor="#e2e8f0") # ───────────────────────────────────────────────────────────────────── # Guardar # ───────────────────────────────────────────────────────────────────── out = RESULTS_DIR / "comparacao_Protocol_3.png" fig.savefig(out, dpi=250, bbox_inches="tight", facecolor="white") plt.close() print(f"\n[+] Gráfico guardado: {out}") # ══════════════════════════════════════════════════════════════════════ # 6. JSON SUMÁRIO # ══════════════════════════════════════════════════════════════════════ summary = { "protocolo": "DF40 Protocol-3 — Test only, 7 geradores Table 5", "auc_split_test": float(auc_test), "acc_split_test": float(acc_test), "auc_cv_mean": float(auc_cv_mean), "auc_cv_std": float(auc_cv_std), "auc_avg_table5_gens": float(avg_t5), "n_total": int(len(y)), "n_real": int((y==0).sum()), "n_fake": int((y==1).sum()), "per_generator": { "table5": {k: float(v) for k, v in our_per_gen.items() if k in TABLE5_GENS}, "additional": {k: float(v) for k, v in our_per_gen.items() if k not in TABLE5_GENS}, }, "literatura_avg_table5": { m: float(np.mean([LITERATURA_PER_GEN[g][m] for g in TABLE5_GENS])) for m in modelos_ordem[:-1] # exclui "Ours" }, } with open(RESULTS_DIR / "comparison_summary.json", "w") as f: json.dump(summary, f, indent=4, ensure_ascii=False) print("[+] Sumário JSON guardado.")