Update app.py

app.py

@@ -1,5 +1,4 @@
 import os
-import torch
 from dotenv import load_dotenv
 from datasets import load_dataset
 from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments
@@ -8,15 +7,13 @@ from huggingface_hub import login
 # === トークン読み込み ===
 load_dotenv()
 HF_TOKEN = os.getenv("HF_TOKEN")
-
 if not HF_TOKEN:
     raise ValueError("Hugging Faceのトークンが見つかりません。`.env`ファイルまたは環境変数を確認してください。")
-
 login(HF_TOKEN)
 
 # === 設定 ===
-BASE_MODEL = "Sakalti/Template-4"
-HF_REPO = "Sakalti/Template-16"
+BASE_MODEL = "Sakalti/template-4"  # 修正対象モデル名
+HF_REPO = "Sakalti/template-16"
 
 # === データ読み込み ===
 dataset = load_dataset("Verah/JParaCrawl-Filtered-English-Japanese-Parallel-Corpus", split="train")
@@ -25,12 +22,12 @@ dataset = load_dataset("Verah/JParaCrawl-Filtered-English-Japanese-Parallel-Corp
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)
 
-# === データ前処理 ===
+# === 超簡素トークナイズ関数 ===
 def preprocess(examples):
-    texts = [f"英語: {ex['en']}\n日本語:" for ex in examples]
-    model_inputs = tokenizer(texts, max_length=256, truncation=True)
-    model_inputs["labels"] = model_inputs["input_ids"]
-    return model_inputs
+    texts = [ex["en"] + " " + ex["ja"] for ex in examples]
+    tokenized = tokenizer(texts, max_length=256, truncation=True)
+    tokenized["labels"] = tokenized["input_ids"].copy()
+    return tokenized
 
 tokenized_dataset = dataset.map(preprocess, batched=True, remove_columns=dataset.column_names)
 
@@ -42,7 +39,7 @@ training_args = TrainingArguments(
     per_device_train_batch_size=2,
     num_train_epochs=3,
     save_total_limit=2,
-    save_steps=500,  # 500ステップごとに保存（ご要望通り）
+    save_steps=500,
     push_to_hub=True,
     hub_model_id=HF_REPO,
     hub_token=HF_TOKEN,