| |
| """Build a compact Arabic SFT JSONL from the downloaded Arabic corpus.""" |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| from pathlib import Path |
|
|
|
|
| def clean_line(line: str) -> str: |
| return " ".join(line.strip().split()) |
|
|
|
|
| def main() -> int: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--input", required=True) |
| parser.add_argument("--output", required=True) |
| parser.add_argument("--max-examples", type=int, default=2000) |
| parser.add_argument("--min-chars", type=int, default=180) |
| parser.add_argument("--max-chars", type=int, default=900) |
| args = parser.parse_args() |
|
|
| input_path = Path(args.input) |
| output_path = Path(args.output) |
| output_path.parent.mkdir(parents=True, exist_ok=True) |
|
|
| prompts = [ |
| "لخص النص العربي التالي في فقرة قصيرة وواضحة.", |
| "استخرج الفكرة الرئيسية من النص التالي.", |
| "اكتب جوابا تعليميا مختصرا اعتمادا على النص التالي.", |
| "حول النص التالي إلى نقاط معرفية واضحة.", |
| ] |
|
|
| count = 0 |
| with input_path.open("r", encoding="utf-8", errors="ignore") as src, output_path.open( |
| "w", encoding="utf-8" |
| ) as dst: |
| for raw in src: |
| text = clean_line(raw) |
| if len(text) < args.min_chars: |
| continue |
| text = text[: args.max_chars] |
| prompt = prompts[count % len(prompts)] |
| messages = [ |
| {"role": "user", "content": f"{prompt}\n\n{text}"}, |
| {"role": "assistant", "content": text[: min(len(text), 420)]}, |
| ] |
| dst.write(json.dumps(messages, ensure_ascii=False) + "\n") |
| count += 1 |
| if count >= args.max_examples: |
| break |
|
|
| if count == 0: |
| raise SystemExit(f"No examples written from {input_path}") |
| print(f"Wrote {count} Arabic SFT examples to {output_path}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
