| { |
| "schema_version": 3, |
| "recipe_id": "nllb200-spm8k-directional-v3", |
| "model_family": "nllb", |
| "corpus_pipeline_version": "formosan-mt-corpus-v3", |
| "mt_standardization": { |
| "id": "formosan-mt-standard-v3", |
| "sha256": "4bbded87eb4833b2d1cd5a88a1f2b059560416e0e69ecfa55433a77f418c903e", |
| "namespace": "formosan-mt" |
| }, |
| "project_root": ".", |
| "corpus_variants": ["public_no_bible", "private_no_bible"], |
| "base_model": { |
| "name": "facebook/nllb-200-distilled-600M", |
| "revision": "f8d333a098d19b4fd9a8b18f94170487ad3f821d" |
| }, |
| "primary_input_pattern": "corpus_builds/{corpus_name}/pivot_corpora_final/big_corpus_{target}_in_domain_hard.csv", |
| "targets": { |
| "english": { |
| "input_pattern": "corpus_builds/{corpus_name}/pivot_corpora_final/big_corpus_en_in_domain_hard.csv", |
| "target_col": "english_sentence", |
| "directions": ["f2en", "en2f"], |
| "selected_recipe": "NLLB-200 SPM8k directional" |
| }, |
| "chinese": { |
| "input_pattern": "corpus_builds/{corpus_name}/pivot_corpora_final/big_corpus_zh_in_domain_hard.csv", |
| "target_col": "chinese_sentence", |
| "directions": ["f2zh", "zh2f"], |
| "selected_recipe": "NLLB-200 SPM8k directional" |
| } |
| }, |
| "splits": { |
| "english_output_dir": "formosan_mt_experiments/data/splits_en_v1", |
| "chinese_output_dir": "formosan_mt_experiments/data/splits_zh_v1", |
| "train_ratio": 0.9, |
| "validate_ratio": 0.025, |
| "test_ratio": 0.075, |
| "min_test_rows": 500, |
| "min_validate_rows": 150, |
| "character_ngram_jaccard_threshold": 0.82, |
| "tiers": ["in_domain_hard"], |
| "headline_tier": "in_domain_hard" |
| }, |
| "tokenizer": { |
| "mode": "spm", |
| "spm_vocab_sweep": [8192], |
| "default_spm_vocab": 8192, |
| "chinese_default_spm_vocab": 8192, |
| "min_char_frequency": 3, |
| "max_dialect_tags": 200, |
| "min_dialect_frequency": 3, |
| "training_columns": ["formosan_sentence"], |
| "setup_splits": ["train"] |
| }, |
| "training_defaults": { |
| "steps": 300000, |
| "batch_size": 16, |
| "grad_accum_steps": 4, |
| "effective_batch_size": 64, |
| "max_length": 384, |
| "learning_rate": 0.00002, |
| "warmup_steps": 4000, |
| "weight_decay": 0.001, |
| "alpha": 0.5, |
| "precision": "bf16", |
| "label_smoothing": 0.1, |
| "save_interval": 0, |
| "log_interval": 500, |
| "generation_eval_interval": 10000, |
| "generation_eval_samples_per_language": 128, |
| "generation_eval_batch_size": 16, |
| "validation_beam": 2, |
| "best_metric": "chrF2", |
| "early_stopping_patience": 5, |
| "early_stopping_min_delta": 0.05, |
| "early_stopping_start_step": 30000, |
| "f2en_easy_source_weight": 0.05, |
| "en2f_easy_source_weight": 0.15, |
| "f2zh_easy_source_weight": 0.05, |
| "zh2f_easy_source_weight": 0.15 |
| }, |
| "generation_defaults": { |
| "beam": 4, |
| "max_length": 384, |
| "max_new_tokens": 256, |
| "min_new_tokens": 1, |
| "no_repeat_ngram_size": 0, |
| "repetition_penalty": 1.0, |
| "length_penalty": 1.0, |
| "metadata_modes": ["default", "oracle"], |
| "bootstrap_samples": 200, |
| "bootstrap_seed": 42 |
| } |
| } |
|
|
