| {"model_path": "/workspace/LMOps/minillm/checkpoints/llama-7b/", "ckpt_name": "llama-7b", "model_type": "llama", "teacher_model_type": null, "n_gpu": 2, "n_nodes": 1, "teacher_model_path": null, "teacher_ckpt_name": null, "teacher_model_fp16": false, "model_parallel": false, "model_parallel_size": null, "no_value": false, "dropout_path_rate": null, "dtype": "torch.float16", "type": "lm", "do_train": true, "do_valid": true, "do_eval": false, "base_path": "/workspace/LMOps/minillm", "load": null, "save": "/workspace/LMOps/minillm/results/llama/train/sft/e20-bs4-lr0.0005-G1-N2-NN1-lora-8-32-0.1", "log_interval": 4, "mid_log_num": 1, "save_interval": -1, "eval_interval": -1, "local_rank": 0, "save_additional_suffix": "", "save_rollout": false, "eb_sample_times": 3, "data_dir": "/workspace/LMOps/minillm/processed_data/dolly/", "processed_data_dir": null, "force_process": false, "force_process_demo": false, "data_process_workers": -1, "train_num": -1, "train_ratio": 1, "dev_num": 1000, "dev_ratio": 1, "gen_num": -1, "data_names": null, "prompt_type": null, "num_workers": 0, "max_prompt_length": 256, "min_prompt_length": 128, "json_data": false, "bin_data": false, "txt_data": false, "prompt_data_dir": null, "lm_data_dir": null, "eval_ppl": false, "eval_rw": false, "eval_gen": true, "only_prompt": false, "batch_size": 4, "eval_batch_size": 8, "clip_grad": 1.0, "total_iters": null, "train_iters_per_epoch": -1, "max_length": 512, "seed": 20, "seed_order": 10, "seed_data": 42, "seed_ppo": 42, "seed_lm": 7, "epochs": 20, "training_epochs": 10000, "gradient_accumulation_steps": 1, "gradient_checkpointing": true, "attn_dtype": null, "lr": 0.0005, "lr_min": 1e-07, "weight_decay": 0.01, "loss_scale": 65536, "kd_ratio": null, "warmup_iters": 0, "lr_decay_iters": null, "lr_decay_style": "cosine", "scheduler_name": "constant_trm", "reward_scaling": null, "cliprange_reward": 1, "ppo_epochs": null, "num_rollouts": 256, "num_rollouts_per_device": null, "cliprange": 0.2, "chunk_size": null, "gamma": 0.95, "length_norm": false, "single_step_reg": false, "teacher_mixed_alpha": null, "lm_coef": 1, "top_k": 0, "top_p": 1.0, "do_sample": true, "no_repeat_ngram_size": 6, "repetition_penalty": null, "num_beams": 1, "temperature": 1.0, "peft": "lora", "peft_lora_r": 8, "peft_lora_alpha": 32, "peft_lora_dropout": 0.1, "peft_name": null, "peft_path": null, "teacher_peft_name": null, "teacher_peft_path": null, "deepspeed": true, "deepspeed_config": "/workspace/LMOps/minillm/configs/deepspeed/ds_config_zero2_fp16.json", "deepscale": false, "deepscale_config": null, "rank": 0, "world_size": 2} |