| { | |
| "model": { | |
| "hf_path": "EleutherAI/pythia-6.9b", | |
| "short_name": "pythia-7b", | |
| "hidden_size": 4096, | |
| "max_length": 2048 | |
| }, | |
| "dataset": { | |
| "hf_path": "hendrycks/ethics", | |
| "subset": "commonsense", | |
| "text_column": "input", | |
| "label_column": "label", | |
| "split": "train" | |
| }, | |
| "sae": { | |
| "expansion_factor": 4, | |
| "sparsity_coeff": 0.0001, | |
| "learning_rate": 5e-05, | |
| "num_epochs": 20, | |
| "batch_size": 64, | |
| "early_stopping_patience": 10, | |
| "early_stopping_metric": "val_total_loss" | |
| }, | |
| "probe": { | |
| "learning_rate": 5e-05, | |
| "weight_decay": 0, | |
| "l1_penalty": 1e-05, | |
| "num_epochs": 10, | |
| "batch_size": 64, | |
| "early_stopping_patience": 15, | |
| "early_stopping_metric": "val_loss" | |
| }, | |
| "layer": 3, | |
| "is_best_layer": false | |
| } |