| { | |
| "model": { | |
| "hf_path": "EleutherAI/pythia-6.9b", | |
| "short_name": "pythia-7b", | |
| "hidden_size": 4096, | |
| "max_length": 2048 | |
| }, | |
| "dataset": { | |
| "hf_path": "hendrycks/ethics", | |
| "subset": "commonsense", | |
| "text_column": "input", | |
| "label_column": "label", | |
| "dataset_class": "CommsenseDataset", | |
| "split": "train" | |
| }, | |
| "sae": { | |
| "expansion_factor": 4, | |
| "sparsity_coeff": 0.0001, | |
| "learning_rate": 0.0005, | |
| "num_epochs": 200, | |
| "batch_size": 850, | |
| "early_stopping_patience": 10, | |
| "early_stopping_metric": "val_total_loss" | |
| }, | |
| "probe": { | |
| "learning_rate": 0.0005, | |
| "weight_decay": 1e-06, | |
| "num_epochs": 100, | |
| "batch_size": 700, | |
| "early_stopping_patience": 15, | |
| "early_stopping_metric": "val_loss" | |
| }, | |
| "layer": 5, | |
| "is_best_layer": false | |
| } |