| { | |
| "base_model": "Qwen/Qwen2.5-VL-3B-Instruct", | |
| "layer": 12, | |
| "variant": "diversified_v1", | |
| "pretraining": { | |
| "dataset": "cc3m", | |
| "n_samples": 1242328, | |
| "format": "natural_caption_chat" | |
| }, | |
| "sae": { | |
| "sparsity": "topk", | |
| "k": 32, | |
| "width_mult": 8, | |
| "d_model": 4096, | |
| "d_sae": 32768 | |
| }, | |
| "training": { | |
| "epochs": 50, | |
| "batch_size": 4096, | |
| "lr": 0.0003, | |
| "weight_decay": 0.0 | |
| } | |
| } |