Cormac Shannon
84b04d12e9
Pipeline works on MPS (~105 s per 300-step run), results are reproducible, and the divergence stop works. Not experimental evidence. Claude-Session: https://claude.ai/code/session_01S8KN7NuUCm8nXLmy6eDuVv
38 lines
919 B
JSON
38 lines
919 B
JSON
{
|
|
"out_dir": "/Users/nik/ISE/9.CS6513/thesis/06-experiments/runs/20261028-180656_spike-qkoff-lr1e-3",
|
|
"eval_interval": 100,
|
|
"log_interval": 10,
|
|
"eval_iters": 20,
|
|
"eval_only": false,
|
|
"always_save_checkpoint": false,
|
|
"init_from": "scratch",
|
|
"wandb_log": false,
|
|
"wandb_project": "owt",
|
|
"wandb_run_name": "gpt2",
|
|
"dataset": "tinystories_spike",
|
|
"gradient_accumulation_steps": 1,
|
|
"batch_size": 32,
|
|
"block_size": 256,
|
|
"n_layer": 4,
|
|
"n_head": 4,
|
|
"n_embd": 256,
|
|
"dropout": 0.0,
|
|
"bias": false,
|
|
"qk_norm": false,
|
|
"seed": 1337,
|
|
"data_root": "/Users/nik/ISE/9.CS6513/thesis/06-experiments/data",
|
|
"learning_rate": 0.001,
|
|
"max_iters": 300,
|
|
"weight_decay": 0.1,
|
|
"beta1": 0.9,
|
|
"beta2": 0.95,
|
|
"grad_clip": 1.0,
|
|
"decay_lr": true,
|
|
"warmup_iters": 30,
|
|
"lr_decay_iters": 300,
|
|
"min_lr": 0.0001,
|
|
"backend": "nccl",
|
|
"device": "mps",
|
|
"dtype": "float32",
|
|
"compile": false
|
|
} |