Cormac Shannon
29e323b2fc
nanoGPT gets a qk_norm flag (off = upstream behaviour), a seed option and a resolved-config dump. run.py logs each run's git commit, config, data checksums, environment and metrics, and stops diverged runs. Claude-Session: https://claude.ai/code/session_01S8KN7NuUCm8nXLmy6eDuVv
36 lines
961 B
Python
36 lines
961 B
Python
# Tech-spike config: a tiny model on the small TinyStories subset, to check that the
|
|
# pipeline (data -> train on MPS -> run log) works end to end. NOT an experiment config:
|
|
# sizes, steps and LR here are placeholders, not decisions.
|
|
|
|
dataset = 'tinystories_spike'
|
|
device = 'mps'
|
|
dtype = 'float32' # train.py only uses autocast on CUDA; MPS runs in fp32
|
|
compile = False # torch.compile on MPS is not worth it for a spike
|
|
|
|
# model (~3.2M non-embedding params; the GPT-2 vocab adds ~12.9M embedding params)
|
|
n_layer = 4
|
|
n_head = 4
|
|
n_embd = 256
|
|
block_size = 256
|
|
bias = False
|
|
dropout = 0.0
|
|
qk_norm = False
|
|
|
|
# optimisation (AdamW, nanoGPT defaults otherwise)
|
|
batch_size = 32
|
|
gradient_accumulation_steps = 1
|
|
learning_rate = 1e-3
|
|
max_iters = 300
|
|
lr_decay_iters = 300
|
|
min_lr = 1e-4
|
|
warmup_iters = 30
|
|
weight_decay = 0.1
|
|
grad_clip = 1.0
|
|
|
|
# logging / eval
|
|
eval_interval = 100
|
|
eval_iters = 20
|
|
log_interval = 10
|
|
always_save_checkpoint = False
|
|
seed = 1337
|