Add QK-norm toggle, TinyStories prep and automatic run logging

nanoGPT gets a qk_norm flag (off = upstream behaviour), a seed option and
a resolved-config dump. run.py logs each run's git commit, config, data
checksums, environment and metrics, and stops diverged runs.

Claude-Session: https://claude.ai/code/session_01S8KN7NuUCm8nXLmy6eDuVv
This commit is contained in:
Cormac Shannon committed 2026-10-28 18:05:31 +00:00
1 parent 971caa808c
commit 29e323b2fc
8 files changed
+435 -7

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
# Tech-spike config: a tiny model on the small TinyStories subset, to check that the
# pipeline (data -> train on MPS -> run log) works end to end. NOT an experiment config:
# sizes, steps and LR here are placeholders, not decisions.
dataset = 'tinystories_spike'
device = 'mps'
dtype = 'float32' # train.py only uses autocast on CUDA; MPS runs in fp32
compile = False # torch.compile on MPS is not worth it for a spike
# model (~3.2M non-embedding params; the GPT-2 vocab adds ~12.9M embedding params)
n_layer = 4
n_head = 4
n_embd = 256
block_size = 256
bias = False
dropout = 0.0
qk_norm = False
# optimisation (AdamW, nanoGPT defaults otherwise)
batch_size = 32
gradient_accumulation_steps = 1
learning_rate = 1e-3
max_iters = 300
lr_decay_iters = 300
min_lr = 1e-4
warmup_iters = 30
weight_decay = 0.1
grad_clip = 1.0
# logging / eval
eval_interval = 100
eval_iters = 20
log_interval = 10
always_save_checkpoint = False
seed = 1337