Add QK-norm toggle, TinyStories prep and automatic run logging
nanoGPT gets a qk_norm flag (off = upstream behaviour), a seed option and a resolved-config dump. run.py logs each run's git commit, config, data checksums, environment and metrics, and stops diverged runs. Claude-Session: https://claude.ai/code/session_01S8KN7NuUCm8nXLmy6eDuVv
This commit is contained in:
1 parent
971caa808c
commit
29e323b2fc
8 files changed
+435
-7
No files matched your search
@@ -0,0 +1,35 @@
|
||||
# Tech-spike config: a tiny model on the small TinyStories subset, to check that the
|
||||
# pipeline (data -> train on MPS -> run log) works end to end. NOT an experiment config:
|
||||
# sizes, steps and LR here are placeholders, not decisions.
|
||||
|
||||
dataset = 'tinystories_spike'
|
||||
device = 'mps'
|
||||
dtype = 'float32' # train.py only uses autocast on CUDA; MPS runs in fp32
|
||||
compile = False # torch.compile on MPS is not worth it for a spike
|
||||
|
||||
# model (~3.2M non-embedding params; the GPT-2 vocab adds ~12.9M embedding params)
|
||||
n_layer = 4
|
||||
n_head = 4
|
||||
n_embd = 256
|
||||
block_size = 256
|
||||
bias = False
|
||||
dropout = 0.0
|
||||
qk_norm = False
|
||||
|
||||
# optimisation (AdamW, nanoGPT defaults otherwise)
|
||||
batch_size = 32
|
||||
gradient_accumulation_steps = 1
|
||||
learning_rate = 1e-3
|
||||
max_iters = 300
|
||||
lr_decay_iters = 300
|
||||
min_lr = 1e-4
|
||||
warmup_iters = 30
|
||||
weight_decay = 0.1
|
||||
grad_clip = 1.0
|
||||
|
||||
# logging / eval
|
||||
eval_interval = 100
|
||||
eval_iters = 20
|
||||
log_interval = 10
|
||||
always_save_checkpoint = False
|
||||
seed = 1337
|
||||
Reference in new issue
Block a user