Tech spike: four smoke-test runs and a divergence-stop test
Pipeline works on MPS (~105 s per 300-step run), results are reproducible, and the divergence stop works. Not experimental evidence. Claude-Session: https://claude.ai/code/session_01S8KN7NuUCm8nXLmy6eDuVv
This commit is contained in:
Cormac Shannon committed
1 parent
55ba128dcc
commit
84b04d12e9
17 files changed
+792
-1
No files matched your search
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"out_dir": "/Users/nik/ISE/9.CS6513/thesis/06-experiments/runs/20261028-181025_spike-qkoff-lr1e-1",
|
||||
"eval_interval": 100,
|
||||
"log_interval": 10,
|
||||
"eval_iters": 20,
|
||||
"eval_only": false,
|
||||
"always_save_checkpoint": false,
|
||||
"init_from": "scratch",
|
||||
"wandb_log": false,
|
||||
"wandb_project": "owt",
|
||||
"wandb_run_name": "gpt2",
|
||||
"dataset": "tinystories_spike",
|
||||
"gradient_accumulation_steps": 1,
|
||||
"batch_size": 32,
|
||||
"block_size": 256,
|
||||
"n_layer": 4,
|
||||
"n_head": 4,
|
||||
"n_embd": 256,
|
||||
"dropout": 0.0,
|
||||
"bias": false,
|
||||
"qk_norm": false,
|
||||
"seed": 1337,
|
||||
"data_root": "/Users/nik/ISE/9.CS6513/thesis/06-experiments/data",
|
||||
"learning_rate": 0.1,
|
||||
"max_iters": 300,
|
||||
"weight_decay": 0.1,
|
||||
"beta1": 0.9,
|
||||
"beta2": 0.95,
|
||||
"grad_clip": 1.0,
|
||||
"decay_lr": true,
|
||||
"warmup_iters": 30,
|
||||
"lr_decay_iters": 300,
|
||||
"min_lr": 0.01,
|
||||
"backend": "nccl",
|
||||
"device": "mps",
|
||||
"dtype": "float32",
|
||||
"compile": false
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{"type": "eval", "iter": 0, "train_loss": 10.9141, "val_loss": 10.9167, "t": 5.41}
|
||||
{"type": "train", "iter": 0, "loss": 10.9193, "ms": 4331.0, "t": 5.74}
|
||||
{"type": "train", "iter": 10, "loss": 7.6884, "ms": 279.87, "t": 8.54}
|
||||
{"type": "train", "iter": 20, "loss": 6.7757, "ms": 280.21, "t": 11.34}
|
||||
{"type": "train", "iter": 30, "loss": 5.8674, "ms": 279.61, "t": 14.14}
|
||||
{"type": "train", "iter": 40, "loss": 5.9878, "ms": 280.23, "t": 16.95}
|
||||
{"type": "train", "iter": 50, "loss": 5.9047, "ms": 280.35, "t": 19.75}
|
||||
{"type": "train", "iter": 60, "loss": 5.8552, "ms": 280.38, "t": 22.55}
|
||||
{"type": "train", "iter": 70, "loss": 5.8407, "ms": 279.0, "t": 25.36}
|
||||
{"type": "train", "iter": 80, "loss": 5.7461, "ms": 280.14, "t": 28.16}
|
||||
{"type": "train", "iter": 90, "loss": 5.7984, "ms": 280.12, "t": 30.96}
|
||||
{"type": "eval", "iter": 100, "train_loss": 5.7546, "val_loss": 5.789, "t": 37.47}
|
||||
{"type": "train", "iter": 100, "loss": 5.7697, "ms": 4495.39, "t": 37.99}
|
||||
{"type": "train", "iter": 110, "loss": 5.7763, "ms": 280.61, "t": 40.79}
|
||||
{"type": "train", "iter": 120, "loss": 5.6788, "ms": 279.57, "t": 43.6}
|
||||
{"type": "train", "iter": 130, "loss": 5.7067, "ms": 292.17, "t": 46.47}
|
||||
{"type": "train", "iter": 140, "loss": 5.6992, "ms": 281.82, "t": 49.28}
|
||||
{"type": "train", "iter": 150, "loss": 5.6794, "ms": 292.66, "t": 52.14}
|
||||
{"type": "train", "iter": 160, "loss": 5.6159, "ms": 295.05, "t": 55.09}
|
||||
{"type": "train", "iter": 170, "loss": 5.6578, "ms": 280.39, "t": 57.92}
|
||||
{"type": "train", "iter": 180, "loss": 5.5731, "ms": 281.59, "t": 60.74}
|
||||
{"type": "train", "iter": 190, "loss": 5.3993, "ms": 298.74, "t": 63.59}
|
||||
{"type": "eval", "iter": 200, "train_loss": 5.4974, "val_loss": 5.5322, "t": 70.3}
|
||||
{"type": "train", "iter": 200, "loss": 5.4276, "ms": 4449.02, "t": 70.73}
|
||||
{"type": "train", "iter": 210, "loss": 5.524, "ms": 290.66, "t": 73.6}
|
||||
{"type": "train", "iter": 220, "loss": 5.4767, "ms": 281.67, "t": 76.42}
|
||||
{"type": "train", "iter": 230, "loss": 5.485, "ms": 282.2, "t": 79.23}
|
||||
{"type": "train", "iter": 240, "loss": 5.4442, "ms": 281.09, "t": 82.04}
|
||||
{"type": "train", "iter": 250, "loss": 5.3695, "ms": 281.49, "t": 84.86}
|
||||
{"type": "train", "iter": 260, "loss": 5.3708, "ms": 281.08, "t": 87.67}
|
||||
{"type": "train", "iter": 270, "loss": 5.36, "ms": 282.22, "t": 90.48}
|
||||
{"type": "train", "iter": 280, "loss": 5.2385, "ms": 281.0, "t": 93.29}
|
||||
{"type": "train", "iter": 290, "loss": 5.2615, "ms": 280.38, "t": 96.1}
|
||||
{"type": "eval", "iter": 300, "train_loss": 5.2472, "val_loss": 5.2948, "t": 102.62}
|
||||
{"type": "train", "iter": 300, "loss": 5.2321, "ms": 4418.66, "t": 103.06}
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"run_id": "20261028-181025_spike-qkoff-lr1e-1",
|
||||
"name": "spike-qkoff-lr1e-1",
|
||||
"note": "tech spike smoke test (not an experiment); divergence-detection check",
|
||||
"status": "completed",
|
||||
"started": "2026-10-28T18:10:25+00:00",
|
||||
"command": "/Users/nik/ISE/9.CS6513/thesis/06-experiments/.venv/bin/python /Users/nik/ISE/9.CS6513/thesis/06-experiments/nanogpt/train.py configs/spike.py --out_dir=/Users/nik/ISE/9.CS6513/thesis/06-experiments/runs/20261028-181025_spike-qkoff-lr1e-1 --data_root=/Users/nik/ISE/9.CS6513/thesis/06-experiments/data --qk_norm=False --learning_rate=1e-1 --min_lr=1e-2",
|
||||
"config_file": "configs/spike.py",
|
||||
"overrides": [
|
||||
"qk_norm=False",
|
||||
"learning_rate=1e-1",
|
||||
"min_lr=1e-2"
|
||||
],
|
||||
"config_requested": {
|
||||
"dataset": "tinystories_spike",
|
||||
"device": "mps",
|
||||
"dtype": "float32",
|
||||
"compile": false,
|
||||
"n_layer": 4,
|
||||
"n_head": 4,
|
||||
"n_embd": 256,
|
||||
"block_size": 256,
|
||||
"bias": false,
|
||||
"dropout": 0.0,
|
||||
"qk_norm": false,
|
||||
"batch_size": 32,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"learning_rate": 0.1,
|
||||
"max_iters": 300,
|
||||
"lr_decay_iters": 300,
|
||||
"min_lr": 0.01,
|
||||
"warmup_iters": 30,
|
||||
"weight_decay": 0.1,
|
||||
"grad_clip": 1.0,
|
||||
"eval_interval": 100,
|
||||
"eval_iters": 20,
|
||||
"log_interval": 10,
|
||||
"always_save_checkpoint": false,
|
||||
"seed": 1337
|
||||
},
|
||||
"git": {
|
||||
"commit": "55ba128dcca8310f85e99e172fe920a8c53a3a79",
|
||||
"branch": "main",
|
||||
"dirty": false
|
||||
},
|
||||
"nanogpt_upstream_commit": "3adf61e154c3fe3fca428ad6bc3818b27a3b8291",
|
||||
"data": {
|
||||
"dataset": "TinyStories V2 (GPT-4)",
|
||||
"source": "https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/",
|
||||
"tokenizer": "tiktoken gpt2",
|
||||
"files": {
|
||||
"TinyStoriesV2-GPT4-valid.txt": "6874bae9a4c1a4e7edcf0e53b86c17817e9cf881fc75ff2368da457b80c0585d"
|
||||
},
|
||||
"split": "valid file split 90/10 by story order: 24867 train / 2763 val stories",
|
||||
"tokens": {
|
||||
"train": 4920360,
|
||||
"val": 548664
|
||||
},
|
||||
"train_bin_bytes": 9840720,
|
||||
"val_bin_bytes": 1097328
|
||||
},
|
||||
"environment": {
|
||||
"host": "Cormacs-MacBook-Pro.local",
|
||||
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
||||
"machine": "arm64",
|
||||
"cpu": "Apple M4 Pro",
|
||||
"python": "3.12.14",
|
||||
"torch": "2.15.0",
|
||||
"mps_available": true,
|
||||
"cuda_available": false
|
||||
},
|
||||
"finished": "2026-10-28T18:12:09+00:00",
|
||||
"wall_seconds": 103.4,
|
||||
"summary": {
|
||||
"params_millions_nanogpt_count": 16.03,
|
||||
"first_train_loss": 10.9193,
|
||||
"last_train_loss": 5.2321,
|
||||
"last_eval": {
|
||||
"iter": 300,
|
||||
"train_loss": 5.2472,
|
||||
"val_loss": 5.2948
|
||||
},
|
||||
"best_val_loss": 5.2948,
|
||||
"diverged_at_iter": null,
|
||||
"returncode": 0
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user