165 lines
13 KiB
BibTeX
165 lines
13 KiB
BibTeX
% references.bib — MSc thesis, QK-norm at small scale
|
|
% Generated 28/10/2026 from arXiv API metadata via tools/arxiv2bib.py.
|
|
% Authors and titles = arXiv records (the authors' own metadata).
|
|
% Venues filled ONLY where the authors' arXiv comment states acceptance;
|
|
% otherwise left as @misc with a TODO. Import into Zotero: File > Import.
|
|
|
|
@inproceedings{henry2020,
|
|
title = {{Query-Key Normalization for Transformers}},
|
|
author = {Alex Henry and Prudhvi Raj Dachapally and Shubham Pawar and Yuxuan Chen},
|
|
year = {2020},
|
|
booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2020},
|
|
eprint = {2010.04245},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: 8 pages, 2 figures, accepted at Findings of EMNLP 2020},
|
|
}
|
|
|
|
% TODO verify venue: ICML 2023 per common citation (not confirmed by arXiv/OpenAlex)
|
|
@misc{vit22b,
|
|
title = {{Scaling Vision Transformers to 22 Billion Parameters}},
|
|
author = {Mostafa Dehghani and Josip Djolonga and Basil Mustafa and Piotr Padlewski and Jonathan Heek and Justin Gilmer and Andreas Steiner and Mathilde Caron and Robert Geirhos and Ibrahim Alabdulmohsin and Rodolphe Jenatton and Lucas Beyer and Michael Tschannen and Anurag Arnab and Xiao Wang and Carlos Riquelme and Matthias Minderer and Joan Puigcerver and Utku Evci and Manoj Kumar and Sjoerd van Steenkiste and Gamaleldin F. Elsayed and Aravindh Mahendran and Fisher Yu and Avital Oliver and Fantine Huot and Jasmijn Bastings and Mark Patrick Collier and Alexey Gritsenko and Vighnesh Birodkar and Cristina Vasconcelos and Yi Tay and Thomas Mensink and Alexander Kolesnikov and Filip Pavetić and Dustin Tran and Thomas Kipf and Mario Lučić and Xiaohua Zhai and Daniel Keysers and Jeremiah Harmsen and Neil Houlsby},
|
|
year = {2023},
|
|
eprint = {2302.05442},
|
|
archivePrefix = {arXiv},
|
|
}
|
|
|
|
@inproceedings{zhai2023,
|
|
title = {{Stabilizing Transformer Training by Preventing Attention Entropy Collapse}},
|
|
author = {Shuangfei Zhai and Tatiana Likhomanenko and Etai Littwin and Dan Busbridge and Jason Ramapuram and Yizhe Zhang and Jiatao Gu and Josh Susskind},
|
|
year = {2023},
|
|
booktitle = {Proceedings of the 40th International Conference on Machine Learning (ICML)},
|
|
eprint = {2303.06296},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: In International Conference on Machine Learning (pp. 40770-40803). PMLR. 2023},
|
|
}
|
|
|
|
% TODO verify venue: ICLR 2024 (oral) per CS6513 paper (not confirmed by arXiv/OpenAlex)
|
|
@misc{wortsman2024,
|
|
title = {{Small-scale proxies for large-scale Transformer training instabilities}},
|
|
author = {Mitchell Wortsman and Peter J. Liu and Lechao Xiao and Katie Everett and Alex Alemi and Ben Adlam and John D. Co-Reyes and Izzeddin Gur and Abhishek Kumar and Roman Novak and Jeffrey Pennington and Jascha Sohl-dickstein and Kelvin Xu and Jaehoon Lee and Justin Gilmer and Simon Kornblith},
|
|
year = {2023},
|
|
eprint = {2309.14322},
|
|
archivePrefix = {arXiv},
|
|
}
|
|
|
|
@inproceedings{olmo2,
|
|
title = {{2 OLMo 2 Furious}},
|
|
author = {Team OLMo and Pete Walsh and Luca Soldaini and Dirk Groeneveld and Kyle Lo and Shane Arora and Akshita Bhagia and Yuling Gu and Shengyi Huang and Matt Jordan and Nathan Lambert and Dustin Schwenk and Oyvind Tafjord and Taira Anderson and David Atkinson and Faeze Brahman and Christopher Clark and Pradeep Dasigi and Nouha Dziri and Allyson Ettinger and Michal Guerquin and David Heineman and Hamish Ivison and Pang Wei Koh and Jiacheng Liu and Saumya Malik and William Merrill and Lester James V. Miranda and Jacob Morrison and Tyler Murray and Crystal Nam and Jake Poznanski and Valentina Pyatkin and Aman Rangapur and Michael Schmitz and Sam Skjonsberg and David Wadden and Christopher Wilhelm and Michael Wilson and Luke Zettlemoyer and Ali Farhadi and Noah A. Smith and Hannaneh Hajishirzi},
|
|
year = {2025},
|
|
booktitle = {Conference on Language Modeling (COLM) (shorter version)},
|
|
eprint = {2501.00656},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: Shorter version accepted to COLM 2025. Updated to include 32B results. Model demo available at playground.allenai.org},
|
|
}
|
|
|
|
@misc{imu1,
|
|
title = {{IMU-1: Sample-Efficient Pre-training of Small Language Models}},
|
|
author = {George Grigorev},
|
|
year = {2026},
|
|
eprint = {2602.02522},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: 16 pages},
|
|
}
|
|
|
|
@misc{slmtradeoffs,
|
|
title = {{Architectural Trade-offs in Small Language Models Under Compute Constraints}},
|
|
author = {Shivraj Singh Bhatti},
|
|
year = {2025},
|
|
eprint = {2512.20877},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: 15 pages, 11 images},
|
|
}
|
|
|
|
@inproceedings{mup,
|
|
title = {{Tensor Programs V: Tuning Large Neural Networks via Zero-Shot Hyperparameter Transfer}},
|
|
author = {Greg Yang and Edward J. Hu and Igor Babuschkin and Szymon Sidor and Xiaodong Liu and David Farhi and Nick Ryder and Jakub Pachocki and Weizhu Chen and Jianfeng Gao},
|
|
year = {2021},
|
|
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
|
|
eprint = {2203.03466},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: NeurIPS 2021},
|
|
}
|
|
|
|
@inproceedings{sun_massive,
|
|
title = {{Massive Activations in Large Language Models}},
|
|
author = {Mingjie Sun and Xinlei Chen and J. Zico Kolter and Zhuang Liu},
|
|
year = {2024},
|
|
booktitle = {Conference on Language Modeling (COLM)},
|
|
eprint = {2402.17762},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: First Conference on Language Modeling (COLM), 2024. Website at https://eric-mingjie.github.io/massive-activations/index.html},
|
|
}
|
|
|
|
@inproceedings{attnsink,
|
|
title = {{When Attention Sink Emerges in Language Models: An Empirical View}},
|
|
author = {Xiangming Gu and Tianyu Pang and Chao Du and Qian Liu and Fengzhuo Zhang and Cunxiao Du and Ye Wang and Min Lin},
|
|
year = {2025},
|
|
booktitle = {International Conference on Learning Representations (ICLR)},
|
|
eprint = {2410.10781},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: ICLR 2025 (Spotlight)},
|
|
}
|
|
|
|
@misc{softcap,
|
|
title = {{Gemma 2: Improving Open Language Models at a Practical Size}},
|
|
author = { Gemma Team and Morgane Riviere and Shreya Pathak and Pier Giuseppe Sessa and Cassidy Hardin and Surya Bhupatiraju and Léonard Hussenot and Thomas Mesnard and Bobak Shahriari and Alexandre Ramé and Johan Ferret and Peter Liu and Pouya Tafti and Abe Friesen and Michelle Casbon and Sabela Ramos and Ravin Kumar and Charline Le Lan and Sammy Jerome and Anton Tsitsulin and Nino Vieillard and Piotr Stanczyk and Sertan Girgin and Nikola Momchev and Matt Hoffman and Shantanu Thakoor and Jean-Bastien Grill and Behnam Neyshabur and Olivier Bachem and Alanna Walton and Aliaksei Severyn and Alicia Parrish and Aliya Ahmad and Allen Hutchison and Alvin Abdagic and Amanda Carl and Amy Shen and Andy Brock and Andy Coenen and Anthony Laforge and Antonia Paterson and Ben Bastian and Bilal Piot and Bo Wu and Brandon Royal and Charlie Chen and Chintu Kumar and Chris Perry and Chris Welty and Christopher A. Choquette-Choo and Danila Sinopalnikov and David Weinberger and Dimple Vijaykumar and Dominika Rogozińska and Dustin Herbison and Elisa Bandy and Emma Wang and Eric Noland and Erica Moreira and Evan Senter and Evgenii Eltyshev and Francesco Visin and Gabriel Rasskin and Gary Wei and Glenn Cameron and Gus Martins and Hadi Hashemi and Hanna Klimczak-Plucińska and Harleen Batra and Harsh Dhand and Ivan Nardini and Jacinda Mein and Jack Zhou and James Svensson and Jeff Stanway and Jetha Chan and Jin Peng Zhou and Joana Carrasqueira and Joana Iljazi and Jocelyn Becker and Joe Fernandez and Joost van Amersfoort and Josh Gordon and Josh Lipschultz and Josh Newlan and Ju-yeong Ji and Kareem Mohamed and Kartikeya Badola and Kat Black and Katie Millican and Keelin McDonell and Kelvin Nguyen and Kiranbir Sodhia and Kish Greene and Lars Lowe Sjoesund and Lauren Usui and Laurent Sifre and Lena Heuermann and Leticia Lago and Lilly McNealus and Livio Baldini Soares and Logan Kilpatrick and Lucas Dixon and Luciano Martins and Machel Reid and Manvinder Singh and Mark Iverson and Martin Görner and Mat Velloso and Mateo Wirth and Matt Davidow and Matt Miller and Matthew Rahtz and Matthew Watson and Meg Risdal and Mehran Kazemi and Michael Moynihan and Ming Zhang and Minsuk Kahng and Minwoo Park and Mofi Rahman and Mohit Khatwani and Natalie Dao and Nenshad Bardoliwalla and Nesh Devanathan and Neta Dumai and Nilay Chauhan and Oscar Wahltinez and Pankil Botarda and Parker Barnes and Paul Barham and Paul Michel and Pengchong Jin and Petko Georgiev and Phil Culliton and Pradeep Kuppala and Ramona Comanescu and Ramona Merhej and Reena Jana and Reza Ardeshir Rokni and Rishabh Agarwal and Ryan Mullins and Samaneh Saadat and Sara Mc Carthy and Sarah Cogan and Sarah Perrin and Sébastien M. R. Arnold and Sebastian Krause and Shengyang Dai and Shruti Garg and Shruti Sheth and Sue Ronstrom and Susan Chan and Timothy Jordan and Ting Yu and Tom Eccles and Tom Hennigan and Tomas Kocisky and Tulsee Doshi and Vihan Jain and Vikas Yadav and Vilobh Meshram and Vishal Dharmadhikari and Warren Barkley and Wei Wei and Wenming Ye and Woohyun Han and Woosuk Kwon and Xiang Xu and Zhe Shen and Zhitao Gong and Zichuan Wei and Victor Cotruta and Phoebe Kirk and Anand Rao and Minh Giang and Ludovic Peran and Tris Warkentin and Eli Collins and Joelle Barral and Zoubin Ghahramani and Raia Hadsell and D. Sculley and Jeanine Banks and Anca Dragan and Slav Petrov and Oriol Vinyals and Jeff Dean and Demis Hassabis and Koray Kavukcuoglu and Clement Farabet and Elena Buchatskaya and Sebastian Borgeaud and Noah Fiedel and Armand Joulin and Kathleen Kenealy and Robert Dadashi and Alek Andreev},
|
|
year = {2024},
|
|
eprint = {2408.00118},
|
|
archivePrefix = {arXiv},
|
|
}
|
|
|
|
% NB: check z-loss is the right attribution (PaLM popularised it; see Wortsman et al. for discussion)
|
|
% TODO verify venue: JMLR 2023 per common citation (not confirmed by arXiv/OpenAlex)
|
|
@misc{zloss,
|
|
title = {{PaLM: Scaling Language Modeling with Pathways}},
|
|
author = {Aakanksha Chowdhery and Sharan Narang and Jacob Devlin and Maarten Bosma and Gaurav Mishra and Adam Roberts and Paul Barham and Hyung Won Chung and Charles Sutton and Sebastian Gehrmann and Parker Schuh and Kensen Shi and Sasha Tsvyashchenko and Joshua Maynez and Abhishek Rao and Parker Barnes and Yi Tay and Noam Shazeer and Vinodkumar Prabhakaran and Emily Reif and Nan Du and Ben Hutchinson and Reiner Pope and James Bradbury and Jacob Austin and Michael Isard and Guy Gur-Ari and Pengcheng Yin and Toju Duke and Anselm Levskaya and Sanjay Ghemawat and Sunipa Dev and Henryk Michalewski and Xavier Garcia and Vedant Misra and Kevin Robinson and Liam Fedus and Denny Zhou and Daphne Ippolito and David Luan and Hyeontaek Lim and Barret Zoph and Alexander Spiridonov and Ryan Sepassi and David Dohan and Shivani Agrawal and Mark Omernick and Andrew M. Dai and Thanumalayan Sankaranarayana Pillai and Marie Pellat and Aitor Lewkowycz and Erica Moreira and Rewon Child and Oleksandr Polozov and Katherine Lee and Zongwei Zhou and Xuezhi Wang and Brennan Saeta and Mark Diaz and Orhan Firat and Michele Catasta and Jason Wei and Kathy Meier-Hellstern and Douglas Eck and Jeff Dean and Slav Petrov and Noah Fiedel},
|
|
year = {2022},
|
|
eprint = {2204.02311},
|
|
archivePrefix = {arXiv},
|
|
}
|
|
|
|
@misc{normformer,
|
|
title = {{NormFormer: Improved Transformer Pretraining with Extra Normalization}},
|
|
author = {Sam Shleifer and Jason Weston and Myle Ott},
|
|
year = {2021},
|
|
eprint = {2110.09456},
|
|
archivePrefix = {arXiv},
|
|
}
|
|
|
|
@misc{tinystories,
|
|
title = {{TinyStories: How Small Can Language Models Be and Still Speak Coherent English?}},
|
|
author = {Ronen Eldan and Yuanzhi Li},
|
|
year = {2023},
|
|
eprint = {2305.07759},
|
|
archivePrefix = {arXiv},
|
|
}
|
|
|
|
% TODO verify venue: ICML 2023 per CS6513 paper (not confirmed by arXiv/OpenAlex)
|
|
@misc{cramming,
|
|
title = {{Cramming: Training a Language Model on a Single GPU in One Day}},
|
|
author = {Jonas Geiping and Tom Goldstein},
|
|
year = {2022},
|
|
eprint = {2212.14034},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: 22 pages, we provide code at https://github.com/JonasGeiping/cramming},
|
|
}
|
|
|
|
% Cited in main.tex scaffold only, not in the submitted draft.
|
|
@misc{qknmla,
|
|
title = {{QK-Normed MLA: QK normalization without full key caching}},
|
|
author = {Yizhou Han and Yao Zhao and Jun Zhou and Longfei Li and Ruoyu Sun},
|
|
year = {2026},
|
|
eprint = {2606.16310},
|
|
archivePrefix = {arXiv},
|
|
note = {arXiv comment: 13 pages, 5 figures, conference-style manuscript},
|
|
}
|
|
|
|
@misc{nanogpt,
|
|
title = {{nanoGPT}},
|
|
author = {Andrej Karpathy},
|
|
year = {2022},
|
|
howpublished = {\url{https://github.com/karpathy/nanoGPT}},
|
|
note = {GitHub repository},
|
|
}
|
|
|