refs.bib

@article{cho2009kernel,
  title={Kernel methods for deep learning},
  author={Cho, Youngmin and Saul, Lawrence},
  journal={Advances in neural information processing systems},
  volume={22},
  year={2009}
}

@article{smola2004tutorial,
  title={A tutorial on support vector regression},
  author={Smola, Alex J and Sch{\"o}lkopf, Bernhard},
  journal={Statistics and computing},
  volume={14},
  pages={199--222},
  year={2004},
  publisher={Springer}
}

@book{williams2006gaussian,
  title={Gaussian processes for machine learning},
  author={Williams, Christopher KI and Rasmussen, Carl Edward},
  volume={2},
  number={3},
  year={2006},
  publisher={MIT press Cambridge, MA}
}

@book{smola2000advances,
  title={Advances in large margin classifiers},
  author={Smola, Alexander J},
  year={2000},
  publisher={MIT press}
}


@article{mairal2014convolutional,
  title={Convolutional kernel networks},
  author={Mairal, Julien and Koniusz, Piotr and Harchaoui, Zaid and Schmid, Cordelia},
  journal={Advances in neural information processing systems},
  volume={27},
  year={2014}
}

@article{zhang2021understanding,
  title={Understanding deep learning (still) requires rethinking generalization},
  author={Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
  journal={Communications of the ACM},
  volume={64},
  number={3},
  pages={107--115},
  year={2021},
  publisher={ACM New York, NY, USA}
}

@inproceedings{zhang2017understanding,
title={Understanding deep learning requires rethinking generalization},
author={Chiyuan Zhang and Samy Bengio and Moritz Hardt and Benjamin Recht and Oriol Vinyals},
booktitle={International Conference on Learning Representations},
year={2017},
}

@article{arora2019exact,
  title={On exact computation with an infinitely wide neural net},
  author={Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Russ R and Wang, Ruosong},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@book{scholkopf2002learning,
  title={Learning with kernels: support vector machines, regularization, optimization, and beyond},
  author={Sch{\"o}lkopf, Bernhard and Smola, Alexander J},
  year={2002},
  publisher={MIT press}
}

@article{bengio2013representation,
  title={Representation learning: A review and new perspectives},
  author={Bengio, Yoshua and Courville, Aaron and Vincent, Pascal},
  journal={IEEE transactions on pattern analysis and machine intelligence},
  volume={35},
  number={8},
  pages={1798--1828},
  year={2013},
  publisher={IEEE}
}

@inproceedings{wilson2016deep,
  title={Deep kernel learning},
  author={Wilson, Andrew Gordon and Hu, Zhiting and Salakhutdinov, Ruslan and Xing, Eric P},
  booktitle={Artificial intelligence and statistics},
  pages={370--378},
  year={2016},
  organization={PMLR}
}

@inproceedings{schroff2015facenet,
  title={Facenet: A unified embedding for face recognition and clustering},
  author={Schroff, Florian and Kalenichenko, Dmitry and Philbin, James},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={815--823},
  year={2015}
}

@inproceedings{raghu2017expressive,
  title={On the expressive power of deep neural networks},
  author={Raghu, Maithra and Poole, Ben and Kleinberg, Jon and Ganguli, Surya and Sohl-Dickstein, Jascha},
  booktitle={international conference on machine learning},
  pages={2847--2854},
  year={2017},
  organization={PMLR}
}

@inproceedings{choromanski2018geometry,
  title={The geometry of random features},
  author={Choromanski, Krzysztof and Rowland, Mark and Sarl{\'o}s, Tam{\'a}s and Sindhwani, Vikas and Turner, Richard and Weller, Adrian},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1--9},
  year={2018},
  organization={PMLR}
}

@inproceedings{chen2018dynamical,
  title={Dynamical isometry and a mean field theory of RNNs: Gating enables signal propagation in recurrent neural networks},
  author={Chen, Minmin and Pennington, Jeffrey and Schoenholz, Samuel},
  booktitle={International Conference on Machine Learning},
  pages={873--882},
  year={2018},
  organization={PMLR}
}

@inproceedings{koch2015siamese,
  title={Siamese neural networks for one-shot image recognition},
  author={Koch, Gregory and Zemel, Richard and Salakhutdinov, Ruslan and others},
  booktitle={ICML deep learning workshop},
  volume={2},
  number={1},
  pages={1--30},
  year={2015},
  organization={Lille}
}

@InProceedings{hayou2019impact,
  title = 	 {On the Impact of the Activation function on Deep Neural Networks Training},
  author =       {Hayou, Soufiane and Doucet, Arnaud and Rousseau, Judith},
  booktitle = 	 {Proceedings of the 36th International Conference on Machine Learning},
  pages = 	 {2672--2680},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
  volume = 	 {97},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v97/hayou19a/hayou19a.pdf},
  abstract = 	 {The weight initialization and the activation function of deep neural networks have a crucial impact on the performance of the training procedure. An inappropriate selection can lead to the loss of information of the input during forward propagation and the exponential vanishing/exploding of gradients during back-propagation. Understanding the theoretical properties of untrained random networks is key to identifying which deep networks may be trained successfully as recently demonstrated by Samuel et al. (2017) who showed that for deep feedforward neural networks only a specific choice of hyperparameters known as the ‘Edge of Chaos’ can lead to good performance. While the work by Samuel et al. (2017) discuss trainability issues, we focus here on training acceleration and overall performance. We give a comprehensive theoretical analysis of the Edge of Chaos and show that we can indeed tune the initialization parameters and the activation function in order to accelerate the training and improve the performance.}
}


@inproceedings{allen2019convergence,
  title={A convergence theory for deep learning via over-parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International conference on machine learning},
  pages={242--252},
  year={2019},
  organization={PMLR}
}

@inproceedings{arora2019fine,
  title={Fine-grained analysis of optimization and generalization for overparameterized two-layer neural networks},
  author={Arora, Sanjeev and Du, Simon and Hu, Wei and Li, Zhiyuan and Wang, Ruosong},
  booktitle={International Conference on Machine Learning},
  pages={322--332},
  year={2019},
  organization={PMLR}
}

@book{latouche1999introduction,
 author = {Latouche, Guy and Ramaswami, Vaidyanathan},
 publisher = {SIAM},
 title = {Introduction to matrix analytic methods in stochastic modeling},
 year = {1999}
}

@article{sawa1972finite,
  title={Finite-sample properties of the k-class estimators},
  author={Sawa, Takamitsu},
  journal={Econometrica: Journal of the Econometric Society},
  pages={653--680},
  year={1972},
  publisher={JSTOR}
}

@inproceedings{meterez2024towards,
title={Towards Training Without Depth Limits: Batch Normalization Without Gradient Explosion},
author={Alexandru Meterez and Amir Joudaki and Francesco Orabona and Alexander Immer and Gunnar Ratsch and Hadi Daneshmand},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
}


@article{braun2010variational,
 author = {Braun, Michael and McAuliffe, Jon},
 journal = {Journal of the American Statistical Association},
 number = {489},
 pages = {324--335},
 publisher = {Taylor \& Francis},
 title = {Variational inference for large-scale models of discrete choice},
 volume = {105},
 year = {2010}
}


@inproceedings{hazra2019det,
 author = {Hazra, Sayan and Banerjee, Soumi and Ghosh, Kripabandhu and Ghosh, Saptarshi and Mehta, Parth},
 booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
 title = {A DET for Natural Language Inference: Generating Natural Language Inference Explanation Trees using Deep Generative Models},
 year = {2019}
}


@article{bach2021gradient,
 author = {Chizat, L\'{e}na\"{\i}c and Bach, Francis},
 booktitle = {Advances in Neural Information Processing Systems},
 journal = {Advances in Neural Information Processing Systems},
 title = {On the Global Convergence of Gradient Descent for Over-parameterized Models using Optimal Transport},
 year = {2018}
}

@inproceedings{chizat2020implicit,
 author = {Chizat, Lenaic and Bach, Francis},
 booktitle = {Conference on Learning Theory},
 pages = {1305--1338},
 title = {Implicit bias of gradient descent for wide two-layer neural networks trained with the logistic loss},
 year = {2020}
}

@inproceedings{ba2019generalization,
 author = {Ba, Jimmy and Erdogdu, Murat and Suzuki, Taiji and Wu, Denny and Zhang, Tianzong},
 booktitle = {International Conference on Learning Representations},
 title = {Generalization of two-layer neural networks: An asymptotic viewpoint},
 year = {2019}
}

@inproceedings{pmlrv9glorot10a,
 author = {Glorot, Xavier and Bengio, Yoshua},
 booktitle = {Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics},
 pages = {249--256},
 title = {Understanding the difficulty of training deep feedforward neural networks},
 year = {2010}
}

@article{hanin2019finite,
 author = {Hanin, Boris and Nica, Mihai},
 journal = {arXiv preprint arXiv:1909.05989},
 title = {Finite depth and width corrections to the neural tangent kernel},
 year = {2019}
}

@article{hanin2022correlation,
 author = {Hanin, Boris},
 journal = {arXiv preprint arXiv:2204.01058},
 title = {Correlation Functions in Random Fully Connected Neural Networks at Finite Width},
 year = {2022}
}

@article{rosenthal1995minorization,
 author = {Rosenthal, Jeffrey S.},
 journal = {Journal of the American Statistical Association},
 number = {430},
 pages = {558--566},
 publisher = {Taylor \& Francis},
 title = {{Minorization conditions and convergence rates for Markov chain Monte Carlo}},
 volume = {90},
 year = {1995}
}

@article{jones2001honest,
 author = {{Jones, Galin L. and Hobert, James P.}},
 journal = {Statistical Science},
 pages = {312--334},
 publisher = {JSTOR},
 title = {{Honest exploration of intractable probability distributions via Markov chain Monte Carlo}},
 year = {2001}
}

@article{doeblin1938deux,
 author = {Doeblin, Wolfgang},
 journal = {Bulletin de la Soci{\'e}t{\'e} Math{\'e}matique de France},
 pages = {210--220},
 title = {{Sur deux probl{\`e}mes de M. {K}olmogoroff concernant les cha{\^\i}nes d{\'e}nombrables}},
 volume = {66},
 year = {1938}
}

@incollection{hoffman2003variation,
 author = {Hoffman, Alan J. and Wielandt, Helmut W},
 booktitle = {Selected Papers Of Alan J. Hoffman: With Commentary},
 pages = {118--120},
 publisher = {World Scientific},
 title = {The variation of the spectrum of a normal matrix},
 year = {2003}
}

@article{pastur1967distribution,
 author = {Pastur, LA and Martchenko, VA},
 journal = {Math. USSR-Sbornik},
 number = {4},
 pages = {457--483},
 title = {The distribution of eigenvalues in certain sets of random matrices},
 volume = {1},
 year = {1967}
}

@article{daneshmand2022polynomial,
  title={Polynomial-time Sparse Measure Recovery: From Mean Field Theory to Algorithm Design},
  author={Daneshmand, Hadi and Bach, Francis},
  journal={arXiv preprint arXiv:2204.07879},
  year={2022}
}

@article{devroye2018total,
 author = {Devroye, Luc and Mehrabian, Abbas and Reddad, Tommy},
 journal = {arXiv preprint arXiv:1810.08693},
 title = {The total variation distance between high-dimensional {G}aussians},
 year = {2018}
}

@article{liu2019roberta,
 author = {Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
 journal = {arXiv preprint arXiv:1907.11692},
 title = {{RoBERTa}: A robustly optimized {BERT} pretraining approach},
 year = {2019}
}

@article{brown2020language,
 author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
 journal = {Advances in neural information processing systems},
 pages = {1877--1901},
 title = {Language models are few-shot learners},
 volume = {33},
 year = {2020}
}

@article{goyal2021larger,
 author = {Goyal, Naman and Du, Jingfei and Ott, Myle and Anantharaman, Giri and Conneau, Alexis},
 journal = {arXiv preprint arXiv:2105.00572},
 title = {Larger-scale transformers for multilingual masked language modeling},
 year = {2021}
}

@article{raffel2020exploring,
 author = {Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J},
 journal = {The Journal of Machine Learning Research},
 number = {1},
 pages = {5485--5551},
 publisher = {JMLRORG},
 title = {Exploring the limits of transfer learning with a unified text-to-text transformer},
 volume = {21},
 year = {2020}
}

@inproceedings{liu2022convnet,
 author = {Liu, Zhuang and Mao, Hanzi and Wu, Chao-Yuan and Feichtenhofer, Christoph and Darrell, Trevor and Xie, Saining},
 booktitle = {Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
 pages = {11976--11986},
 title = {A {ConvNet} for the 2020s},
 year = {2022}
}

@inproceedings{woo2023convnext,
 author = {Woo, Sanghyun and Debnath, Shoubhik and Hu, Ronghang and Chen, Xinlei and Liu, Zhuang and Kweon, In So and Xie, Saining},
 booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
 pages = {16133--16142},
 title = {{ConvNeXt V2}: Co-designing and scaling convnets with masked autoencoders},
 year = {2023}
}

@inproceedings{wu2021cvt,
 author = {Wu, Haiping and Xiao, Bin and Codella, Noel and Liu, Mengchen and Dai, Xiyang and Yuan, Lu and Zhang, Lei},
 booktitle = {Proceedings of the IEEE/CVF international conference on computer vision},
 pages = {22--31},
 title = {{Cvt}: Introducing convolutions to vision transformers},
 year = {2021}
}

@article{feng2022rank,
 author = {Feng, Ruili and Zheng, Kecheng and Huang, Yukun and Zhao, Deli and Jordan, Michael and Zha, Zheng-Jun},
 journal = {Advances in Neural Information Processing Systems},
 pages = {33054--33065},
 title = {Rank diminishing in deep neural networks},
 volume = {35},
 year = {2022}
}

@article{joudaki2023impact,
  title={On the impact of activation and normalization in obtaining isometric embeddings at initialization},
  author={Joudaki, Amir and Daneshmand, Hadi and Bach, Francis},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={39855--39875},
  year={2023}
}

@inproceedings{he2016res,
 author = {He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
 booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},
 pages = {770--778},
 title = {Deep residual learning for image recognition},
 year = {2016}
}

@article{burkholz2019initialization,
 author = {Burkholz, Rebekka and Dubatovka, Alina},
 journal = {Advances in Neural Information Processing Systems},
 title = {Initialization of relus for dynamical isometry},
 volume = {32},
 year = {2019}
}

@article{brock2021characterizing,
 author = {Brock, Andrew and De, Soham and Smith, Samuel L},
 journal = {arXiv preprint arXiv:2101.08692},
 title = {Characterizing signal propagation to close the performance gap in unnormalized resnets},
 year = {2021}
}

@inproceedings{blumenfeld2020beyond,
 author = {Blumenfeld, Yaniv and Gilboa, Dar and Soudry, Daniel},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {960--969},
 title = {Beyond signal propagation: is feature diversity necessary in deep neural network initialization?},
 year = {2020}
}

@article{mishkin2015all,
 author = {Mishkin, Dmytro and Matas, Jiri},
 journal = {arXiv preprint arXiv:1511.06422},
 title = {All you need is a good init},
 year = {2015}
}

@inproceedings{arjovsky2016unitary,
 author = {Arjovsky, Martin and Shah, Amar and Bengio, Yoshua},
 booktitle = {International conference on machine learning},
 organization = {PMLR},
 pages = {1120--1128},
 title = {Unitary evolution recurrent neural networks},
 year = {2016}
}

@article{le2015simple,
 author = {Le, Quoc V and Jaitly, Navdeep and Hinton, Geoffrey E},
 journal = {arXiv preprint arXiv:1504.00941},
 title = {A simple way to initialize recurrent networks of rectified linear units},
 year = {2015}
}

@inproceedings{henaff2016recurrent,
 author = {Henaff, Mikael and Szlam, Arthur and LeCun, Yann},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {2034--2042},
 title = {Recurrent orthogonal networks and long-memory tasks},
 year = {2016}
}

@article{fukumizu1998effect,
 author = {Fukumizu, Kenji},
 journal = {Gen},
 number = {04},
 pages = {1E--03},
 title = {Effect of batch learning in multilayer neural networks},
 volume = {1},
 year = {1998}
}

@article{baldi1995learning,
 author = {Baldi, Pierre F and Hornik, Kurt},
 journal = {IEEE Transactions on neural networks},
 number = {4},
 pages = {837--858},
 publisher = {IEEE},
 title = {Learning in linear neural networks: A survey},
 volume = {6},
 year = {1995}
}

@inproceedings{saxe2013learning,
 author = {Saxe, Andrew M and McClellans, James L and Ganguli, Surya},
 booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society},
 title = {Learning hierarchical categories in deep neural networks},
 volume = {35},
 year = {2013}
}

@article{saxe2019mathematical,
 author = {Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
 journal = {Proceedings of the National Academy of Sciences},
 number = {23},
 pages = {11537--11546},
 publisher = {National Acad Sciences},
 title = {A mathematical theory of semantic development in deep neural networks},
 volume = {116},
 year = {2019}
}

@article{baldi1989neural,
 author = {Baldi, Pierre and Hornik, Kurt},
 journal = {Neural networks},
 number = {1},
 pages = {53--58},
 publisher = {Elsevier},
 title = {Neural networks and principal component analysis: Learning from examples without local minima},
 volume = {2},
 year = {1989}
}

@article{yun2017global,
 author = {Yun, Chulhee and Sra, Suvrit and Jadbabaie, Ali},
 journal = {arXiv preprint arXiv:1707.02444},
 title = {Global optimality conditions for deep neural networks},
 year = {2017}
}

@inproceedings{shamir2019exponential,
 author = {Shamir, Ohad},
 booktitle = {Conference on Learning Theory},
 organization = {PMLR},
 pages = {2691--2713},
 title = {Exponential convergence time of gradient descent for one-dimensional deep linear neural networks},
 year = {2019}
}

@inproceedings{du2019width,
 author = {Du, Simon and Hu, Wei},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {1655--1664},
 title = {Width provably matters in optimization for deep linear neural networks},
 year = {2019}
}

@article{collins2006integration,
 author = {Collins, Beno{\^\i}t and {\'S}niady, Piotr},
 journal = {Communications in Mathematical Physics},
 number = {3},
 pages = {773--795},
 publisher = {Springer},
 title = {Integration with respect to the Haar measure on unitary, orthogonal and symplectic group},
 volume = {264},
 year = {2006}
}

@article{weingarten1978asymptotic,
 author = {Weingarten, Don},
 journal = {Journal of Mathematical Physics},
 number = {5},
 pages = {999--1001},
 publisher = {American Institute of Physics},
 title = {Asymptotic behavior of group integrals in the limit of infinite rank},
 volume = {19},
 year = {1978}
}

@article{banica2011polynomial,
 author = {Banica, Teodor and Collins, Benoit and Schlenker, Jean-Marc},
 journal = {Journal of Combinatorial Theory, Series A},
 number = {3},
 pages = {778--795},
 publisher = {Elsevier},
 title = {On polynomial integrals over the orthogonal group},
 volume = {118},
 year = {2011}
}

@article{collins2022weingarten,
 author = {Collins, Benoit and Matsumoto, Sho and Novak, Jonathan},
 journal = {arXiv preprint arXiv:2109.14890},
 title = {The {Weingarten} calculus},
 year = {2022}
}

@article{hendrycks2016gaussian,
 author = {Hendrycks, Dan and Gimpel, Kevin},
 journal = {arXiv preprint arXiv:1606.08415},
 title = {Gaussian error linear units {(GELUs)}},
 year = {2016}
}

@article{he2023deep,
 author = {He, Bobby and Martens, James and Zhang, Guodong and Botev, Aleksandar and Brock, Andrew and Smith, Samuel L and Teh, Yee Whye},
 journal = {arXiv preprint arXiv:2302.10322},
 title = {Deep transformers without shortcuts: Modifying self-attention for faithful signal propagation},
 year = {2023}
}

@article{noci2023shaped,
 author = {Noci, Lorenzo and Li, Chuning and Li, Mufan Bill and He, Bobby and Hofmann, Thomas and Maddison, Chris and Roy, Daniel M},
 journal = {arXiv preprint arXiv:2306.17759},
 title = {The Shaped Transformer: Attention Models in the Infinite Depth-and-Width Limit},
 year = {2023}
}

@inproceedings{vorontsov2017orthogonality,
 author = {Vorontsov, Eugene and Trabelsi, Chiheb and Kadoury, Samuel and Pal, Chris},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {3570--3578},
 title = {On orthogonality and learning recurrent networks with long term dependencies},
 year = {2017}
}

@inproceedings{lezcano2019cheap,
 author = {Lezcano-Casado, Mario and Mart{\i}nez-Rubio, David},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {3794--3803},
 title = {Cheap orthogonal constraints in neural networks: A simple parametrization of the orthogonal and unitary group},
 year = {2019}
}

@inproceedings{jose2018kronecker,
 author = {Jose, Cijo and Ciss{\'e}, Moustapha and Fleuret, Francois},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {2380--2389},
 title = {Kronecker recurrent units},
 year = {2018}
}

@inproceedings{mhammedi2017efficient,
 author = {Mhammedi, Zakaria and Hellicar, Andrew and Rahman, Ashfaqur and Bailey, James},
 booktitle = {International Conference on Machine Learning},
 organization = {PMLR},
 pages = {2401--2409},
 title = {Efficient orthogonal parametrisation of recurrent neural networks using {Householder} reflections},
 year = {2017}
}

@article{collins2003moments,
 author = {Collins, Beno{\^\i}t},
 journal = {International Mathematics Research Notices},
 number = {17},
 pages = {953--982},
 publisher = {OUP},
 title = {Moments and cumulants of polynomial random variables on unitary groups, the {Itzykson-Zuber} integral, and free probability},
 volume = {2003},
 year = {2003}
}

@article{collins2009some,
 author = {Collins, Beno{\^\i}t and Matsumoto, Sho},
 journal = {Journal of Mathematical Physics},
 number = {11},
 publisher = {AIP Publishing},
 title = {On some properties of orthogonal {Weingarten} functions},
 volume = {50},
 year = {2009}
}


@inproceedings{krizhevsky2012imagenet,
 author = {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
 booktitle = {Advances in neural information processing systems},
 pages = {1097--1105},
 title = {Imagenet classification with deep convolutional neural networks},
 volume = {25},
 year = {2012}
}

@article{alain2016understanding,
 author = {Alain, Guillaume and Bengio, Yoshua},
 journal = {arXiv preprint arXiv:1610.01644},
 title = {Understanding intermediate layers using linear classifier probes},
 year = {2016}
}

@article{kingma2014adam,
 author = {Kingma, Diederik P and Ba, Jimmy},
 journal = {arXiv preprint arXiv:1412.6980},
 title = {Adam: A method for stochastic optimization},
 year = {2014}
}

@article{bengio1994learning,
 author = {Bengio, Yoshua and Simard, Patrice and Frasconi, Paolo},
 journal = {IEEE transactions on neural networks},
 number = {2},
 pages = {157--166},
 publisher = {IEEE},
 title = {Learning long-term dependencies with gradient descent is difficult},
 volume = {5},
 year = {1994}
}

@inproceedings{he2015delving,
 author = {He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
 booktitle = {Proceedings of the IEEE international conference on computer vision},
 pages = {1026--1034},
 title = {Delving deep into rectifiers: Surpassing human-level performance on {ImageNet} classification},
 year = {2015}
}

@inproceedings{glorot2011deep,
 author = {Glorot, Xavier and Bordes, Antoine and Bengio, Yoshua},
 booktitle = {Proceedings of the fourteenth international conference on artificial intelligence and statistics},
 pages = {315--323},
 publisher = {JMLR Workshop and Conference Proceedings},
 title = {Deep sparse rectifier neural networks},
 year = {2011}
}

@inproceedings{maas2013rectifier,
 author = {Maas, Andrew L and Hannun, Awni Y and Ng, Andrew Y},
 booktitle = {Proc. ICML},
 number = {1},
 pages = {3},
 title = {Rectifier nonlinearities improve neural network acoustic models},
 volume = {30},
 year = {2013}
}

@article{clevert2015fast,
 author = {Clevert, Djork-Arn{\'e} and Unterthiner, Thomas and Hochreiter, Sepp},
 journal = {arXiv preprint arXiv:1511.07289},
 title = {Fast and accurate deep network learning by exponential linear units (elus)},
 year = {2015}
}

@article{ramachandran2017searching,
 author = {Ramachandran, Prajit and Zoph, Barret and Le, Quoc V},
 journal = {arXiv preprint arXiv:1710.05941},
 title = {Searching for activation functions},
 year = {2017}
}

@article{pennington2017resurrecting,
 author = {Pennington, Jeffrey and Schoenholz, Samuel and Ganguli, Surya},
 journal = {Advances in neural information processing systems},
 title = {Resurrecting the sigmoid in deep learning through dynamical isometry: theory and practice},
 volume = {30},
 year = {2017}
}

@article{frankle2018lottery,
 author = {Frankle, Jonathan and Carbin, Michael},
 journal = {arXiv preprint arXiv:1803.03635},
 title = {The lottery ticket hypothesis: Finding sparse, trainable neural networks},
 year = {2018}
}

@article{pascanu2013difficulty,
 author = {Pascanu, Razvan and Mikolov, Tomas and Bengio, Yoshua},
 journal = {arXiv preprint arXiv:1211.5063},
 title = {On the difficulty of training recurrent neural networks},
 year = {2013}
}

@article{hanin2018neural,
 author = {Hanin, Boris},
 journal = {Advances in neural information processing systems},
 title = {Which neural net architectures give rise to exploding and vanishing gradients?},
 volume = {31},
 year = {2018}
}

@article{noci2022signal,
 author = {Noci, Lorenzo and Anagnostidis, Sotiris and Biggio, Luca and Orvieto, Antonio and Singh, Sidak Pal and Lucchi, Aurelien},
 journal = {Advances in Neural Information Processing Systems},
 pages = {27198--27211},
 title = {Signal propagation in {Transformers}: Theoretical perspectives and the role of rank collapse},
 volume = {35},
 year = {2022}
}

@article{hochreiter1991untersuchungen,
 author = {Hochreiter, Sepp},
 journal = {Diploma, Technische Universit{\"a}t M{\"u}nchen},
 number = {1},
 pages = {31},
 title = {Untersuchungen zu dynamischen neuronalen Netzen},
 volume = {91},
 year = {1991}
}

@article{hochreiter1998vanishing,
 author = {Hochreiter, Sepp},
 journal = {International Journal of Uncertainty, Fuzziness and Knowledge-Based Systems},
 number = {02},
 pages = {107--116},
 publisher = {World Scientific},
 title = {The vanishing gradient problem during learning recurrent neural nets and problem solutions},
 volume = {6},
 year = {1998}
}

@article{srivastava2015training,
 author = {Srivastava, Rupesh Kumar and Greff, Klaus and Schmidhuber, J{\"u}rgen},
 journal = {Advances in Neural Information Processing Systems},
 title = {Training very deep networks},
 year = {2015}
}

@article{bengio2007scaling,
 author = {Bengio, Yoshua and LeCun, Yann},
 journal = {Large-scale kernel machines},
 pages = {321-360},
 title = {Scaling learning algorithms toward AI},
 year = {2007}
}

@article{hanin2018start,
 author = {Hanin, Boris and Rolnick, David},
 journal = {Advances in neural information processing systems},
 title = {How to start training: The effect of initialization and architecture},
 volume = {31},
 year = {2018}
}

@inproceedings{zeiler2014visualizing,
 author = {Zeiler, Matthew D and Fergus, Rob},
 booktitle = {Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part I 13},
 organization = {Springer},
 pages = {818--833},
 title = {Visualizing and understanding convolutional networks},
 year = {2014}
}

@article{zhang2019fixup,
 author = {Zhang, Hongyi and Dauphin, Yann N and Ma, Tengyu},
 journal = {arXiv preprint arXiv:1901.09321},
 title = {Fixup initialization: Residual learning without normalization},
 year = {2019}
}

@inproceedings{luo2019adaptive,
 author = {Luo, Liangchen and Xiong, Yuanhao and Liu, Yan and Sun, Xu},
 journal = {arXiv preprint arXiv:1902.09843},
 title = {Adaptive gradient methods with dynamic bound of learning rate},
 year = {2019}
}

@article{zhang2019gradient,
 author = {Zhang, Jiawei},
 journal = {arXiv preprint arXiv:1903.03614},
 title = {Gradient descent based optimization algorithms for deep learning models training},
 year = {2019}
}

@article{yang2020tensor,
 author = {Yang, Greg},
 journal = {arXiv preprint arXiv:2006.14548},
 title = {Tensor programs ii: Neural tangent kernel for any architecture},
 year = {2020}
}

@article{pennington2017nonlinear,
 author = {Pennington, Jeffrey and Worah, Pratik},
 journal = {Advances in {N}eural {I}nformation {P}rocessing {S}ystems},
 title = {Nonlinear random matrix theory for deep learning},
 volume = {30},
 year = {2017}
}

@article{newman1986distribution,
  title={The distribution of Lyapunov exponents: Exact results for random matrices},
  author={Newman, Charles M},
  journal={Communications in mathematical physics},
  year={1986},
  publisher={Springer}
}

@article{bahri2020statistical,
  title={Statistical mechanics of deep learning},
  author={Bahri, Yasaman and Kadmon, Jonathan and Pennington, Jeffrey and Schoenholz, Sam S and Sohl-Dickstein, Jascha and Ganguli, Surya},
  journal={Annual Review of Condensed Matter Physics},
  year={2020},
  publisher={Annual Reviews}
}

@incollection{NEURIPS2019_9015,
title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library},
author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and Desmaison, Alban and Kopf, Andreas and Yang, Edward and DeVito, Zachary and Raison, Martin and Tejani, Alykhan and Chilamkurthy, Sasank and Steiner, Benoit and Fang, Lu and Bai, Junjie and Chintala, Soumith},
booktitle = {Advances in Neural Information Processing Systems},
year = {2019},
}

@article{hairer2010convergence,
  title={Convergence of Markov processes},
  author={Hairer, Martin},
  journal={Lecture notes},
  year={2010}
}

@book{bougerol2012products,
  title={Products of random matrices with applications to Schr{\"o}dinger operators},
  author={Bougerol, Philippe and others},
  year={2012},
  publisher={Springer Science \& Business Media}
}

@book{kushner2003stochastic,
  title={Stochastic approximation and recursive algorithms and applications},
  author={Kushner, Harold and Yin, G George},
  year={2003},
  publisher={Springer Science \& Business Media}
}

@article{garriga2018deep,
  title={Deep convolutional networks as shallow gaussian processes},
  author={Garriga-Alonso, Adria and Rasmussen, Carl Edward and Aitchison, Laurence},
  journal={International Conference on Learning Representations},
  year={2019}
}

@article{de2018random,
  title={Random deep neural networks are biased towards simple functions},
  author={De Palma, Giacomo and Kiani, Bobak Toussi and Lloyd, Seth},
  journal={Advances in Neural Information Processing Systems},
  year={2019}
}

@article{lee2017deep,
  title={Deep neural networks as gaussian processes},
  author={Lee, Jaehoon and Bahri, Yasaman and Novak, Roman and Schoenholz, Samuel S and Pennington, Jeffrey and Sohl-Dickstein, Jascha},
  journal={International Conference on Learning Representations},
  year={2019}
}

@book{neal2012bayesian,
  title={Bayesian learning for neural networks},
  author={Neal, Radford M},
  year={1996},
  publisher={Springer Science \& Business Media}
}

@article{schoenholz2016deep,
  title={Deep information propagation},
  author={Schoenholz, Samuel S and Gilmer, Justin and Ganguli, Surya and Sohl-Dickstein, Jascha},
  journal={International Conference on Learning Representations},
  year={2017}
}

@article{frankle2020training,
  title={Training batchnorm and only batchnorm: On the expressive power of random features in CNNs},
  author={Frankle, Jonathan and Schwab, David J and Morcos, Ari S},
  journal={ICLR},
  year={2021}
}

@book{khasminskii2011stochastic,
  title={Stochastic stability of differential equations},
  author={Khasminskii, Rafail},
  volume={66},
  year={2011},
  publisher={Springer Science \& Business Media}
}

@techreport{kushner1967stochastic,
  title={Stochastic stability and control},
  author={Kushner, Harold J},
  publisher={Elsevier Academic Press},
  year={1967}
}

@book{kemeny1976markov,
  title={Markov chains},
  author={Kemeny, John G and Snell, J Laurie},
  year={1976},
  publisher={Springer-Verlag, New York}
}

@article{eberle2009markov,
  title={Markov processes},
  author={Eberle, Andreas},
  journal={Lecture Notes at University of Bonn},
  year={2009}
}

@article{bartlett2019gradient,
  title={Gradient descent with identity initialization efficiently learns positive-definite linear transformations by deep residual networks},
  author={Bartlett, Peter L. and Helmbold, David P. and Long, Philip M.},
  journal={Neural computation},
  year={2019},
  publisher={MIT Press}
}

@article{arora2018convergence,
  title={A convergence analysis of gradient descent for deep linear neural networks},
  author={Arora, Sanjeev and Cohen, Nadav and Golowich, Noah and Hu, Wei},
  journal={International Conference on Learning Representations},
  year={2019}
}

@inproceedings{huang2014kernel,
  title={Kernel methods match deep neural networks on timit},
  author={Huang, Po-Sen and Avron, Haim and Sainath, Tara N and Sindhwani, Vikas and Ramabhadran, Bhuvana},
  booktitle={ICASSP},
  year={2014},
}

@article{bietti2019inductive,
  title={On the inductive bias of neural tangent kernels},
  author={Bietti, Alberto and Mairal, Julien},
  journal={Advances in Neural Information Processing Systems},
  year={2019}
}

@article{hazan2015steps,
  title={Steps toward deep kernel methods from infinite neural networks},
  author={Hazan, Tamir and Jaakkola, Tommi},
  journal={arXiv preprint arXiv:1508.05133},
  year={2015}
}

@article{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey and others},
  year={2009},
  publisher={Technical Report TR-2009, University of Toronto, Toronto.}
}

@inproceedings{agarwal2021deep,
  title={A deep conditioning treatment of neural networks},
  author={Agarwal, Naman and Awasthi, Pranjal and Kale, Satyen},
  booktitle={Algorithmic Learning Theory},
  year={2021},
  organization={PMLR}
}

@article{brock2021high,
  title={High-performance large-scale image recognition without normalization},
  author={Brock, Andrew and De, Soham and Smith, Samuel L and Simonyan, Karen},
  journal={arXiv preprint arXiv:2102.06171},
  year={2021}
}

@inproceedings{huang2017densely,
  title={Densely connected convolutional networks},
  author={Huang, Gao and Liu, Zhuang and Van Der Maaten, Laurens and Weinberger, Kilian Q},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  year={2017}
}

@article{silver2017mastering,
  title={Mastering the game of go without human knowledge},
  author={Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
  journal={nature},
  year={2017},
}

@article{santurkar2018does,
  title={How Does Batch Normalization Help Optimization?(No, It Is Not About Internal Covariate Shift)},
  author={Santurkar, Shibani and Tsipras, Dimitris and Ilyas, Andrew and Madry, Aleksander},
  journal={Advances in Neural Information Processing Systems},
  year={2018}
}

@inproceedings{karakida2019normalization,
  title={The normalization method for alleviating pathological sharpness in wide neural networks},
  author={Karakida, Ryo and Akaho, Shotaro and Amari, Shun-ichi},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@article{arora2018theoretical,
  title={Theoretical analysis of auto rate-tuning by batch normalization},
  author={Arora, Sanjeev and Li, Zhiyuan and Lyu, Kaifeng},
  journal={International Conference on Learning Representations},
  year={2019}
}

@article{kohler2018exponential,
  title={Exponential convergence rates for Batch Normalization: The power of length-direction decoupling in non-convex optimization},
  author={Kohler, Jonas and Daneshmand, Hadi and Lucchi, Aurelien and Zhou, Ming and Neymeyr, Klaus and Hofmann, Thomas},
  journal={arXiv preprint arXiv:1805.10694},
  year={2018}
}


@inproceedings{schwab2021deep,
  title={Deep learning in high dimension: Neural network expression rates for analytic functions in L2 (Rd, $\gamma$d)},
  author={Schwab, Ch and Zech, J},
  year={2021},
  organization={Tech. Report 2021-40, Seminar for Applied Mathematics, ETH Z{\"u}rich}
}

@article{yang2019scaling,
  title={Scaling limits of wide neural networks with weight sharing: Gaussian process behavior, gradient independence, and neural tangent kernel derivation},
  author={Yang, Greg},
  journal={arXiv preprint arXiv:1902.04760},
  year={2019}
}


@inproceedings{rigatos2006feed,
  title={Feed-forward neural networks using Hermite polynomial activation functions},
  author={Rigatos, Gerasimos G. and Tzafestas, Spyros G.},
  booktitle={Advances in Artificial Intelligence: 4th Helenic Conference on AI, SETN 2006, Proceedings 4},
  pages={323--333},
  year={2006},
  organization={Springer}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@inproceedings{lokhande2020generating,
  title={Generating accurate pseudo-labels in semi-supervised learning and avoiding overconfident predictions via hermite polynomial activations},
  author={Lokhande, Vishnu Suresh and Tasneeyapant, Songwong and Venkatesh, Abhay and Ravi, Sathya N. and Singh, Vikas},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
  pages={11435--11443},
  year={2020}
}
@techreport{cifar10,
  author       = {Krizhevsky, Alex and Hinton, Geoffrey},
  title        = {Learning Multiple Layers of Features from Tiny Images},
  institution  = {University of Toronto},
  year         = {2009}
}

@article{ma2005constructive,
  title={Constructive feedforward neural networks using Hermite polynomial activation functions},
  author={Ma, Liying and Khorasani, Khashayar},
  journal={IEEE Transactions on Neural Networks},
  volume={16},
  number={4},
  pages={821--833},
  year={2005},
  publisher={IEEE}
}



@article{bai2008large,
  title={Large sample covariance matrices without independence structures in columns},
  author={Bai, Zhidong and Zhou, Wang},
  journal={Statistica Sinica},
  pages={425--442},
  year={2008},
  publisher={JSTOR}
}

@article{li2021future,
  title={The future is log-Gaussian: ResNets and their infinite-depth-and-width limit at initialization},
  author={Li, Mufan and Nica, Mihai and Roy, Dan},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={7852--7864},
  year={2021}
}

@article{bryson2021marchenko,
  title={Marchenko--Pastur law with relaxed independence conditions},
  author={Bryson, Jennifer and Vershynin, Roman and Zhao, Hongkai},
  journal={Random Matrices: Theory and Applications},
  volume={10},
  number={04},
  pages={2150040},
  year={2021},
  publisher={World Scientific}
}

@article{anagnostidissignal,
  title={Signal Propagation in Transformers: Theoretical Perspectives and the Role of Rank Collapse},
  author={Noci, Lorenzo and Anagnostidis, Sotiris and Biggio, Luca and Orvieto, Antonio and Singh, Sidak Pal and Lucchi, Aurelien},
  journal={arXiv preprint arXiv:2206.03126},
  year={2022}
}

@inproceedings{xiong2020layer,
  title={On layer normalization in the transformer architecture},
  author={Xiong, Ruibin and Yang, Yunchang and He, Di and Zheng, Kai and Zheng, Shuxin and Xing, Chen and Zhang, Huishuai and Lan, Yanyan and Wang, Liwei and Liu, Tieyan},
  booktitle={International Conference on Machine Learning},
  pages={10524--10533},
  year={2020},
}

@inproceedings{
schoenholz2017deep,
title={Deep Information Propagation},
author={Samuel S. Schoenholz and Justin Gilmer and Surya Ganguli and Jascha Sohl-Dickstein},
booktitle={International Conference on Learning Representations},
year={2017}
}

@article{lee2019wide,
  title={Wide neural networks of any depth evolve as linear models under gradient descent},
  author={Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@article{chizat2019lazy,
  title={On lazy training in differentiable programming},
  author={Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{li2022neural,
  title={The Neural Covariance SDE: Shaped Infinite Depth-and-Width Networks at Initialization},
  author={Li, Mufan Bill and Nica, Mihai and Roy, Daniel M.},
  journal={Advances in Neural Information Processing Systems},
  year={2022}
}
@inproceedings{matthews2018gaussian,
title={Gaussian Process Behaviour in Wide Deep Neural Networks},
author={Alexander G. de G. Matthews and Jiri Hron and Mark Rowland and Richard E. Turner and Zoubin Ghahramani},
booktitle={International Conference on Learning Representations},
year={2018},
}
@inproceedings{pennington2018emergence,
  title={The emergence of spectral universality in deep networks},
  author={Pennington, Jeffrey and Schoenholz, Samuel and Ganguli, Surya},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={1924--1932},
  year={2018},
}
@article{schur1911bemerkungen,
  title={Bemerkungen zur Theorie der beschr{\"a}nkten Bilinearformen mit unendlich vielen Ver{\"a}nderlichen.},
  author={Schur, Jssai},
  year={1911},
  publisher={Walter de Gruyter, Berlin/New York Berlin, New York}
}
@article{mehler1866ueber,
  title={Ueber die Entwicklung einer Function von beliebig vielen Variablen nach Laplaceschen Functionen h{\"o}herer Ordnung.},
  author={Mehler, F Gustav},
  year={1866},
  journal={Journal für die Reine und Angewandte Mathematik (in German) },
  publisher={Walter de Gruyter, Berlin/New York Berlin, New York}
}
@article{gershgorin1931uber,
  title={Uber die abgrenzung der eigenwerte einer matrix},
  author={Gershgorin, Semyon Aranovich},
  number={6},
  pages={749--754},
  year={1931}
}

@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}

@inproceedings{ioffe2015batch,
  title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
  author={Ioffe, Sergey and Szegedy, Christian},
  booktitle={International Conference on Machine Learning},
  pages={448--456},
  year={2015},
  organization={pmlr}
}

@inproceedings{yang2018a,
title={A Mean Field Theory of Batch Normalization},
author={Greg Yang and Jeffrey Pennington and Vinay Rao and Jascha Sohl-Dickstein and Samuel S. Schoenholz},
booktitle={International Conference on Learning Representations},
year={2019},
}

@article{daniely2016toward,
  title={Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
  author={Daniely, Amit and Frostig, Roy and Singer, Yoram},
  journal={Advances in Neural Information Processing Systems},
  volume={29},
  year={2016}
}

@article{styan1973hadamard,
  title={Hadamard products and multivariate statistical analysis},
  author={Styan, George PH},
  journal={Linear algebra and its applications},
  volume={6},
  pages={217--240},
  year={1973},
  publisher={Elsevier}
}

@article{zhang2020expressivity,
  title={The expressivity and training of deep neural networks: Toward the edge of chaos?},
  author={Zhang, Gege and Li, Gangwei and Shen, Weining and Zhang, Weidong},
  journal={Neurocomputing},
  volume={386},
  pages={8--17},
  year={2020},
  publisher={Elsevier}
}
@article{lubana2021beyond,
  title={Beyond batchnorm: Towards a unified understanding of normalization in deep learning},
  author={Lubana, Ekdeep S and Dick, Robert and Tanaka, Hidenori},
  journal={Advances in Neural Information Processing Systems},
  year={2021}
}
@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={NAACL},
  year={2018}
}
@article{touvron2023llama,
  title={Llama: Open and efficient foundation language models},
  author={Touvron, Hugo and Lavril, Thibaut and Izacard, Gautier and Martinet, Xavier and Lachaux, Marie-Anne and Lacroix, Timoth{\'e}e and Rozi{\`e}re, Baptiste and Goyal, Naman and Hambro, Eric and Azhar, Faisal and others},
  journal={arXiv preprint arXiv:2302.13971},
  year={2023}
}
@inproceedings{nair2010rectified,
  title={Rectified linear units improve restricted boltzmann machines},
  author={Nair, Vinod and Hinton, Geoffrey E},
  booktitle={Proceedings of the 27th international conference on machine learning (ICML-10)},
  pages={807--814},
  year={2010}
}

@article{bapat1985majorization,
  title={On majorization and Schur products},
  author={Bapat, Ravindra B and Sunder, Vaikalathur S},
  journal={Linear algebra and its applications},
  volume={72},
  pages={107--117},
  year={1985},
  publisher={Elsevier}
}

@article{aldaz2008selfimprovemvent,
  title={Selfimprovemvent of the inequality between arithmetic and geometric means},
  author={Aldaz, JM},
  journal={arXiv preprint arXiv:0807.1788},
  year={2008}
}

@article{mallat2016understanding,
  title={Understanding deep convolutional networks},
  author={Mallat, St{\'e}phane},
  journal={Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
  volume={374},
  number={2065},
  pages={20150203},
  year={2016},
  publisher={The Royal Society Publishing}
}

@article{zhang2022deep,
  title={Deep learning without shortcuts: Shaping the kernel with tailored rectifiers},
  author={Zhang, Guodong and Botev, Aleksandar and Martens, James},
  journal={arXiv preprint arXiv:2203.08120},
  year={2022}
}
@article{martens2021rapid,
  title={Rapid training of deep neural networks without skip connections or normalization layers using deep kernel shaping},
  author={Martens, James and Ballard, Andy and Desjardins, Guillaume and Swirszcz, Grzegorz and Dalibard, Valentin and Sohl-Dickstein, Jascha and Schoenholz, Samuel S.},
  journal={arXiv preprint arXiv:2110.01765},
  year={2021}
}
@article{salimans2016weight,
  title={Weight normalization: A simple reparameterization to accelerate training of deep neural networks},
  author={Salimans, Tim and Kingma, Durk P},
  journal={Advances in Neural Information Processing Systems},
  volume={29},
  year={2016}
}
@article{daneshmand2021batch,
  title={Batch normalization orthogonalizes representations in deep random networks},
  author={Daneshmand, Hadi and Joudaki, Amir and Bach, Francis},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={4896--4906},
  year={2021}
}
@article{daneshmand2020batch,
  title={Batch normalization provably avoids ranks collapse for randomly initialised deep networks},
  author={Daneshmand, Hadi and Kohler, Jonas and Bach, Francis and Hofmann, Thomas and Lucchi, Aurelien},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={18387--18398},
  year={2020}
}

@article{poole2016exponential,
  title={Exponential expressivity in deep neural networks through transient chaos},
  author={Poole, Ben and Lahiri, Subhaneil and Raghu, Maithra and Sohl-Dickstein, Jascha and Ganguli, Surya},
  journal={Advances in Neural Information Processing Systems},
  volume={29},
  year={2016}
}
@article{klambauer2017self,
  title={Self-normalizing neural networks},
  author={Klambauer, G{\"u}nter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}


@article{ba2016layer,
  title={Layer normalization},
  author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E.},
  journal={arXiv preprint arXiv:1607.06450},
  year={2016}
}
@article{joudaki2023bridging,
      title={On Bridging the Gap between Mean Field and Finite Width in Deep Random Neural Networks with Batch Normalization}, 
      author={Joudaki, Amir and Daneshmand, Hadi and Bach, Francis},
      year={2023},
      journal={International Conference on Machine Learning}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017}
}
@inproceedings{dong2021attention,
  title={Attention is not all you need: Pure attention loses rank doubly exponentially with depth},
  author={Dong, Yihe and Cordonnier, Jean-Baptiste and Loukas, Andreas},
  booktitle={International Conference on Machine Learning},
  pages={2793--2803},
  year={2021},
}
@article{bjorck2018understanding,
  title={Understanding batch normalization},
  author={Bjorck, Nils and Gomes, Carla P. and Selman, Bart and Weinberger, Kilian Q.},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
 
@inproceedings{xiao2018dynamical,
  title={Dynamical isometry and a mean field theory of {CNN}s: How to train 10,000-layer vanilla convolutional neural networks},
  author={Xiao, Lechao and Bahri, Yasaman and Sohl-Dickstein, Jascha and Schoenholz, Samuel and Pennington, Jeffrey},
  booktitle={International Conference on Machine Learning},
  pages={5393--5402},
  year={2018},
}
@article{saxe2013exact,
  title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
  author={Saxe, Andrew M. and McClelland, James L. and Ganguli, Surya},
  journal={arXiv preprint arXiv:1312.6120},
  year={2013}
}

@article{mourtada2022exact,
  title={Exact minimax risk for linear least squares, and the lower tail of sample covariance matrices},
  author={Mourtada, Jaouad},
  journal={The Annals of Statistics},
  volume={50},
  number={4},
  pages={2157--2178},
  year={2022},
  publisher={Institute of Mathematical Statistics}
}

@article{boyd1984asymptotic,
  title={Asymptotic coefficients of Hermite function series},
  author={Boyd, John P},
  journal={Journal of Computational Physics},
  volume={54},
  number={3},
  pages={382--410},
  year={1984},
  publisher={Elsevier}
}

@article{hille1940contributions,
  title={Contributions to the theory of Hermitian series. II. The representation problem},
  author={Hille, Einar},
  journal={Transactions of the American Mathematical Society},
  volume={47},
  number={1},
  pages={80--94},
  year={1940}
}

@book{bateman1953higher,
  title={Higher transcendental functions [volumes i-iii]},
  author={Bateman, Harry},
  volume={1},
  year={1953},
  publisher={McGRAW-HILL book company}
}