Upgrade to Pro — share decks privately, control downloads, hide ads and more …

力学系から見た現代的な機械学習

Avatar for Han Bao Han Bao
November 25, 2025

 力学系から見た現代的な機械学習

Avatar for Han Bao

Han Bao

November 25, 2025
Tweet

Other Decks in Science

Transcript

  1. む 2013 17 : む 2017 19 : む 2019

    22 : む 2022 25 む 2025 00 2018 2019 2020 2021 ।ڭत @౷਺ݚ 2022 2023 2024 2025 2017 ത࢜՝ఔ @౦େCS म࢜՝ఔ @౦େCS ಛఆॿڭ @ژେനඑɾ৘ใֶ ͖͕͚͞
  2. Hop eld <latexit sha1_base64="9fIns3pW6kl532WcVrKnw2Ak05I=">AAACinicfZDfahNBFMYn678aq6Z66c1iEETasBtKWxGl0AreiBVMW+iE5ezk7GbSmdll5mzbsOzb+DTe6o1v42waQVPxwDA/vvMNZ86Xlko6iqKfneDW7Tt3763d7z5Yf/jocW/jybErKitwJApV2NMUHCppcESSFJ6WFkGnCk/S84O2f3KB1snCfKF5iWMNuZGZFEBeSnrvOOEV1e8N2nzevN3imQURD7mrdFLLzVlz6a9Z4xLpktnWQpacpkjQKjLp9aNBtKjwJsRL6LNlHSUbnW0+KUSl0ZBQ4NxZHJU0rsGSFAqbLq8cliDOIcczjwY0unG9WLQJX3hlEmaF9cdQuFD/fFGDdm6uU+/UQFO32mvFf/ZSvTKZsr1xLU1ZERpxPTirVEhF2KYYTqRFQWruAYSV/u+hmIKPjnzW3S4/RL+cxY9+0KcSLVBhX9UcbK6lafyyOd9s6X9GuPpt9ORzjldTvQnHw0G8M9j5vN3fP1gmvsaesefsJYvZLttnH9gRGzHBvrJv7Dv7EawHw+B18ObaGnSWb56yvyo4/AVinsoT</latexit> Energy = 1 2 X i,j

    wijsisj X i ✓isi Input <latexit sha1_base64="EmPRd3f6t2Tl+5EEWNhjanjozwc=">AAACsnicfVFNb9QwEPWGrzZ8bdsjF4stCAFdJWjVcqxUDlwQRWLbinoVOd5J1lvbiewJ7SrKP+wf4G9wLQecbRCwRYxk6em9NzP2c1oq6TCKvvWCW7fv3L23th7ef/Dw0eP+xuaRKyorYCwKVdiTlDtQ0sAYJSo4KS1wnSo4Ts8OWv34K1gnC/MZFyVMNM+NzKTg6Kmkn7lEshzQsRRyaWrhZ7kmfBXT55QhXGAtM7rNXKWT+XlSy3njkrlvYDgD5Incbihj4c5ve+EFey4dNCEDM+0GJv1BNIyWRW+CuAMD0tVhstEbsWkhKg0GheLOncZRiZOaW5RCtcMrByUXZzyHUw8N1+Am9TKQhj7zzJRmhfXHIF2yf3bUXDu30Kl3ao4zt6q15D+1VK9sxuztpJamrBCMuF6cVYpiQdu06VRaEKgWHnBhpb87FTNuuUD/J2HI3oF/nIUPftHHEizHwr6sGbe5lqbxj83Z6xb9z8gvfhk98jnHq6neBEdvhvHucPfTaLB/0CW+Rp6Qp+QFicke2SfvySEZE0EuyXdyRX4Eo+BLwANxbQ16Xc8W+asC9RMR+djl</latexit> si ( +1 if P j wijsj ✓i 1 otherwise
  3. Hop eld <latexit sha1_base64="9fIns3pW6kl532WcVrKnw2Ak05I=">AAACinicfZDfahNBFMYn678aq6Z66c1iEETasBtKWxGl0AreiBVMW+iE5ezk7GbSmdll5mzbsOzb+DTe6o1v42waQVPxwDA/vvMNZ86Xlko6iqKfneDW7Tt3763d7z5Yf/jocW/jybErKitwJApV2NMUHCppcESSFJ6WFkGnCk/S84O2f3KB1snCfKF5iWMNuZGZFEBeSnrvOOEV1e8N2nzevN3imQURD7mrdFLLzVlz6a9Z4xLpktnWQpacpkjQKjLp9aNBtKjwJsRL6LNlHSUbnW0+KUSl0ZBQ4NxZHJU0rsGSFAqbLq8cliDOIcczjwY0unG9WLQJX3hlEmaF9cdQuFD/fFGDdm6uU+/UQFO32mvFf/ZSvTKZsr1xLU1ZERpxPTirVEhF2KYYTqRFQWruAYSV/u+hmIKPjnzW3S4/RL+cxY9+0KcSLVBhX9UcbK6lafyyOd9s6X9GuPpt9ORzjldTvQnHw0G8M9j5vN3fP1gmvsaesefsJYvZLttnH9gRGzHBvrJv7Dv7EawHw+B18ObaGnSWb56yvyo4/AVinsoT</latexit> Energy = 1 2 X i,j

    wijsisj X i ✓isi Input <latexit sha1_base64="EmPRd3f6t2Tl+5EEWNhjanjozwc=">AAACsnicfVFNb9QwEPWGrzZ8bdsjF4stCAFdJWjVcqxUDlwQRWLbinoVOd5J1lvbiewJ7SrKP+wf4G9wLQecbRCwRYxk6em9NzP2c1oq6TCKvvWCW7fv3L23th7ef/Dw0eP+xuaRKyorYCwKVdiTlDtQ0sAYJSo4KS1wnSo4Ts8OWv34K1gnC/MZFyVMNM+NzKTg6Kmkn7lEshzQsRRyaWrhZ7kmfBXT55QhXGAtM7rNXKWT+XlSy3njkrlvYDgD5Incbihj4c5ve+EFey4dNCEDM+0GJv1BNIyWRW+CuAMD0tVhstEbsWkhKg0GheLOncZRiZOaW5RCtcMrByUXZzyHUw8N1+Am9TKQhj7zzJRmhfXHIF2yf3bUXDu30Kl3ao4zt6q15D+1VK9sxuztpJamrBCMuF6cVYpiQdu06VRaEKgWHnBhpb87FTNuuUD/J2HI3oF/nIUPftHHEizHwr6sGbe5lqbxj83Z6xb9z8gvfhk98jnHq6neBEdvhvHucPfTaLB/0CW+Rp6Qp+QFicke2SfvySEZE0EuyXdyRX4Eo+BLwANxbQ16Xc8W+asC9RMR+djl</latexit> si ( +1 if P j wijsj ✓i 1 otherwise Output
  4. Hop eld <latexit sha1_base64="9fIns3pW6kl532WcVrKnw2Ak05I=">AAACinicfZDfahNBFMYn678aq6Z66c1iEETasBtKWxGl0AreiBVMW+iE5ezk7GbSmdll5mzbsOzb+DTe6o1v42waQVPxwDA/vvMNZ86Xlko6iqKfneDW7Tt3763d7z5Yf/jocW/jybErKitwJApV2NMUHCppcESSFJ6WFkGnCk/S84O2f3KB1snCfKF5iWMNuZGZFEBeSnrvOOEV1e8N2nzevN3imQURD7mrdFLLzVlz6a9Z4xLpktnWQpacpkjQKjLp9aNBtKjwJsRL6LNlHSUbnW0+KUSl0ZBQ4NxZHJU0rsGSFAqbLq8cliDOIcczjwY0unG9WLQJX3hlEmaF9cdQuFD/fFGDdm6uU+/UQFO32mvFf/ZSvTKZsr1xLU1ZERpxPTirVEhF2KYYTqRFQWruAYSV/u+hmIKPjnzW3S4/RL+cxY9+0KcSLVBhX9UcbK6lafyyOd9s6X9GuPpt9ORzjldTvQnHw0G8M9j5vN3fP1gmvsaesefsJYvZLttnH9gRGzHBvrJv7Dv7EawHw+B18ObaGnSWb56yvyo4/AVinsoT</latexit> Energy = 1 2 X i,j

    wijsisj X i ✓isi Space of <latexit sha1_base64="SxYxau1idJ47yfvCqZ9fy5fQN9I=">AAACVHicfZBNSwMxEIazWz/rV6tHL4tFEJGyK6V6LOjBi6hgVXRLmU2nNTbJLklWLcv+C6/6swT/iwfTWkGrOBB4eOcNM/NGCWfa+P6b4xampmdm5+aLC4tLyyul8uqFjlNFsUljHqurCDRyJrFpmOF4lSgEEXG8jPoHw/7lPSrNYnluBgm2BPQk6zIKxkrXYfbQzthdHubtUsWv+qPyfkMwhgoZ12m77NTCTkxTgdJQDlrfBH5iWhkowyjHvBimGhOgfejhjUUJAnUrG62ce5tW6XjdWNknjTdSv//IQGg9EJF1CjC3erI3FP/sRWJisunutzImk9SgpJ+Duyn3TOwN8/A6TCE1fGABqGJ2d4/eggJqbGrFYniI9jiFx3bQSYIKTKy2sxBUTzCZ22N74c6Q/jPC45fRks05mEz1N1zsVoN6tX5WqzQOxonPkXWyQbZIQPZIgxyRU9IklEjyRJ7Ji/PqvLsFd/rT6jrjP2vkR7nLH0Untac=</latexit> {wij } Energy
  5. Hop eld <latexit sha1_base64="9fIns3pW6kl532WcVrKnw2Ak05I=">AAACinicfZDfahNBFMYn678aq6Z66c1iEETasBtKWxGl0AreiBVMW+iE5ezk7GbSmdll5mzbsOzb+DTe6o1v42waQVPxwDA/vvMNZ86Xlko6iqKfneDW7Tt3763d7z5Yf/jocW/jybErKitwJApV2NMUHCppcESSFJ6WFkGnCk/S84O2f3KB1snCfKF5iWMNuZGZFEBeSnrvOOEV1e8N2nzevN3imQURD7mrdFLLzVlz6a9Z4xLpktnWQpacpkjQKjLp9aNBtKjwJsRL6LNlHSUbnW0+KUSl0ZBQ4NxZHJU0rsGSFAqbLq8cliDOIcczjwY0unG9WLQJX3hlEmaF9cdQuFD/fFGDdm6uU+/UQFO32mvFf/ZSvTKZsr1xLU1ZERpxPTirVEhF2KYYTqRFQWruAYSV/u+hmIKPjnzW3S4/RL+cxY9+0KcSLVBhX9UcbK6lafyyOd9s6X9GuPpt9ORzjldTvQnHw0G8M9j5vN3fP1gmvsaesefsJYvZLttnH9gRGzHBvrJv7Dv7EawHw+B18ObaGnSWb56yvyo4/AVinsoT</latexit> Energy = 1 2 X i,j

    wijsisj X i ✓isi Space of <latexit sha1_base64="SxYxau1idJ47yfvCqZ9fy5fQN9I=">AAACVHicfZBNSwMxEIazWz/rV6tHL4tFEJGyK6V6LOjBi6hgVXRLmU2nNTbJLklWLcv+C6/6swT/iwfTWkGrOBB4eOcNM/NGCWfa+P6b4xampmdm5+aLC4tLyyul8uqFjlNFsUljHqurCDRyJrFpmOF4lSgEEXG8jPoHw/7lPSrNYnluBgm2BPQk6zIKxkrXYfbQzthdHubtUsWv+qPyfkMwhgoZ12m77NTCTkxTgdJQDlrfBH5iWhkowyjHvBimGhOgfejhjUUJAnUrG62ce5tW6XjdWNknjTdSv//IQGg9EJF1CjC3erI3FP/sRWJisunutzImk9SgpJ+Duyn3TOwN8/A6TCE1fGABqGJ2d4/eggJqbGrFYniI9jiFx3bQSYIKTKy2sxBUTzCZ22N74c6Q/jPC45fRks05mEz1N1zsVoN6tX5WqzQOxonPkXWyQbZIQPZIgxyRU9IklEjyRJ7Ji/PqvLsFd/rT6jrjP2vkR7nLH0Untac=</latexit> {wij } Energy <latexit sha1_base64="jbxw4VlEsrOjJcyoYcGCaVKKfZg=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hobaFbymw6XUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkZmbn5vMLi4Wl5ZXVtfXixo2OU0WxTmMeq2YIGjmTWDfMcGwmCkGEHBthvzbqN+5QaRbLazNIsC0gkqzHKBgrXTU6Xme95JW9cbm/wZ9AiUzqolN0KkE3pqlAaSgHrVu+l5h2BsowynFYCFKNCdA+RNiyKEGgbmfjXYfujlW6bi9W9knjjtXvPzIQWg9EaJ0CzK2e7o3EP3uhmJpsesftjMkkNSjp5+Beyl0Tu6Mg3C5TSA0fWACqmN3dpbeggBobV6EQnKA9TuGZHXSeoAITq70sABUJJof22CjYH9F/Rrj/MlqyOfvTqf6Gm4Oyf1g+vKyUqrVJ4gtki2yTXeKTI1Ilp+SC1AklEXkgj+TJeXZenTfn/dOacyZ/NsmPyuU/APCEs/Q=</latexit> W0 <latexit sha1_base64="7QfmSbRU9jUZ7w5eHyWAh0pTY30=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hobaFbymw6XUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkZmbn5vMLi4Wl5ZXVtfXixo2OU0WxTmMeq2YIGjmTWDfMcGwmCkGEHBthvzbqN+5QaRbLazNIsC0gkqzHKBgrXTU6fme95JW9cbm/wZ9AiUzqolN0KkE3pqlAaSgHrVu+l5h2BsowynFYCFKNCdA+RNiyKEGgbmfjXYfujlW6bi9W9knjjtXvPzIQWg9EaJ0CzK2e7o3EP3uhmJpsesftjMkkNSjp5+Beyl0Tu6Mg3C5TSA0fWACqmN3dpbeggBobV6EQnKA9TuGZHXSeoAITq70sABUJJof22CjYH9F/Rrj/MlqyOfvTqf6Gm4Oyf1g+vKyUqrVJ4gtki2yTXeKTI1Ilp+SC1AklEXkgj+TJeXZenTfn/dOacyZ/NsmPyuU/APJrs/U=</latexit> W1 <latexit sha1_base64="TWCE2cASg/NetlRic2wlKMwAIRI=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiU3SLqsaAHL6KitUK3lNl0uoYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88vLBaWlldW14ql9Rsdp4pincY8VrchaORMYt0ww/E2UQgi5NgIe8fDfuMelWaxvDb9BFsCIsm6jIKx0lWjXW0Xy17FG5U7Df4YymRcF+2Ssx90YpoKlIZy0Lrpe4lpZaAMoxwHhSDVmADtQYRNixIE6lY22nXgblul43ZjZZ807kj9+SMDoXVfhNYpwNzpyd5Q/LMXionJpnvUyphMUoOSfg3uptw1sTsMwu0whdTwvgWgitndXXoHCqixcRUKwQna4xSe2UHnCSowsdrNAlCRYHJgj42CvSH9Z4SHb6Mlm7M/meo03FQr/kHl4HK/XDseJ75ANskW2SE+OSQ1ckouSJ1QEpFH8kSenRfnzXl3Pr6sOWf8Z4P8qlz+E/RSs/Y=</latexit> W2 attractor <latexit sha1_base64="P+BWQ+cxOm67fTzjf3D/m22OLUM=">AAACTXicfZBNSwMxEIaz9avWr6pHL4tFkCJlV6R6FPTgRaxoW8EtZTadrsEkuyRZsSz9CV71Z3n2h3gTMf0QtBUHAg/vvGFm3jDhTBvPe3NyM7Nz8wv5xcLS8srqWnF9o6HjVFGs05jH6iYEjZxJrBtmON4kCkGEHJvh/cmg33xApVksr00vwZaASLIuo2CsdNVsl9vFklfxhuVOgz+GEhlXrb3uHASdmKYCpaEctL71vcS0MlCGUY79QpBqTIDeQ4S3FiUI1K1suGvf3bFKx+3Gyj5p3KH680cGQuueCK1TgLnTk72B+GcvFBOTTfeolTGZpAYlHQ3uptw1sTsIwu0whdTwngWgitndXXoHCqixcRUKwSna4xSe20EXCSowsSpnAahIMNm3x0bB3oD+M8Ljt9GSzdmfTHUaGvsVv1qpXh6Ujk/GiefJFtkmu8Qnh+SYnJEaqRNKIvJEnsmL8+q8Ox/O58iac8Z/Nsmvyi18AeUas+4=</latexit> W⇤
  6. Hop eld <latexit sha1_base64="9fIns3pW6kl532WcVrKnw2Ak05I=">AAACinicfZDfahNBFMYn678aq6Z66c1iEETasBtKWxGl0AreiBVMW+iE5ezk7GbSmdll5mzbsOzb+DTe6o1v42waQVPxwDA/vvMNZ86Xlko6iqKfneDW7Tt3763d7z5Yf/jocW/jybErKitwJApV2NMUHCppcESSFJ6WFkGnCk/S84O2f3KB1snCfKF5iWMNuZGZFEBeSnrvOOEV1e8N2nzevN3imQURD7mrdFLLzVlz6a9Z4xLpktnWQpacpkjQKjLp9aNBtKjwJsRL6LNlHSUbnW0+KUSl0ZBQ4NxZHJU0rsGSFAqbLq8cliDOIcczjwY0unG9WLQJX3hlEmaF9cdQuFD/fFGDdm6uU+/UQFO32mvFf/ZSvTKZsr1xLU1ZERpxPTirVEhF2KYYTqRFQWruAYSV/u+hmIKPjnzW3S4/RL+cxY9+0KcSLVBhX9UcbK6lafyyOd9s6X9GuPpt9ORzjldTvQnHw0G8M9j5vN3fP1gmvsaesefsJYvZLttnH9gRGzHBvrJv7Dv7EawHw+B18ObaGnSWb56yvyo4/AVinsoT</latexit> Energy = 1 2 X i,j

    wijsisj X i ✓isi Space of <latexit sha1_base64="SxYxau1idJ47yfvCqZ9fy5fQN9I=">AAACVHicfZBNSwMxEIazWz/rV6tHL4tFEJGyK6V6LOjBi6hgVXRLmU2nNTbJLklWLcv+C6/6swT/iwfTWkGrOBB4eOcNM/NGCWfa+P6b4xampmdm5+aLC4tLyyul8uqFjlNFsUljHqurCDRyJrFpmOF4lSgEEXG8jPoHw/7lPSrNYnluBgm2BPQk6zIKxkrXYfbQzthdHubtUsWv+qPyfkMwhgoZ12m77NTCTkxTgdJQDlrfBH5iWhkowyjHvBimGhOgfejhjUUJAnUrG62ce5tW6XjdWNknjTdSv//IQGg9EJF1CjC3erI3FP/sRWJisunutzImk9SgpJ+Duyn3TOwN8/A6TCE1fGABqGJ2d4/eggJqbGrFYniI9jiFx3bQSYIKTKy2sxBUTzCZ22N74c6Q/jPC45fRks05mEz1N1zsVoN6tX5WqzQOxonPkXWyQbZIQPZIgxyRU9IklEjyRJ7Ji/PqvLsFd/rT6jrjP2vkR7nLH0Untac=</latexit> {wij } Energy <latexit sha1_base64="jbxw4VlEsrOjJcyoYcGCaVKKfZg=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hobaFbymw6XUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkZmbn5vMLi4Wl5ZXVtfXixo2OU0WxTmMeq2YIGjmTWDfMcGwmCkGEHBthvzbqN+5QaRbLazNIsC0gkqzHKBgrXTU6Xme95JW9cbm/wZ9AiUzqolN0KkE3pqlAaSgHrVu+l5h2BsowynFYCFKNCdA+RNiyKEGgbmfjXYfujlW6bi9W9knjjtXvPzIQWg9EaJ0CzK2e7o3EP3uhmJpsesftjMkkNSjp5+Beyl0Tu6Mg3C5TSA0fWACqmN3dpbeggBobV6EQnKA9TuGZHXSeoAITq70sABUJJof22CjYH9F/Rrj/MlqyOfvTqf6Gm4Oyf1g+vKyUqrVJ4gtki2yTXeKTI1Ilp+SC1AklEXkgj+TJeXZenTfn/dOacyZ/NsmPyuU/APCEs/Q=</latexit> W0 <latexit sha1_base64="7QfmSbRU9jUZ7w5eHyWAh0pTY30=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hobaFbymw6XUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkZmbn5vMLi4Wl5ZXVtfXixo2OU0WxTmMeq2YIGjmTWDfMcGwmCkGEHBthvzbqN+5QaRbLazNIsC0gkqzHKBgrXTU6fme95JW9cbm/wZ9AiUzqolN0KkE3pqlAaSgHrVu+l5h2BsowynFYCFKNCdA+RNiyKEGgbmfjXYfujlW6bi9W9knjjtXvPzIQWg9EaJ0CzK2e7o3EP3uhmJpsesftjMkkNSjp5+Beyl0Tu6Mg3C5TSA0fWACqmN3dpbeggBobV6EQnKA9TuGZHXSeoAITq70sABUJJof22CjYH9F/Rrj/MlqyOfvTqf6Gm4Oyf1g+vKyUqrVJ4gtki2yTXeKTI1Ilp+SC1AklEXkgj+TJeXZenTfn/dOacyZ/NsmPyuU/APJrs/U=</latexit> W1 <latexit sha1_base64="TWCE2cASg/NetlRic2wlKMwAIRI=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiU3SLqsaAHL6KitUK3lNl0uoYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88vLBaWlldW14ql9Rsdp4pincY8VrchaORMYt0ww/E2UQgi5NgIe8fDfuMelWaxvDb9BFsCIsm6jIKx0lWjXW0Xy17FG5U7Df4YymRcF+2Ssx90YpoKlIZy0Lrpe4lpZaAMoxwHhSDVmADtQYRNixIE6lY22nXgblul43ZjZZ807kj9+SMDoXVfhNYpwNzpyd5Q/LMXionJpnvUyphMUoOSfg3uptw1sTsMwu0whdTwvgWgitndXXoHCqixcRUKwQna4xSe2UHnCSowsdrNAlCRYHJgj42CvSH9Z4SHb6Mlm7M/meo03FQr/kHl4HK/XDseJ75ANskW2SE+OSQ1ckouSJ1QEpFH8kSenRfnzXl3Pr6sOWf8Z4P8qlz+E/RSs/Y=</latexit> W2 attractor <latexit sha1_base64="P+BWQ+cxOm67fTzjf3D/m22OLUM=">AAACTXicfZBNSwMxEIaz9avWr6pHL4tFkCJlV6R6FPTgRaxoW8EtZTadrsEkuyRZsSz9CV71Z3n2h3gTMf0QtBUHAg/vvGFm3jDhTBvPe3NyM7Nz8wv5xcLS8srqWnF9o6HjVFGs05jH6iYEjZxJrBtmON4kCkGEHJvh/cmg33xApVksr00vwZaASLIuo2CsdNVsl9vFklfxhuVOgz+GEhlXrb3uHASdmKYCpaEctL71vcS0MlCGUY79QpBqTIDeQ4S3FiUI1K1suGvf3bFKx+3Gyj5p3KH680cGQuueCK1TgLnTk72B+GcvFBOTTfeolTGZpAYlHQ3uptw1sTsIwu0whdTwngWgitndXXoHCqixcRUKwSna4xSe20EXCSowsSpnAahIMNm3x0bB3oD+M8Ljt9GSzdmfTHUaGvsVv1qpXh6Ujk/GiefJFtkmu8Qnh+SYnJEaqRNKIvJEnsmL8+q8Ox/O58iac8Z/Nsmvyi18AeUas+4=</latexit> W⇤ <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W)
  7. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y =

    W3 (W2 (W1x))
  8. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y =

    W3 (W2 (W1x)) <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> W3 ij W3 ij @E @W3 ij <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="UYf6vVaAUMM2bux1oNQvUSZOrcQ=">AAACjXicfZDfShwxFMazY612rXXVy96ELkIpssz4/0JE0aI3UoWuKzjrciZ7Zo0mmSHJiMMwz9On6a3i2zSzrlTX0gOBH993kpPzRangxvr+Y82beDf5fmr6Q33m4+ynucb8wplJMs2wzRKR6PMIDAqusG25FXieagQZCexEN/uV37lFbXiifto8xa6EgeIxZ2Cd1GvshbEGVoQpaMtB0O/lX87LMTN/YXZ6Bb8uL1fLXqPpt/xh0bcQjKBJRnXSm6+thf2EZRKVZQKMuQj81HaL6mEmsKyHmcEU2A0M8MKhAommWwx3LemSU/o0TrQ7ytKh+vJGAdKYXEauU4K9MuNeJf7Ti+TYZBtvdQuu0syiYk+D40xQm9AqSNrnGpkVuQNgmru/U3YFLi/r4q7XwwN0y2k8doN+pKjBJvpbEYIeSK5Kt+wgXK7of41w99zoyOUcjKf6Fs5WWsFGa+N0rbm7P0p8mnwmX8hXEpBNskuOyAlpE0Z+kd/knjx4c966t+3tPLV6tdGdRfKqvMM/f/3LbA==</latexit> @E @y @y @W3 ij
  9. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y =

    W3 (W2 (W1x)) <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> W3 ij W3 ij @E @W3 ij <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="SdUMqO14fIfUv258zoU2mi9fPy0=">AAACuXicfVHRShtBFJ1dW5vG1ib2sS9DgyBFwq4VK/giWKEvpRaMEdwY7k7uJmNmZpeZ2dJl2a/0yU/xzdmYtnYVLwwczjmXM3MmzgQ3NghuPH/lxcvVV63X7bU3b9ffdbobZybNNcMBS0Wqz2MwKLjCgeVW4HmmEWQscBjPj2p9+Au14ak6tUWGIwlTxRPOwDpq3JGRyeW4nG+LKko0sDLKQFsOgh5X/3DRFIsH4tCti+ryc9Pzly8d4lfV5U417vSCfrAY+hiES9AjyzkZd73daJKyXKKyTIAxF2GQ2VFZRzCBVTvKDWbA5jDFCwcVSDSjctFLRTcdM6FJqt1Rli7YhxslSGMKGTunBDszTa0mn9Ri2Ui2yf6o5CrLLSp2H5zkgtqU1qXTCdfIrCgcAKa5uztlM3BtWfc17Xb0Fd3jNH53QT8y1GBT/amMQE8lV5V77DTartFzRvj9x+iQ6zlstvoYnO30w73+3s/d3uHRsvEW+UA+ki0Ski/kkHwjJ2RAGLkmt57vrfgHPvgz/+re6nvLnffkv/HNHf6u3K0=</latexit> X k,l @E @y @y @W3 kl @W3 kl W2 ij
  10. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="KtqdR/cVi/jjVAibxoQbBh/fN1A=">AAACZHicfZDdSgMxEIXT9b/+VYsgCLJYBBUpuyrqjSDohTeigrWCW8tsOm2DSXZJstKy1KfxVt/HF/A5TOsKWsWBkI8zJ0zmhDFn2njeW84ZGR0bn5icyk/PzM7NFxYWb3SUKIoVGvFI3YagkTOJFcMMx9tYIYiQYzV8OOn3q4+oNIvktenGWBPQkqzJKBgr1QvL3aPq/W4Qt9lG9X4nu/3O5ma9UPLK3qDc3+BnUCJZXdYXcntBI6KJQGkoB63vfC82tRSUYZRjLx8kGmOgD9DCO4sSBOpaOlih565bpeE2I2WPNO5A/f4iBaF1V4TWKcC09XCvL/7ZC8XQZNM8rKVMxolBST8HNxPumsjt5+M2mEJqeNcCUMXs313aBgXU2BTz+eAU7XIKz+2gixgVmEhtpQGolmCyZ5dtBdt9+s8InS+jJZuzP5zqb7jZKfv75f2rvdLxSZb4JFkha2SD+OSAHJMzckkqhJIn8kxeyGvu3Zlxis7Sp9XJZW+K5Ec5qx+BzLjT</latexit> y =

    W3 (W2 (W1x)) <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> W3 ij W3 ij @E @W3 ij <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> W1 ij W1 ij @E @W1 ij
  11. <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> W3 ij W3 ij @E @W3 ij <latexit

    sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> W1 ij W1 ij @E @W1 ij <latexit sha1_base64="16tW0XfXb2RwDG2mlIPN3uCIsSE=">AAACS3icfZDbSgMxEIaz1Xqop6qX3iwWQUTKrkj1UlDBG7EFe4BuKbPptAaT7JJkxbL0CbzVx/IBfA7vxAvTg6CtOBD4+OcPM/OHMWfaeN6bk5mbzy4sLi3nVlbX1jfym1s1HSWKYpVGPFKNEDRyJrFqmOHYiBWCCDnWw/vzYb/+gEqzSN6afowtAT3JuoyCsVLlsp0veEVvVO4s+BMokEmV25vOcdCJaCJQGspB66bvxaaVgjKMchzkgkRjDPQeeti0KEGgbqWjTQfunlU6bjdS9knjjtSfP1IQWvdFaJ0CzJ2e7g3FP3uhmJpsuqetlMk4MSjpeHA34a6J3GEMbocppIb3LQBVzO7u0jtQQI0NK5cLLtAep/DaDrqJUYGJ1EEagOoJJgf22F5wOKT/jPD4bbRkc/anU52F2lHRLxVLlePC2fkk8SWyQ3bJPvHJCTkjV6RMqoQSJE/kmbw4r8678+F8jq0ZZ/Jnm/yqTPYLcpGzPw==</latexit> E
  12. <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> W3 ij W3 ij @E @W3 ij <latexit

    sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> W1 ij W1 ij @E @W1 ij <latexit sha1_base64="16tW0XfXb2RwDG2mlIPN3uCIsSE=">AAACS3icfZDbSgMxEIaz1Xqop6qX3iwWQUTKrkj1UlDBG7EFe4BuKbPptAaT7JJkxbL0CbzVx/IBfA7vxAvTg6CtOBD4+OcPM/OHMWfaeN6bk5mbzy4sLi3nVlbX1jfym1s1HSWKYpVGPFKNEDRyJrFqmOHYiBWCCDnWw/vzYb/+gEqzSN6afowtAT3JuoyCsVLlsp0veEVvVO4s+BMokEmV25vOcdCJaCJQGspB66bvxaaVgjKMchzkgkRjDPQeeti0KEGgbqWjTQfunlU6bjdS9knjjtSfP1IQWvdFaJ0CzJ2e7g3FP3uhmJpsuqetlMk4MSjpeHA34a6J3GEMbocppIb3LQBVzO7u0jtQQI0NK5cLLtAep/DaDrqJUYGJ1EEagOoJJgf22F5wOKT/jPD4bbRkc/anU52F2lHRLxVLlePC2fkk8SWyQ3bJPvHJCTkjV6RMqoQSJE/kmbw4r8678+F8jq0ZZ/Jnm/yqTPYLcpGzPw==</latexit> E <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W)
  13. (dynamical system) む [...] from Wikipedia む : : む

    : : <latexit sha1_base64="9vKcCGgdeonqGO4ipEfyo1Riiv4=">AAACXHicfZBdSxtBFIYnWz/jV6zQG2+GBkFbCbsitjcFwV54Iyo0KrhhOTs5mwzOzC4zZyVhyT/prf1Pvelv6SRG0CgeGHh4zzucc960UNJRGP6tBR/m5hcWl5brK6tr6xuNzY9XLi+twLbIVW5vUnCopME2SVJ4U1gEnSq8Tu9Oxv3re7RO5uYXDQvsaOgZmUkB5KWk0RgkFX2NRvwHz3YHCe0ljWbYCifFX0M0hSab1kWyWTuMu7koNRoSCpy7jcKCOhVYkkLhqB6XDgsQd9DDW48GNLpONVl9xHe80uVZbv0zxCfq8x8VaOeGOvVODdR3s72x+GYv1TOTKfveqaQpSkIjHgdnpeKU83EuvCstClJDDyCs9Ltz0QcLgnx69Xr8E/1xFs/8oPMCLVBuv1Qx2J6WZuSP7cX7Y3rPCIMnoyefczSb6mu4OmhFR62jy8Pm8ck08SW2zT6zXRaxb+yYnbIL1maC3bPf7IH9qf0L5oKVYO3RGtSmf7bYiwo+/QcgALXi</latexit> xt+1 = f(xt) <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit> Wk+1 = Wk ⌘rE(Wk) <latexit sha1_base64="WmRsZRfB+qtGPCbZ3qqAgLQ5vyo=">AAACZXicfZDRShtBFIYn21ZtWjW2pVB60aGhEEXCbhH1RhDsRW+KFhoV3BDOzp6NgzOzy8zZYliWPo237fP4BL5GZ2MKNooHBj7+8x/OnD8plHQUhtet4MnTZwuLS8/bL14ur6x21l4du7y0AgciV7k9TcChkgYHJEnhaWERdKLwJLk4aPonP9E6mZsfNClwqGFsZCYFkJdGnXc8ziyIKr3s0XpdpVTvZb2G10edbtgPp8XvQzSDLpvV0WittRWnuSg1GhIKnDuLwoKGFViSQmHdjkuHBYgLGOOZRwMa3bCa3lDzT15JeZZb/wzxqXp3ogLt3EQn3qmBzt18rxEf7CV6bjNlu8NKmqIkNOJ2cVYqTjlvAuKptChITTyAsNL/nYtz8BmRj7Hdjr+gP87iN7/osEALlNuNKgY71tLU/thxvNnQY0a4/Gf05HOO5lO9D8ef+9F2f/v7Vnf/YJb4EnvPPrIei9gO22df2REbMMF+sSv2m/1p3QTLwZvg7a01aM1mXrP/KvjwF4VjudA=</latexit> dx(t) dt = f(x(t)) <latexit sha1_base64="XnF9xm1lo4DodOip3sIdDFqiak8=">AAACWHicfZDfSxtBEMcnV6sxbf0RH/tyGAqllHAnoj4K9sGXohVjAl4qc5tJXNzdO3bnxHDk//BV/6v617gXI2iUDix8+M53mZlvmivpOIr+1YIPCx8Xl+rLjU+fv6ysrq03z1xWWEEdkanM9lJ0pKShDktW1MstoU4VddOrg6rfvSbrZGZOeZxTX+PIyKEUyF76y4k0iUa+TNPyZHKx1ora0bTCtxDPoAWzOr5Yr20ng0wUmgwLhc6dx1HO/RItS6Fo0kgKRzmKKxzRuUeDmly/nK49Cb95ZRAOM+uf4XCqvvxRonZurFPvrFZ0871KfLeX6rnJPNzrl9LkBZMRT4OHhQo5C6tMwoG0JFiNPaCw0u8eiku0KNgn12gkv8gfZ+m3H3SUk0XO7I8yQTvS0kz8saPkZ0X/M+LNs9GTzzmeT/UtnG214532zp/t1v7BLPE6fIVN+A4x7MI+HMIxdECAhVu4g/vaQwDBUrD8ZA1qsz8b8KqC5iN4ZrY6</latexit> t 2 R <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W)
  14. む む <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W) <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit>

    Wk+1 = Wk ⌘rE(Wk) <latexit sha1_base64="dBuiNwrvo/kgc9MCDMIOd0lY2Ec=">AAACXHicfZDfShtBFMZnV2ttrJooeNOboaGgUsJuEdubgpBeeFOq0BjBDcvZyUkcMjO7zJwNDUvexFt9p970WTr5I2gsPTDw4zvfcM75skJJR1H0OwjX1l9tvN58U9t6u72zW2/sXbm8tAI7Ile5vc7AoZIGOyRJ4XVhEXSmsJuN2rN+d4zWydz8pEmBPQ1DIwdSAHkprde7h6MECY74V95Nq9E0rTejVjQv/hLiJTTZsi7SRnCS9HNRajQkFDh3E0cF9SqwJIXCaS0pHRYgRjDEG48GNLpeNV99yj94pc8HufXPEJ+rT39UoJ2b6Mw7NdCtW+3NxH/2Mr0ymQZfepU0RUloxGLwoFSccj7LhfelRUFq4gGElX53Lm7BgiCfXq2WfEN/nMXvftCPAi1Qbo+rBOxQSzP1xw6TjzP6nxF+PRo9+Zzj1VRfwtWnVnzaOr08aZ61l4lvsnfsPTtkMfvMztg5u2AdJtiY3bF79hD8CdfDrXB7YQ2D5Z999qzCg79bc7YD</latexit> W(k⌘) = Wk
  15. む む <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W) <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit>

    Wk+1 = Wk ⌘rE(Wk) <latexit sha1_base64="dBuiNwrvo/kgc9MCDMIOd0lY2Ec=">AAACXHicfZDfShtBFMZnV2ttrJooeNOboaGgUsJuEdubgpBeeFOq0BjBDcvZyUkcMjO7zJwNDUvexFt9p970WTr5I2gsPTDw4zvfcM75skJJR1H0OwjX1l9tvN58U9t6u72zW2/sXbm8tAI7Ile5vc7AoZIGOyRJ4XVhEXSmsJuN2rN+d4zWydz8pEmBPQ1DIwdSAHkprde7h6MECY74V95Nq9E0rTejVjQv/hLiJTTZsi7SRnCS9HNRajQkFDh3E0cF9SqwJIXCaS0pHRYgRjDEG48GNLpeNV99yj94pc8HufXPEJ+rT39UoJ2b6Mw7NdCtW+3NxH/2Mr0ymQZfepU0RUloxGLwoFSccj7LhfelRUFq4gGElX53Lm7BgiCfXq2WfEN/nMXvftCPAi1Qbo+rBOxQSzP1xw6TjzP6nxF+PRo9+Zzj1VRfwtWnVnzaOr08aZ61l4lvsnfsPTtkMfvMztg5u2AdJtiY3bF79hD8CdfDrXB7YQ2D5Z999qzCg79bc7YD</latexit> W(k⌘) = Wk <latexit sha1_base64="IfFSNFS4MAtiGLpHp//LggCLChM=">AAACeXicfZDfahNBFMYn22rb+KepXvZmNAhJNWG3lupNoVAFb4oVTFPohnB2cjYdOjO7zJwthmWfok/jrX2KPos3zqYRNBUPDPz4znc4c74kV9JRGN42gpXVBw/X1jeajx4/ebrZ2np26rLCChyITGX2LAGHShockCSFZ7lF0InCYXJ5VPeHV2idzMxXmuU40jA1MpUCyEvjVi9OLYhy2KHXMRJ0e566VVlzxQ94LzaQKOAfO7XeHbfaYT+cF78P0QLabFEn463GXjzJRKHRkFDg3HkU5jQqwZIUCqtmXDjMQVzCFM89GtDoRuX8roq/8sqEp5n1zxCfq39OlKCdm+nEOzXQhVvu1eI/e4le2kzp+1EpTV4QGnG3OC0Up4zXofGJtChIzTyAsNL/nYsL8MGRj7bZjD+gP87isV/0OUcLlNmdMgY71dJU/thp/Kam/xnh22+jJ59ztJzqfTjd7Uf7/f0ve+3Do0Xi62ybvWQdFrF37JB9YidswAS7Zt/ZD3bT+Bm8CDrBzp01aCxmnrO/Knj7C1wTwGA=</latexit> W(t + ⌘) W(t) ⌘ = rE(W(t))
  16. む む <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W) <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit>

    Wk+1 = Wk ⌘rE(Wk) <latexit sha1_base64="dBuiNwrvo/kgc9MCDMIOd0lY2Ec=">AAACXHicfZDfShtBFMZnV2ttrJooeNOboaGgUsJuEdubgpBeeFOq0BjBDcvZyUkcMjO7zJwNDUvexFt9p970WTr5I2gsPTDw4zvfcM75skJJR1H0OwjX1l9tvN58U9t6u72zW2/sXbm8tAI7Ile5vc7AoZIGOyRJ4XVhEXSmsJuN2rN+d4zWydz8pEmBPQ1DIwdSAHkprde7h6MECY74V95Nq9E0rTejVjQv/hLiJTTZsi7SRnCS9HNRajQkFDh3E0cF9SqwJIXCaS0pHRYgRjDEG48GNLpeNV99yj94pc8HufXPEJ+rT39UoJ2b6Mw7NdCtW+3NxH/2Mr0ymQZfepU0RUloxGLwoFSccj7LhfelRUFq4gGElX53Lm7BgiCfXq2WfEN/nMXvftCPAi1Qbo+rBOxQSzP1xw6TjzP6nxF+PRo9+Zzj1VRfwtWnVnzaOr08aZ61l4lvsnfsPTtkMfvMztg5u2AdJtiY3bF79hD8CdfDrXB7YQ2D5Z999qzCg79bc7YD</latexit> W(k⌘) = Wk <latexit sha1_base64="IfFSNFS4MAtiGLpHp//LggCLChM=">AAACeXicfZDfahNBFMYn22rb+KepXvZmNAhJNWG3lupNoVAFb4oVTFPohnB2cjYdOjO7zJwthmWfok/jrX2KPos3zqYRNBUPDPz4znc4c74kV9JRGN42gpXVBw/X1jeajx4/ebrZ2np26rLCChyITGX2LAGHShockCSFZ7lF0InCYXJ5VPeHV2idzMxXmuU40jA1MpUCyEvjVi9OLYhy2KHXMRJ0e566VVlzxQ94LzaQKOAfO7XeHbfaYT+cF78P0QLabFEn463GXjzJRKHRkFDg3HkU5jQqwZIUCqtmXDjMQVzCFM89GtDoRuX8roq/8sqEp5n1zxCfq39OlKCdm+nEOzXQhVvu1eI/e4le2kzp+1EpTV4QGnG3OC0Up4zXofGJtChIzTyAsNL/nYsL8MGRj7bZjD+gP87isV/0OUcLlNmdMgY71dJU/thp/Kam/xnh22+jJ59ztJzqfTjd7Uf7/f0ve+3Do0Xi62ybvWQdFrF37JB9YidswAS7Zt/ZD3bT+Bm8CDrBzp01aCxmnrO/Knj7C1wTwGA=</latexit> W(t + ⌘) W(t) ⌘ = rE(W(t)) <latexit sha1_base64="JAD/0sEwNQXS374OIM3uu9xGAOs=">AAACaHicfZDPShxBEMZ7xySazb/VHIIkh8ElYEJYZkTUo2AOuYgGsq7gLEtNT83a2P/srhGXYS55mlzj2/gKeYr0riska0hBw4+vvqK6vtxK4SlJblvR0qPHT5ZXnrafPX/x8lVnde3Em8px7HMjjTvNwaMUGvskSOKpdQgqlzjILw6m/cEVOi+M/kYTi0MFYy1KwYGCNOq8y0oHvC4Gm/ShqQtqsssKigysdeZ61OkmvWRW8UNI59Bl8zoerba2s8LwSqEmLsH7szSxNKzBkeASm3ZWebTAL2CMZwE1KPTDenZGE78PShGXxoWnKZ6pf07UoLyfqDw4FdC5X+xNxX/2crWwmcq9YS20rQg1v1tcVjImE08zigvhkJOcBADuRPh7zM8h5EQhyXY7+4zhOIeHYdGRRQdk3Mc6AzdWQjfh2HH2aUr/M8L1vTFQyDldTPUhnGz10p3eztft7v7BPPEV9pZtsE2Wsl22z76wY9ZnnH1nP9hPdtP6FXWiN9H6nTVqzWdes78q2vgN80K8eQ==</latexit> dW(t) dt ⇡
  17. " Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact

    solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
  18. <latexit sha1_base64="DyBuCViiQvhrSv2O60LuvI3VqVc=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwV3CKz6bSNTbJLkhXL0v/gVX+WN/+JN01rBa3iQMjDO2+YyZtkghsbhq9eaWx8YnJqesafnZtfWFxaXqmZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaCrpHPU71/dkzY8VZe2m1FdYkvxJmdonVSLUWRtvF0qh5VwUMFviIZQhmGd3y57u3EjZbkkZZlAY26iMLP1ArXlTFDPj3NDGbIOtujGoUJJpl4M1u0FG05pBM1Uu6NsMFC/vyhQGtOViXNKtG0z2uuLf/YSOTLZNg/qBVdZbkmxz8HNXAQ2DfpZBA2uiVnRdYBMc7d7wNqokVmXmO/Hx+Q+p+nUDTrLSKNN9VYRo25Jrnrus614u0//GfHhy+jI5RyNpvobajuVaK+yd7FbPjwaJj4Na7AOmxDBPhzCCZxDFRjcwSM8wbP34r157yXv0/p1wyr8qJL/ARd8tI8=</latexit> ↵ <latexit sha1_base64="lsP0ep27qHQJyJ3JRPqhImSZomw=">AAACUnicfVJNSwMxEM3W7/qtRy/BIohI2S1S21uhHryIClYLbimz6bQNJtklyYpl6Z/wqj/Li3/Fk9laQUUdCDzee8PMPBIlghvr+69eYWZ2bn5hcam4vLK6tr6xuXVt4lQzbLFYxLodgUHBFbYstwLbiUaQkcCb6K6Z6zf3qA2P1ZUdJdiRMFC8zxlYR7WjbggiGUJ3o+SXfVfVKs1BUPMDB+r1WqVSp8FE8v0SmdZFd9M7CnsxSyUqywQYcxv4ie1koC1nAsfFMDWYALuDAd46qECi6WSThcd0zzE92o+1e8rSCfu1IwNpzEhGzinBDs1PLSd/1SL5Y7Lt1zoZV0lqUbGPwf1UUBvTPA3a4xqZFSMHgGnudqdsCBqYdZkVi+EJuuM0nrlB5wlqsLE+yELQA8nV2B07CA9z9J8RHj6NDrmcP8Okf4PrSjmolquXR6VGc5r4Itkhu2SfBOSYNMgpuSAtwoggj+SJPHsv3lvB/ZIPa8Gb9myTb1VYeQciTbSi</latexit> b↵ " Saxe, A., McClelland,

    J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
  19. <latexit sha1_base64="Q0WjhZYoqJ4/iyOUeH3fpzXK1ZA=">AAACUnicfVLLahsxFNW4zaPOy26X3YiaQCnBaPJwnF0gXWQT4kKdGDLG3JGvbRFJM0iaEDP4J7JNP6ub/kpX0fgBiUlyQXA491zOvQfFqRTWMfYvKH34uLK6tv6pvLG5tb1TqX6+sklmOLZ5IhPTicGiFBrbTjiJndQgqFjidXx7VvSv79BYkejfbpxiV8FQi4Hg4DzVgV4EMh1Br1JjddY4OmEnlNWPWNicAsYazf0DGnpQVI3Mq9WrBodRP+GZQu24BGtvQpa6bg7GCS5xUo4yiynwWxjijYcaFNpuPl14Qnc906eDxPinHZ2yzydyUNaOVeyVCtzILvcK8tVerJac3aDZzYVOM4eaz4wHmaQuoUUatC8McifHHgA3wu9O+QgMcOczK5ejn+iPM3jhjS5TNOAS8yOPwAyV0BN/7DDaK9B7QrhfCD3yOS/CpG+Dq/162Kg3fh3WTs/mia+Tr+Qb+U5CckxOyTlpkTbhRJIH8kj+BH+D/yX/S2bSUjCf+UJeVGnzCTPrtKs=</latexit> a↵ <latexit sha1_base64="DyBuCViiQvhrSv2O60LuvI3VqVc=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwV3CKz6bSNTbJLkhXL0v/gVX+WN/+JN01rBa3iQMjDO2+YyZtkghsbhq9eaWx8YnJqesafnZtfWFxaXqmZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaCrpHPU71/dkzY8VZe2m1FdYkvxJmdonVSLUWRtvF0qh5VwUMFviIZQhmGd3y57u3EjZbkkZZlAY26iMLP1ArXlTFDPj3NDGbIOtujGoUJJpl4M1u0FG05pBM1Uu6NsMFC/vyhQGtOViXNKtG0z2uuLf/YSOTLZNg/qBVdZbkmxz8HNXAQ2DfpZBA2uiVnRdYBMc7d7wNqokVmXmO/Hx+Q+p+nUDTrLSKNN9VYRo25Jrnrus614u0//GfHhy+jI5RyNpvobajuVaK+yd7FbPjwaJj4Na7AOmxDBPhzCCZxDFRjcwSM8wbP34r157yXv0/p1wyr8qJL/ARd8tI8=</latexit> ↵ <latexit sha1_base64="lsP0ep27qHQJyJ3JRPqhImSZomw=">AAACUnicfVJNSwMxEM3W7/qtRy/BIohI2S1S21uhHryIClYLbimz6bQNJtklyYpl6Z/wqj/Li3/Fk9laQUUdCDzee8PMPBIlghvr+69eYWZ2bn5hcam4vLK6tr6xuXVt4lQzbLFYxLodgUHBFbYstwLbiUaQkcCb6K6Z6zf3qA2P1ZUdJdiRMFC8zxlYR7WjbggiGUJ3o+SXfVfVKs1BUPMDB+r1WqVSp8FE8v0SmdZFd9M7CnsxSyUqywQYcxv4ie1koC1nAsfFMDWYALuDAd46qECi6WSThcd0zzE92o+1e8rSCfu1IwNpzEhGzinBDs1PLSd/1SL5Y7Lt1zoZV0lqUbGPwf1UUBvTPA3a4xqZFSMHgGnudqdsCBqYdZkVi+EJuuM0nrlB5wlqsLE+yELQA8nV2B07CA9z9J8RHj6NDrmcP8Okf4PrSjmolquXR6VGc5r4Itkhu2SfBOSYNMgpuSAtwoggj+SJPHsv3lvB/ZIPa8Gb9myTb1VYeQciTbSi</latexit> b↵ "

    Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
  20. <latexit sha1_base64="Q0WjhZYoqJ4/iyOUeH3fpzXK1ZA=">AAACUnicfVLLahsxFNW4zaPOy26X3YiaQCnBaPJwnF0gXWQT4kKdGDLG3JGvbRFJM0iaEDP4J7JNP6ub/kpX0fgBiUlyQXA491zOvQfFqRTWMfYvKH34uLK6tv6pvLG5tb1TqX6+sklmOLZ5IhPTicGiFBrbTjiJndQgqFjidXx7VvSv79BYkejfbpxiV8FQi4Hg4DzVgV4EMh1Br1JjddY4OmEnlNWPWNicAsYazf0DGnpQVI3Mq9WrBodRP+GZQu24BGtvQpa6bg7GCS5xUo4yiynwWxjijYcaFNpuPl14Qnc906eDxPinHZ2yzydyUNaOVeyVCtzILvcK8tVerJac3aDZzYVOM4eaz4wHmaQuoUUatC8McifHHgA3wu9O+QgMcOczK5ejn+iPM3jhjS5TNOAS8yOPwAyV0BN/7DDaK9B7QrhfCD3yOS/CpG+Dq/162Kg3fh3WTs/mia+Tr+Qb+U5CckxOyTlpkTbhRJIH8kj+BH+D/yX/S2bSUjCf+UJeVGnzCTPrtKs=</latexit> a↵ <latexit sha1_base64="DyBuCViiQvhrSv2O60LuvI3VqVc=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwV3CKz6bSNTbJLkhXL0v/gVX+WN/+JN01rBa3iQMjDO2+YyZtkghsbhq9eaWx8YnJqesafnZtfWFxaXqmZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaCrpHPU71/dkzY8VZe2m1FdYkvxJmdonVSLUWRtvF0qh5VwUMFviIZQhmGd3y57u3EjZbkkZZlAY26iMLP1ArXlTFDPj3NDGbIOtujGoUJJpl4M1u0FG05pBM1Uu6NsMFC/vyhQGtOViXNKtG0z2uuLf/YSOTLZNg/qBVdZbkmxz8HNXAQ2DfpZBA2uiVnRdYBMc7d7wNqokVmXmO/Hx+Q+p+nUDTrLSKNN9VYRo25Jrnrus614u0//GfHhy+jI5RyNpvobajuVaK+yd7FbPjwaJj4Na7AOmxDBPhzCCZxDFRjcwSM8wbP34r157yXv0/p1wyr8qJL/ARd8tI8=</latexit> ↵ <latexit sha1_base64="lsP0ep27qHQJyJ3JRPqhImSZomw=">AAACUnicfVJNSwMxEM3W7/qtRy/BIohI2S1S21uhHryIClYLbimz6bQNJtklyYpl6Z/wqj/Li3/Fk9laQUUdCDzee8PMPBIlghvr+69eYWZ2bn5hcam4vLK6tr6xuXVt4lQzbLFYxLodgUHBFbYstwLbiUaQkcCb6K6Z6zf3qA2P1ZUdJdiRMFC8zxlYR7WjbggiGUJ3o+SXfVfVKs1BUPMDB+r1WqVSp8FE8v0SmdZFd9M7CnsxSyUqywQYcxv4ie1koC1nAsfFMDWYALuDAd46qECi6WSThcd0zzE92o+1e8rSCfu1IwNpzEhGzinBDs1PLSd/1SL5Y7Lt1zoZV0lqUbGPwf1UUBvTPA3a4xqZFSMHgGnudqdsCBqYdZkVi+EJuuM0nrlB5wlqsLE+yELQA8nV2B07CA9z9J8RHj6NDrmcP8Okf4PrSjmolquXR6VGc5r4Itkhu2SfBOSYNMgpuSAtwoggj+SJPHsv3lvB/ZIPa8Gb9myTb1VYeQciTbSi</latexit> b↵ "

    Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
  21. <latexit sha1_base64="Q0WjhZYoqJ4/iyOUeH3fpzXK1ZA=">AAACUnicfVLLahsxFNW4zaPOy26X3YiaQCnBaPJwnF0gXWQT4kKdGDLG3JGvbRFJM0iaEDP4J7JNP6ub/kpX0fgBiUlyQXA491zOvQfFqRTWMfYvKH34uLK6tv6pvLG5tb1TqX6+sklmOLZ5IhPTicGiFBrbTjiJndQgqFjidXx7VvSv79BYkejfbpxiV8FQi4Hg4DzVgV4EMh1Br1JjddY4OmEnlNWPWNicAsYazf0DGnpQVI3Mq9WrBodRP+GZQu24BGtvQpa6bg7GCS5xUo4yiynwWxjijYcaFNpuPl14Qnc906eDxPinHZ2yzydyUNaOVeyVCtzILvcK8tVerJac3aDZzYVOM4eaz4wHmaQuoUUatC8McifHHgA3wu9O+QgMcOczK5ejn+iPM3jhjS5TNOAS8yOPwAyV0BN/7DDaK9B7QrhfCD3yOS/CpG+Dq/162Kg3fh3WTs/mia+Tr+Qb+U5CckxOyTlpkTbhRJIH8kj+BH+D/yX/S2bSUjCf+UJeVGnzCTPrtKs=</latexit> a↵ <latexit sha1_base64="DyBuCViiQvhrSv2O60LuvI3VqVc=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwV3CKz6bSNTbJLkhXL0v/gVX+WN/+JN01rBa3iQMjDO2+YyZtkghsbhq9eaWx8YnJqesafnZtfWFxaXqmZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaCrpHPU71/dkzY8VZe2m1FdYkvxJmdonVSLUWRtvF0qh5VwUMFviIZQhmGd3y57u3EjZbkkZZlAY26iMLP1ArXlTFDPj3NDGbIOtujGoUJJpl4M1u0FG05pBM1Uu6NsMFC/vyhQGtOViXNKtG0z2uuLf/YSOTLZNg/qBVdZbkmxz8HNXAQ2DfpZBA2uiVnRdYBMc7d7wNqokVmXmO/Hx+Q+p+nUDTrLSKNN9VYRo25Jrnrus614u0//GfHhy+jI5RyNpvobajuVaK+yd7FbPjwaJj4Na7AOmxDBPhzCCZxDFRjcwSM8wbP34r157yXv0/p1wyr8qJL/ARd8tI8=</latexit> ↵ <latexit sha1_base64="lsP0ep27qHQJyJ3JRPqhImSZomw=">AAACUnicfVJNSwMxEM3W7/qtRy/BIohI2S1S21uhHryIClYLbimz6bQNJtklyYpl6Z/wqj/Li3/Fk9laQUUdCDzee8PMPBIlghvr+69eYWZ2bn5hcam4vLK6tr6xuXVt4lQzbLFYxLodgUHBFbYstwLbiUaQkcCb6K6Z6zf3qA2P1ZUdJdiRMFC8zxlYR7WjbggiGUJ3o+SXfVfVKs1BUPMDB+r1WqVSp8FE8v0SmdZFd9M7CnsxSyUqywQYcxv4ie1koC1nAsfFMDWYALuDAd46qECi6WSThcd0zzE92o+1e8rSCfu1IwNpzEhGzinBDs1PLSd/1SL5Y7Lt1zoZV0lqUbGPwf1UUBvTPA3a4xqZFSMHgGnudqdsCBqYdZkVi+EJuuM0nrlB5wlqsLE+yELQA8nV2B07CA9z9J8RHj6NDrmcP8Okf4PrSjmolquXR6VGc5r4Itkhu2SfBOSYNMgpuSAtwoggj+SJPHsv3lvB/ZIPa8Gb9myTb1VYeQciTbSi</latexit> b↵ "

    Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 #
  22. <latexit sha1_base64="Q0WjhZYoqJ4/iyOUeH3fpzXK1ZA=">AAACUnicfVLLahsxFNW4zaPOy26X3YiaQCnBaPJwnF0gXWQT4kKdGDLG3JGvbRFJM0iaEDP4J7JNP6ub/kpX0fgBiUlyQXA491zOvQfFqRTWMfYvKH34uLK6tv6pvLG5tb1TqX6+sklmOLZ5IhPTicGiFBrbTjiJndQgqFjidXx7VvSv79BYkejfbpxiV8FQi4Hg4DzVgV4EMh1Br1JjddY4OmEnlNWPWNicAsYazf0DGnpQVI3Mq9WrBodRP+GZQu24BGtvQpa6bg7GCS5xUo4yiynwWxjijYcaFNpuPl14Qnc906eDxPinHZ2yzydyUNaOVeyVCtzILvcK8tVerJac3aDZzYVOM4eaz4wHmaQuoUUatC8McifHHgA3wu9O+QgMcOczK5ejn+iPM3jhjS5TNOAS8yOPwAyV0BN/7DDaK9B7QrhfCD3yOS/CpG+Dq/162Kg3fh3WTs/mia+Tr+Qb+U5CckxOyTlpkTbhRJIH8kj+BH+D/yX/S2bSUjCf+UJeVGnzCTPrtKs=</latexit> a↵ <latexit sha1_base64="DyBuCViiQvhrSv2O60LuvI3VqVc=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwV3CKz6bSNTbJLkhXL0v/gVX+WN/+JN01rBa3iQMjDO2+YyZtkghsbhq9eaWx8YnJqesafnZtfWFxaXqmZNNeMqiwVqb5O0JDgiqqWW0HXmSaUiaCrpHPU71/dkzY8VZe2m1FdYkvxJmdonVSLUWRtvF0qh5VwUMFviIZQhmGd3y57u3EjZbkkZZlAY26iMLP1ArXlTFDPj3NDGbIOtujGoUJJpl4M1u0FG05pBM1Uu6NsMFC/vyhQGtOViXNKtG0z2uuLf/YSOTLZNg/qBVdZbkmxz8HNXAQ2DfpZBA2uiVnRdYBMc7d7wNqokVmXmO/Hx+Q+p+nUDTrLSKNN9VYRo25Jrnrus614u0//GfHhy+jI5RyNpvobajuVaK+yd7FbPjwaJj4Na7AOmxDBPhzCCZxDFRjcwSM8wbP34r157yXv0/p1wyr8qJL/ARd8tI8=</latexit> ↵ <latexit sha1_base64="lsP0ep27qHQJyJ3JRPqhImSZomw=">AAACUnicfVJNSwMxEM3W7/qtRy/BIohI2S1S21uhHryIClYLbimz6bQNJtklyYpl6Z/wqj/Li3/Fk9laQUUdCDzee8PMPBIlghvr+69eYWZ2bn5hcam4vLK6tr6xuXVt4lQzbLFYxLodgUHBFbYstwLbiUaQkcCb6K6Z6zf3qA2P1ZUdJdiRMFC8zxlYR7WjbggiGUJ3o+SXfVfVKs1BUPMDB+r1WqVSp8FE8v0SmdZFd9M7CnsxSyUqywQYcxv4ie1koC1nAsfFMDWYALuDAd46qECi6WSThcd0zzE92o+1e8rSCfu1IwNpzEhGzinBDs1PLSd/1SL5Y7Lt1zoZV0lqUbGPwf1UUBvTPA3a4xqZFSMHgGnudqdsCBqYdZkVi+EJuuM0nrlB5wlqsLE+yELQA8nV2B07CA9z9J8RHj6NDrmcP8Okf4PrSjmolquXR6VGc5r4Itkhu2SfBOSYNMgpuSAtwoggj+SJPHsv3lvB/ZIPa8Gb9myTb1VYeQciTbSi</latexit> b↵ "

    Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 # <latexit sha1_base64="9lo7Ig4L9iW1kjp3PrYcMIEiVqI=">AAACaXicfZDdSgMxEIXT9b/+tXojihAsgoiUXREVQRD0whtRwargljKbTttgkl2SrFiWXvk03urT+Ay+hGldQas4EPg4c4bJnCgR3Fjffyt4I6Nj4xOTU8Xpmdm5+VJ54drEqWZYY7GI9W0EBgVXWLPcCrxNNIKMBN5E98f9/s0DasNjdWW7CdYltBVvcQbWSY3SatoIQSQdoAeHFL44yqFRqvhVf1D0NwQ5VEheF41yYSdsxiyVqCwTYMxd4Ce2noG2nAnsFcPUYALsHtp451CBRFPPBnf06LpTmrQVa/eUpQP1+0QG0piujJxTgu2Y4V5f/LMXyaHNtrVfz7hKUouKfS5upYLamPZDok2ukVnRdQBMc/d3yjqggVkXZbEYnqA7TuOZW3SeoAYb680sBN2WXPXcse1wq0//GeHxy+jI5RwMp/obrrerwW5193KncnScJz5JVsga2SAB2SNH5JRckBph5Ik8kxfyWnj3yt6St/xp9Qr5zCL5UV7lA1Teu5g=</latexit> u↵ := a↵b↵
  23. Part : む Part : <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt =

    rE(W) <latexit sha1_base64="K/4tljgxQB6lyB6xmluEmWTRwSc=">AAACa3icfZDRShtBFIYnW201Wo31TgWHBsGqDbtFbG8EQQVvRAuNEdywnJ2cxCEzs8vM2dKw5LJP460+TB+i7+AkptBG6YGBj//8hzPnT3MlHYXhr0rwamb29Zu5+erC4tul5drKuyuXFVZgU2Qqs9cpOFTSYJMkKbzOLYJOFbbS/vGo3/qO1snMfKNBjm0NPSO7UgB5KalttpKyvxsND1tJn3/kMRLEBlIF/HTbSx+SWj1shOPizyGaQJ1N6jJZqezHnUwUGg0JBc7dRGFO7RIsSaFwWI0LhzmIPvTwxqMBja5dji8Z8i2vdHg3s/4Z4mP174kStHMDnXqnBrp1072R+GIv1VObqfulXUqTF4RGPC3uFopTxkcx8Y60KEgNPICw0v+di1uwIMiHWa3GJ+iPs3juF13kaIEyu1PGYHtamqE/thfvjeh/Rvjxx+jJ5xxNp/ocrj41ooPGwdf9+tHxJPE5ts7es20Wsc/siJ2xS9Zkgv1kd+yePVR+B6vBWrDxZA0qk5lV9k8FW48cSLth</latexit> Wk+1 = Wk ⌘rE(Wk)
  24. む む (double descent) " Rocks, J., & Mehta, P.

    (2022). Memorizing without overfitting: Bias, variance, and interpolation in overparameterized models.
  25. む む (double descent) " Rocks, J., & Mehta, P.

    (2022). Memorizing without overfitting: Bias, variance, and interpolation in overparameterized models. ա৒ύϥϝʔλܥͰ όϦΞϯε͕ݮগ
  26. む む (double descent) " Rocks, J., & Mehta, P.

    (2022). Memorizing without overfitting: Bias, variance, and interpolation in overparameterized models. ա৒ύϥϝʔλܥͰ όϦΞϯε͕ݮগ ݹయతͳτϨʔυΦϑ͸
 ͚ͩ͜͜Λݟ͍ͯͨʂ
  27. む : む む <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> hw⇤, yixi i <latexit

    sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> > 0 <latexit sha1_base64="nTN0VlqjikVD87chh4w6hftdmaA=">AAACnXicfZBNSxxBEIZ7x3zomo81Hj3YZAmsQZaZICYXQTCBHNQoZF3B3gw1vTVjY3fP0N2jLsP8rvyWHHJN/oY960qSNVjQ8HTVW1TVmxRSWBeGP1rBwqPHT54uLrWXnz1/8bKz8urE5qXhOOC5zM1pAhal0Dhwwkk8LQyCSiQOk4u9pj68RGNFrr+6SYEjBZkWqeDgfCruHO/3mAJ3nqTVVb2xw1IDPNLMliquxE5Uf9MMpewxCTqTSP9oN+kkFnff69qzmUo24k437IfToPchmkGXzOIoXmltsXHOS4XacQnWnkVh4UYVGCe4xLrNSosF8AvI8MyjBoV2VE1vr+kbnxnTNDf+aUen2b87KlDWTlTilc2ydr7WJP9bS9TcZJd+GFVCF6VDzW8Hp6WkLqeNsXQsDHInJx6AG+F3p/wcvJ3O299us4/ojzN44Ad9KdCAy83bioHJlNC1PzZjmw09JITrO6En73M07+p9OHnXj7b728db3d29meOLZI28Jj0Skfdkl3wmR2RAOPlOfpJf5HewHnwK9oPDW2nQmvWskn8iGN4AGmfRJg==</latexit> L(w) = 1 n n X i=1 `(hw, yixi i) <latexit sha1_base64="HT//l12ddZJZk79nPIE4VHhdBgA=">AAACZHicfZDdSiNBEIU7o66a1TUqgiDIYBASdw0zIuqNIOiFN6KCUcEJoaZTGRv7Z+juEeMQn8ZbfR9fwOewEyNolC1o+Dh1iuo6ccqZsUHwUvBGRsd+jU9MFn9PTf+ZKc3OnRuVaYp1qrjSlzEY5Exi3TLL8TLVCCLmeBHf7Pf6F7eoDVPyzHZSbAhIJGszCtZJzdJihJxX7qu7EVdJJfwb4V1aWb+vVpulclAL+uV/h3AAZTKok+ZsYTNqKZoJlJZyMOYqDFLbyEFbRjl2i1FmMAV6AwleOZQg0DTy/gldf9UpLb+ttHvS+n3180QOwpiOiJ1TgL02w72e+GMvFkObbXunkTOZZhYlfV/czrhvld/Lx28xjdTyjgOgmrm/+/QaNFDrUiwWowN0x2k8couOU9RglV7LI9CJYLLrjk2ifz36nxHuPoyOXM7hcKrf4XyjFm7Vtk43y3v7g8QnyBJZIRUSkm2yRw7JCakTSh7II3kiz4VXb8qb9xberV5hMDNPvpS3/AYhSbih</latexit> `(z) = log(1 + exp( z)) <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  28. む : む む <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> hw⇤, yixi i <latexit

    sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> > 0 <latexit sha1_base64="nTN0VlqjikVD87chh4w6hftdmaA=">AAACnXicfZBNSxxBEIZ7x3zomo81Hj3YZAmsQZaZICYXQTCBHNQoZF3B3gw1vTVjY3fP0N2jLsP8rvyWHHJN/oY960qSNVjQ8HTVW1TVmxRSWBeGP1rBwqPHT54uLrWXnz1/8bKz8urE5qXhOOC5zM1pAhal0Dhwwkk8LQyCSiQOk4u9pj68RGNFrr+6SYEjBZkWqeDgfCruHO/3mAJ3nqTVVb2xw1IDPNLMliquxE5Uf9MMpewxCTqTSP9oN+kkFnff69qzmUo24k437IfToPchmkGXzOIoXmltsXHOS4XacQnWnkVh4UYVGCe4xLrNSosF8AvI8MyjBoV2VE1vr+kbnxnTNDf+aUen2b87KlDWTlTilc2ydr7WJP9bS9TcZJd+GFVCF6VDzW8Hp6WkLqeNsXQsDHInJx6AG+F3p/wcvJ3O299us4/ojzN44Ad9KdCAy83bioHJlNC1PzZjmw09JITrO6En73M07+p9OHnXj7b728db3d29meOLZI28Jj0Skfdkl3wmR2RAOPlOfpJf5HewHnwK9oPDW2nQmvWskn8iGN4AGmfRJg==</latexit> L(w) = 1 n n X i=1 `(hw, yixi i) <latexit sha1_base64="HT//l12ddZJZk79nPIE4VHhdBgA=">AAACZHicfZDdSiNBEIU7o66a1TUqgiDIYBASdw0zIuqNIOiFN6KCUcEJoaZTGRv7Z+juEeMQn8ZbfR9fwOewEyNolC1o+Dh1iuo6ccqZsUHwUvBGRsd+jU9MFn9PTf+ZKc3OnRuVaYp1qrjSlzEY5Exi3TLL8TLVCCLmeBHf7Pf6F7eoDVPyzHZSbAhIJGszCtZJzdJihJxX7qu7EVdJJfwb4V1aWb+vVpulclAL+uV/h3AAZTKok+ZsYTNqKZoJlJZyMOYqDFLbyEFbRjl2i1FmMAV6AwleOZQg0DTy/gldf9UpLb+ttHvS+n3180QOwpiOiJ1TgL02w72e+GMvFkObbXunkTOZZhYlfV/czrhvld/Lx28xjdTyjgOgmrm/+/QaNFDrUiwWowN0x2k8couOU9RglV7LI9CJYLLrjk2ifz36nxHuPoyOXM7hcKrf4XyjFm7Vtk43y3v7g8QnyBJZIRUSkm2yRw7JCakTSh7II3kiz4VXb8qb9xberV5hMDNPvpS3/AYhSbih</latexit> `(z) = log(1 + exp( z)) <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. ( ) : <latexit sha1_base64="YLv0/ozj2evA7J6/U7CrSd67fOY=">AAACh3icfVBdaxNBFJ2sX238SvXRl8Eg1A/irpS0IEKlPvgiVjBtoRPC3cndzdD5WGbuVsOSH+Ov8VUf/TfOplE0FS8Mczj3XM69J6+0CpSmPzrJlavXrt/Y2OzevHX7zt3e1r2j4GovcSSddv4kh4BaWRyRIo0nlUcwucbj/Oyg7R+fow/K2Y80r3BsoLSqUBIoUpPeS2GAZnnBP23TY/6KixlQ85tbcKFdyYk/5SJ3ehrmJn5c+JmL8kmvnw7SZfHLIFuBPlvV4WSrsyOmTtYGLUkNIZxmaUXjBjwpqXHRFXXACuQZlHgaoQWDYdwsr1zwR5GZ8sL5+CzxJfvnRAMmtAtGZbt/WO+15D97uVlzpmJv3Chb1YRWXhgXtebkeBshnyqPkvQ8ApBexd25nIEHSTHoble8wXicx3fR6H2FHsj5J40AXxplF/HYUjxr0f+E8PmXMKKYc7ae6mVw9GKQDQfDDzv9/YNV4hvsAXvItlnGdtk+e8sO2YhJ9oV9Zd/Y92QzeZ4Mk70LadJZzdxnf1Xy+ieotsXq</latexit> w(t) = ˆ w log t + ⇢(t) <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="DOiigtDB2ez37UFZCYyJ9z27RNA=">AAACXnicfVBNSyNBEO3M+hlXjXoRvAyGBV0kzIioR0EPXkQFo4ITQk2nkjTpj6G7RgxD/sped/+SN3+KPTGCxsWCph+vXvGqXppJ4SiKnivBj5nZufmFxerSz+WV1dra+q0zueXY5EYae5+CQyk0NkmQxPvMIqhU4l06OC37d49onTD6hoYZthT0tOgKDuSpdm09SY3suKHyX2L7Zod227V61IjGFX4F8QTU2aSu2muVg6RjeK5QE5fg3EMcZdQqwJLgEkfVJHeYAR9ADx881KDQtYrx8qPwl2c6YddY/zSFY/bjRAHKlft5pQLqu+leSf63l6opZ+oetwqhs5xQ8zfjbi5DMmGZTNgRFjnJoQfArfC7h7wPFjj5/KrV5Az9cRYvvNFlhhbI2N9FAranhB75Y3vJXom+E8LTu9Ajn3M8nepXcLvfiA8bh9cH9ZPTSeILbIttsx0WsyN2ws7ZFWsyzp7YH/aX/au8BHPBcrD6Jg0qk5kN9qmCzVcQTrfV</latexit> ⇢(t)
  29. む : む む <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> hw⇤, yixi i <latexit

    sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> > 0 <latexit sha1_base64="nTN0VlqjikVD87chh4w6hftdmaA=">AAACnXicfZBNSxxBEIZ7x3zomo81Hj3YZAmsQZaZICYXQTCBHNQoZF3B3gw1vTVjY3fP0N2jLsP8rvyWHHJN/oY960qSNVjQ8HTVW1TVmxRSWBeGP1rBwqPHT54uLrWXnz1/8bKz8urE5qXhOOC5zM1pAhal0Dhwwkk8LQyCSiQOk4u9pj68RGNFrr+6SYEjBZkWqeDgfCruHO/3mAJ3nqTVVb2xw1IDPNLMliquxE5Uf9MMpewxCTqTSP9oN+kkFnff69qzmUo24k437IfToPchmkGXzOIoXmltsXHOS4XacQnWnkVh4UYVGCe4xLrNSosF8AvI8MyjBoV2VE1vr+kbnxnTNDf+aUen2b87KlDWTlTilc2ydr7WJP9bS9TcZJd+GFVCF6VDzW8Hp6WkLqeNsXQsDHInJx6AG+F3p/wcvJ3O299us4/ojzN44Ad9KdCAy83bioHJlNC1PzZjmw09JITrO6En73M07+p9OHnXj7b728db3d29meOLZI28Jj0Skfdkl3wmR2RAOPlOfpJf5HewHnwK9oPDW2nQmvWskn8iGN4AGmfRJg==</latexit> L(w) = 1 n n X i=1 `(hw, yixi i) <latexit sha1_base64="HT//l12ddZJZk79nPIE4VHhdBgA=">AAACZHicfZDdSiNBEIU7o66a1TUqgiDIYBASdw0zIuqNIOiFN6KCUcEJoaZTGRv7Z+juEeMQn8ZbfR9fwOewEyNolC1o+Dh1iuo6ccqZsUHwUvBGRsd+jU9MFn9PTf+ZKc3OnRuVaYp1qrjSlzEY5Exi3TLL8TLVCCLmeBHf7Pf6F7eoDVPyzHZSbAhIJGszCtZJzdJihJxX7qu7EVdJJfwb4V1aWb+vVpulclAL+uV/h3AAZTKok+ZsYTNqKZoJlJZyMOYqDFLbyEFbRjl2i1FmMAV6AwleOZQg0DTy/gldf9UpLb+ttHvS+n3180QOwpiOiJ1TgL02w72e+GMvFkObbXunkTOZZhYlfV/czrhvld/Lx28xjdTyjgOgmrm/+/QaNFDrUiwWowN0x2k8couOU9RglV7LI9CJYLLrjk2ifz36nxHuPoyOXM7hcKrf4XyjFm7Vtk43y3v7g8QnyBJZIRUSkm2yRw7JCakTSh7II3kiz4VXb8qb9xberV5hMDNPvpS3/AYhSbih</latexit> `(z) = log(1 + exp( z)) <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1
  30. <latexit sha1_base64="DtGYJwzS8y9N9JprVZsArKmi2KE=">AAACW3icfZBNS8NAEIa38avGj1bFk5dgEUSkJFLUo1APXkQFq4IpYbKd1KW7m7C7EUvoL/GqP8qD/8VNraBVHFh4eOddZuaNM8608f23ijMzOze/UF10l5ZXVmv1tfUbneaKYoemPFV3MWjkTGLHMMPxLlMIIuZ4Gw/aZf/2EZVmqbw2wwy7AvqSJYyCsVJUrw0jFgowD3FSPI0iFtUbftMfl/cbggk0yKQuo7VKK+ylNBcoDeWg9X3gZ6ZbgDKMchy5Ya4xAzqAPt5blCBQd4vx5iNvxyo9L0mVfdJ4Y/X7jwKE1kMRW2e5pJ7uleKfvVhMTTbJcbdgMssNSvo5OMm5Z1KvjMXrMYXU8KEFoIrZ3T36AAqoseG5bniK9jiF53bQRYYKTKr2ihBUXzA5ssf2w/2S/jPC05fRks05mE71N9wcNIPD5uFVq3HSniReJVtkm+ySgByRE3JGLkmHUJKTZ/JCXivvzozjOsufVqcy+bNBfpSz+QEyUrcB</latexit> yixi ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1

    w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  31. <latexit sha1_base64="DtGYJwzS8y9N9JprVZsArKmi2KE=">AAACW3icfZBNS8NAEIa38avGj1bFk5dgEUSkJFLUo1APXkQFq4IpYbKd1KW7m7C7EUvoL/GqP8qD/8VNraBVHFh4eOddZuaNM8608f23ijMzOze/UF10l5ZXVmv1tfUbneaKYoemPFV3MWjkTGLHMMPxLlMIIuZ4Gw/aZf/2EZVmqbw2wwy7AvqSJYyCsVJUrw0jFgowD3FSPI0iFtUbftMfl/cbggk0yKQuo7VKK+ylNBcoDeWg9X3gZ6ZbgDKMchy5Ya4xAzqAPt5blCBQd4vx5iNvxyo9L0mVfdJ4Y/X7jwKE1kMRW2e5pJ7uleKfvVhMTTbJcbdgMssNSvo5OMm5Z1KvjMXrMYXU8KEFoIrZ3T36AAqoseG5bniK9jiF53bQRYYKTKr2ihBUXzA5ssf2w/2S/jPC05fRks05mE71N9wcNIPD5uFVq3HSniReJVtkm+ySgByRE3JGLkmHUJKTZ/JCXivvzozjOsufVqcy+bNBfpSz+QEyUrcB</latexit> yixi ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1

    w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  32. <latexit sha1_base64="DtGYJwzS8y9N9JprVZsArKmi2KE=">AAACW3icfZBNS8NAEIa38avGj1bFk5dgEUSkJFLUo1APXkQFq4IpYbKd1KW7m7C7EUvoL/GqP8qD/8VNraBVHFh4eOddZuaNM8608f23ijMzOze/UF10l5ZXVmv1tfUbneaKYoemPFV3MWjkTGLHMMPxLlMIIuZ4Gw/aZf/2EZVmqbw2wwy7AvqSJYyCsVJUrw0jFgowD3FSPI0iFtUbftMfl/cbggk0yKQuo7VKK+ylNBcoDeWg9X3gZ6ZbgDKMchy5Ya4xAzqAPt5blCBQd4vx5iNvxyo9L0mVfdJ4Y/X7jwKE1kMRW2e5pJ7uleKfvVhMTTbJcbdgMssNSvo5OMm5Z1KvjMXrMYXU8KEFoIrZ3T36AAqoseG5bniK9jiF53bQRYYKTKr2ihBUXzA5ssf2w/2S/jPC05fRks05mE71N9wcNIPD5uFVq3HSniReJVtkm+ySgByRE3JGLkmHUJKTZ/JCXivvzozjOsufVqcy+bNBfpSz+QEyUrcB</latexit> yixi <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w ( ) :

    <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  33. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k =

    ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  34. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k =

    ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  35. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k =

    ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) <latexit sha1_base64="2beUvCkyvZB/RdhsLSZwXSLFBHA=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVg/37ip1oNGMCj6G8Ih1MmwTm9q3k7USVkuUVkmwJjrMMhsuwBtORPYr0S5wQzYHXTx2qECiaZdDJbu0zWndGiSaveUpQP1+48CpDE9GTtnuaQZ7ZXin71Yjky2yX674CrLLSr2OTjJBbUpLROhHa6RWdFzAExztztlt6CBWZdbpRIdojtO47EbdJKhBpvqzSIC3ZVc9d2x3WirpP+M8PhldORyDkdT/Q0X241wt7F7tlNvHgwTnyYrZJWsk5DskSY5IqekRRjJyBN5Ji/eq/fuT/hTn1bfG/5ZJj/Kr34ADoa1dA==</latexit> w(T) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  36. ( ) : <latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k =

    ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) <latexit sha1_base64="2beUvCkyvZB/RdhsLSZwXSLFBHA=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVg/37ip1oNGMCj6G8Ih1MmwTm9q3k7USVkuUVkmwJjrMMhsuwBtORPYr0S5wQzYHXTx2qECiaZdDJbu0zWndGiSaveUpQP1+48CpDE9GTtnuaQZ7ZXin71Yjky2yX674CrLLSr2OTjJBbUpLROhHa6RWdFzAExztztlt6CBWZdbpRIdojtO47EbdJKhBpvqzSIC3ZVc9d2x3WirpP+M8PhldORyDkdT/Q0X241wt7F7tlNvHgwTnyYrZJWsk5DskSY5IqekRRjJyBN5Ji/eq/fuT/hTn1bfG/5ZJj/Kr34ADoa1dA==</latexit> w(T) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
  37. 2020: 1993: " Chen, T., Kornblith, S., Norouzi, M., &

    Hinton, G. (2020). A simple framework for contrastive learning of visual representations.
  38. 2020: 1993: " Chen, T., Kornblith, S., Norouzi, M., &

    Hinton, G. (2020). A simple framework for contrastive learning of visual representations. σʔλ֦ு (data augmentation)
  39. CLIP (Contrastive Language-Image Pretraining) " Radford et al. (2021). Learning

    transferable visual models from natural language supervision.
  40. (contrastive learning) SimCLR [Chen+ ] <latexit sha1_base64="OYIuD3tQeI2w0dbax8wVkezfngY=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7Xou3Fspepeb5tZ1dOoC97SFUa9SveP0qk2GdtRadtaCdsCxGaZgArRu+l5pmDspwJrDnBpnGFNgtdLBhUUKMupn3h+7RNau0aZQoe6ShffXrjRxirbtxaJ3FkPpnrxB/6zUyE+01cy7TzKBkg4+iTFCT0CIB2uYKmRFdC8AUt7NSdgMKmLE5uW5whHYZhSf24dMUFZhEbeQBqE7MZc8u1wk2C/rPCPefRkuuDfYzPfo3XG5V/Gqler5d3j8YRjxJVsgqWSc+2SX75JickTphJCUP5JE8Oc/Oq/PmvA+sJWd4Z5l8q5LzAQY1sjc=</latexit> xi <latexit sha1_base64="oBHF1h2GFlEaPhB/HhOweL1oh7I=">AAACR3icfZDfShtBFMZnY/3TrdWol70ZGoQiJexKTMxd0F54U4zQxEA2hLOTs8mY2dllZlYMS16jt+3r9BH6FN6VXnY2iVBFPTDw4zvfzJzzhang2njeb6e09mZ9Y3Prrftu+/3Obnlvv6uTTDHssEQkqheCRsEldgw3AnupQohDgdfh9LzoX9+i0jyR38wsxUEMY8kjzsBYKQhiMJMwyu/mw5thueJVm57fPGnQJZzWVlBvUr/qLapCVtUe7jmHwShhWYzSMAFa930vNYMclOFM4NwNMo0psCmMsW9RQox6kC+GntNDq4xolCh7pKEL9f8bOcRaz+LQOosh9dNeIT7X62cmOh3kXKaZQcmWH0WZoCahRQJ0xBUyI2YWgCluZ6VsAgqYsTm5bvAF7TIKv9qHL1NUYBJ1lAegxjGXc7vcOPhc0GtGuHswWnJtsA/p0Zehe1z169X6Va3SOltFvEU+kI/kE/FJg7TIBWmTDmEkJd/JD/LT+eXcO3+cv0tryVndOSCPquT8AwgNsjg=</latexit>

    xj <latexit sha1_base64="gwzGj/YlW7S86DEYY7wqqSaKG3g=">AAACR3icfZDNSiNBFIWrM/62P2N06aYwCCISusWJyU7GWbgRFYwK6RBuV24npdXVTVW1GJq8hlvndeYR5inciUurkwgq6oWCj3NPVd17wlRwbTzvv1P6MTU9Mzs37y4sLi3/XCmvXugkUwybLBGJugpBo+ASm4YbgVepQohDgZfhzWHRv7xFpXkiz80gxXYMPckjzsBYKQhiMP0wyvvDznVnpeJVG57f+LVPx1Dfm0CtQf2qN6oKmdRpp+xsBt2EZTFKwwRo3fK91LRzUIYzgUM3yDSmwG6ghy2LEmLU7Xw09JBuWqVLo0TZIw0dqW9v5BBrPYhD6yyG1B97hfhZr5WZqN7OuUwzg5KNP4oyQU1CiwRolytkRgwsAFPczkpZHxQwY3Ny3eAP2mUUHtuHT1JUYBK1nQegejGXQ7tcL9gp6Dsj3L0aLbk22Nf06NdwsVv1a9Xa2V7l4Pck4jmyTjbIFvHJPjkgR+SUNAkjKbknD+Sv8895dJ6c57G15EzurJF3VXJeAOpOsig=</latexit> hj <latexit sha1_base64="M7L9l5V9YJ8k17bUYdQox0J08kU=">AAACR3icfZDPSiNBEMZ7ou5mR3c1etxLYxBEljAjmj+3sHrYixjBqJAJoaZTkzT29AzdPYthyGt4XV/HR/ApvMke7Uki6LJa0PDjq6+7q74wFVwbz3twSkvLK58+l7+4q2tfv61vVDYvdJIphl2WiERdhaBRcIldw43Aq1QhxKHAy/D6qOhf/kaleSLPzSTFfgwjySPOwFgpCGIw4zDKx9MBH2xUvVrL81uHDTqH5sEC6i3q17xZVcmiOoOKsxMME5bFKA0ToHXP91LTz0EZzgRO3SDTmAK7hhH2LEqIUffz2dBTumOVIY0SZY80dKa+vpFDrPUkDq2zGFL/2yvE//V6mYma/ZzLNDMo2fyjKBPUJLRIgA65QmbExAIwxe2slI1BATM2J9cNjtEuo/DEPnyaogKTqL08ADWKuZza5UbBj4I+MsLNi9GSa4N9SY++Dxf7Nb9eq58dVNs/FxGXyXeyTXaJTxqkTX6RDukSRlJyS/6QO+feeXSenL9za8lZ3Nkib6rkPAPodrIn</latexit> hi <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> zi <latexit sha1_base64="g841n+aIva7pMukoNWeQg2B5H4A=">AAACR3icfZDLbhMxFIY9oUCYcklgycZqFKmqUDRThVx2EbBgUzWVmjRSJorOOGcSE49nZHuqhlFegy28Do/AU7CrWOLJRSoVcCRLn/7z2z7nD1PBtfG8H07pwcHDR4/LT9zDp8+ev6hUXw51kimGA5aIRI1C0Ci4xIHhRuAoVQhxKPAqXL4v+lfXqDRP5KVZpTiJYS55xBkYKwVBDGYRRvnn9fTTtFLzGl3P775t0y10mjtodanf8DZVI7vqT6tOPZglLItRGiZA67HvpWaSgzKcCVy7QaYxBbaEOY4tSohRT/LN0Gtat8qMRomyRxq6Ue/eyCHWehWH1lkMqe/3CvFvvXFmos4k5zLNDEq2/SjKBDUJLRKgM66QGbGyAExxOytlC1DAjM3JdYMPaJdReGYfPk9RgUnUSR6Amsdcru1y8+BNQf8zws3eaMm1we7To/+G4WnDbzVaF81a790u4jJ5TY7IMfFJm/TIR9InA8JISr6Qr+Sb89356dw6v7bWkrO784r8USXnNwvDsjo=</latexit> zj <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> xj <latexit sha1_base64="+f2W4Hg/9jlLsUD5f6us5OTh8bY=">AAACVHicfVDBahRBEO2ZGI0TjZvk4CGXxiUQRIYeN272GNSDl2AENwnsLEtNb81um57uobtHsgzzNV7j7wj+iwd7NhswQS1oeLz3qrrqZaUU1jH2MwjXHqw/fLTxONp88nTrWWd758zqynAcci21ucjAohQKh044iRelQSgyiefZ5btWP/+KxgqtPrtFieMCZkrkgoPz1KTzPNVebrvrtAA3z/J63jSTL5NOl8Ws1+/1DimLe2zAluCIscGbhCYxW1aXrOp0sh3sp1PNqwKV4xKsHSWsdOMajBNcYhOllcUS+CXMcOShggLtuF5e0NB9z0xpro1/ytEl+2dHDYW1iyLzznZLe19ryb9po8rlg3EtVFk5VPzmo7yS1GnaxkGnwiB3cuEBcCP8rpTPwQB3PrQoSt+jP8bgiR/8sUQDTpuXdQpmVgjV+ONm6asW/c8IV7dGjyIf7G169N/g7HWc9OP+p8Pu8dtVxBtkj7wgByQhR+SYfCCnZEg4acg3ck2+Bz+CX+FauH5jDYNVzy65U+HWb4OUtVs=</latexit> hj <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> zj <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> xj <latexit sha1_base64="+f2W4Hg/9jlLsUD5f6us5OTh8bY=">AAACVHicfVDBahRBEO2ZGI0TjZvk4CGXxiUQRIYeN272GNSDl2AENwnsLEtNb81um57uobtHsgzzNV7j7wj+iwd7NhswQS1oeLz3qrrqZaUU1jH2MwjXHqw/fLTxONp88nTrWWd758zqynAcci21ucjAohQKh044iRelQSgyiefZ5btWP/+KxgqtPrtFieMCZkrkgoPz1KTzPNVebrvrtAA3z/J63jSTL5NOl8Ws1+/1DimLe2zAluCIscGbhCYxW1aXrOp0sh3sp1PNqwKV4xKsHSWsdOMajBNcYhOllcUS+CXMcOShggLtuF5e0NB9z0xpro1/ytEl+2dHDYW1iyLzznZLe19ryb9po8rlg3EtVFk5VPzmo7yS1GnaxkGnwiB3cuEBcCP8rpTPwQB3PrQoSt+jP8bgiR/8sUQDTpuXdQpmVgjV+ONm6asW/c8IV7dGjyIf7G169N/g7HWc9OP+p8Pu8dtVxBtkj7wgByQhR+SYfCCnZEg4acg3ck2+Bz+CX+FauH5jDYNVzy65U+HWb4OUtVs=</latexit> hj <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> zj <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> xj <latexit sha1_base64="+f2W4Hg/9jlLsUD5f6us5OTh8bY=">AAACVHicfVDBahRBEO2ZGI0TjZvk4CGXxiUQRIYeN272GNSDl2AENwnsLEtNb81um57uobtHsgzzNV7j7wj+iwd7NhswQS1oeLz3qrrqZaUU1jH2MwjXHqw/fLTxONp88nTrWWd758zqynAcci21ucjAohQKh044iRelQSgyiefZ5btWP/+KxgqtPrtFieMCZkrkgoPz1KTzPNVebrvrtAA3z/J63jSTL5NOl8Ws1+/1DimLe2zAluCIscGbhCYxW1aXrOp0sh3sp1PNqwKV4xKsHSWsdOMajBNcYhOllcUS+CXMcOShggLtuF5e0NB9z0xpro1/ytEl+2dHDYW1iyLzznZLe19ryb9po8rlg3EtVFk5VPzmo7yS1GnaxkGnwiB3cuEBcCP8rpTPwQB3PrQoSt+jP8bgiR/8sUQDTpuXdQpmVgjV+ONm6asW/c8IV7dGjyIf7G169N/g7HWc9OP+p8Pu8dtVxBtkj7wgByQhR+SYfCCnZEg4acg3ck2+Bz+CX+FauH5jDYNVzy65U+HWb4OUtVs=</latexit> hj <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> zj <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> xj <latexit sha1_base64="+f2W4Hg/9jlLsUD5f6us5OTh8bY=">AAACVHicfVDBahRBEO2ZGI0TjZvk4CGXxiUQRIYeN272GNSDl2AENwnsLEtNb81um57uobtHsgzzNV7j7wj+iwd7NhswQS1oeLz3qrrqZaUU1jH2MwjXHqw/fLTxONp88nTrWWd758zqynAcci21ucjAohQKh044iRelQSgyiefZ5btWP/+KxgqtPrtFieMCZkrkgoPz1KTzPNVebrvrtAA3z/J63jSTL5NOl8Ws1+/1DimLe2zAluCIscGbhCYxW1aXrOp0sh3sp1PNqwKV4xKsHSWsdOMajBNcYhOllcUS+CXMcOShggLtuF5e0NB9z0xpro1/ytEl+2dHDYW1iyLzznZLe19ryb9po8rlg3EtVFk5VPzmo7yS1GnaxkGnwiB3cuEBcCP8rpTPwQB3PrQoSt+jP8bgiR/8sUQDTpuXdQpmVgjV+ONm6asW/c8IV7dGjyIf7G169N/g7HWc9OP+p8Pu8dtVxBtkj7wgByQhR+SYfCCnZEg4acg3ck2+Bz+CX+FauH5jDYNVzy65U+HWb4OUtVs=</latexit> hj <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> zj <latexit sha1_base64="zT4jofQAeInRAyLCg8ASv/1Qtug=">AAACVHicfVDBahRBEO2ZJBpHjZvk4MFL4xIQkaHHjZs9BvWQiyQBNwnsLEtNb82mk57uobsnZBnma7zG3xH8Fw/p2WwgilrQ8HjvVXXVy0oprGPsZxCurK49erz+JHr67PnGi87m1onVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy0+tfnqFxgqtvrp5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1ddNMLiadLotZr9/r7VIW99iALcAeY4MPCU1itqguWdbRZDPYSaeaVwUqxyVYO0pY6cY1GCe4xCZKK4sl8EuY4chDBQXacb24oKE7npnSXBv/lKML9mFHDYW18yLzznZL+6fWkn/TRpXLB+NaqLJyqPjdR3klqdO0jYNOhUHu5NwD4Eb4XSk/BwPc+dCiKP2M/hiDX/zgwxINOG3e1imYWSFU44+bpe9a9D8jXN8bPYp8sPfp0X+Dk/dx0o/7x7vd/Y/LiNfJK/KavCEJ2SP75IAckSHhpCHfyA35HvwIfoUr4dqdNQyWPdvktwo3bgGhVLVr</latexit> xj <latexit sha1_base64="+f2W4Hg/9jlLsUD5f6us5OTh8bY=">AAACVHicfVDBahRBEO2ZGI0TjZvk4CGXxiUQRIYeN272GNSDl2AENwnsLEtNb81um57uobtHsgzzNV7j7wj+iwd7NhswQS1oeLz3qrrqZaUU1jH2MwjXHqw/fLTxONp88nTrWWd758zqynAcci21ucjAohQKh044iRelQSgyiefZ5btWP/+KxgqtPrtFieMCZkrkgoPz1KTzPNVebrvrtAA3z/J63jSTL5NOl8Ws1+/1DimLe2zAluCIscGbhCYxW1aXrOp0sh3sp1PNqwKV4xKsHSWsdOMajBNcYhOllcUS+CXMcOShggLtuF5e0NB9z0xpro1/ytEl+2dHDYW1iyLzznZLe19ryb9po8rlg3EtVFk5VPzmo7yS1GnaxkGnwiB3cuEBcCP8rpTPwQB3PrQoSt+jP8bgiR/8sUQDTpuXdQpmVgjV+ONm6asW/c8IV7dGjyIf7G169N/g7HWc9OP+p8Pu8dtVxBtkj7wgByQhR+SYfCCnZEg4acg3ck2+Bz+CX+FauH5jDYNVzy65U+HWb4OUtVs=</latexit> hj <latexit sha1_base64="nD1veWt0VjbSwlag1ylwwCGzRQA=">AAACVHicfVDBahRBEO2ZJBpHjRs9eMilcQmIyNDjxs0eg+bgJSQBNwnsLEtNb82mk57uobsnZB3ma7zq7wT8Fw/2bDYQRS1oeLz3qrrqZaUU1jH2IwhXVtcePFx/FD1+8nTjWWfz+YnVleE45Fpqc5aBRSkUDp1wEs9Kg1BkEk+zy4+tfnqFxgqtPrt5ieMCZkrkgoPz1KTzMtVebrvrtAB3nuX1l6aZXEw6XRazXr/X26Es7rEBW4BdxgbvE5rEbFFdsqyjyWawnU41rwpUjkuwdpSw0o1rME5wiU2UVhZL4Jcww5GHCgq043pxQUO3PTOluTb+KUcX7P2OGgpr50Xmne2W9k+tJf+mjSqXD8a1UGXlUPHbj/JKUqdpGwedCoPcybkHwI3wu1J+Dga486FFUbqP/hiDB37wYYkGnDZv6hTMrBCq8cfN0rct+p8Rru+MHkU+2Lv06L/Bybs46cf9453u3odlxOtki7wir0lCdske+USOyJBw0pCv5Bv5HtwEP8OVcO3WGgbLnhfktwo3fgGlDLVt</latexit> zj " Chen, T., Kornblith, S., Norouzi, M., & Hinton, G. (2020). A simple framework for contrastive learning of visual representations. pred head Encoder
  41. (Non-contrastive learning) " Chen, X. & He, K. (2021). Exploring

    simple siamese representation learning. SimSiam [Chen-He ] <latexit sha1_base64="OYIuD3tQeI2w0dbax8wVkezfngY=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7Xou3Fspepeb5tZ1dOoC97SFUa9SveP0qk2GdtRadtaCdsCxGaZgArRu+l5pmDspwJrDnBpnGFNgtdLBhUUKMupn3h+7RNau0aZQoe6ShffXrjRxirbtxaJ3FkPpnrxB/6zUyE+01cy7TzKBkg4+iTFCT0CIB2uYKmRFdC8AUt7NSdgMKmLE5uW5whHYZhSf24dMUFZhEbeQBqE7MZc8u1wk2C/rPCPefRkuuDfYzPfo3XG5V/Gqler5d3j8YRjxJVsgqWSc+2SX75JickTphJCUP5JE8Oc/Oq/PmvA+sJWd4Z5l8q5LzAQY1sjc=</latexit> xi <latexit sha1_base64="oBHF1h2GFlEaPhB/HhOweL1oh7I=">AAACR3icfZDfShtBFMZnY/3TrdWol70ZGoQiJexKTMxd0F54U4zQxEA2hLOTs8mY2dllZlYMS16jt+3r9BH6FN6VXnY2iVBFPTDw4zvfzJzzhang2njeb6e09mZ9Y3Prrftu+/3Obnlvv6uTTDHssEQkqheCRsEldgw3AnupQohDgdfh9LzoX9+i0jyR38wsxUEMY8kjzsBYKQhiMJMwyu/mw5thueJVm57fPGnQJZzWVlBvUr/qLapCVtUe7jmHwShhWYzSMAFa930vNYMclOFM4NwNMo0psCmMsW9RQox6kC+GntNDq4xolCh7pKEL9f8bOcRaz+LQOosh9dNeIT7X62cmOh3kXKaZQcmWH0WZoCahRQJ0xBUyI2YWgCluZ6VsAgqYsTm5bvAF7TIKv9qHL1NUYBJ1lAegxjGXc7vcOPhc0GtGuHswWnJtsA/p0Zehe1z169X6Va3SOltFvEU+kI/kE/FJg7TIBWmTDmEkJd/JD/LT+eXcO3+cv0tryVndOSCPquT8AwgNsjg=</latexit> xj <latexit sha1_base64="gwzGj/YlW7S86DEYY7wqqSaKG3g=">AAACR3icfZDNSiNBFIWrM/62P2N06aYwCCISusWJyU7GWbgRFYwK6RBuV24npdXVTVW1GJq8hlvndeYR5inciUurkwgq6oWCj3NPVd17wlRwbTzvv1P6MTU9Mzs37y4sLi3/XCmvXugkUwybLBGJugpBo+ASm4YbgVepQohDgZfhzWHRv7xFpXkiz80gxXYMPckjzsBYKQhiMP0wyvvDznVnpeJVG57f+LVPx1Dfm0CtQf2qN6oKmdRpp+xsBt2EZTFKwwRo3fK91LRzUIYzgUM3yDSmwG6ghy2LEmLU7Xw09JBuWqVLo0TZIw0dqW9v5BBrPYhD6yyG1B97hfhZr5WZqN7OuUwzg5KNP4oyQU1CiwRolytkRgwsAFPczkpZHxQwY3Ny3eAP2mUUHtuHT1JUYBK1nQegejGXQ7tcL9gp6Dsj3L0aLbk22Nf06NdwsVv1a9Xa2V7l4Pck4jmyTjbIFvHJPjkgR+SUNAkjKbknD+Sv8895dJ6c57G15EzurJF3VXJeAOpOsig=</latexit> hj <latexit sha1_base64="M7L9l5V9YJ8k17bUYdQox0J08kU=">AAACR3icfZDPSiNBEMZ7ou5mR3c1etxLYxBEljAjmj+3sHrYixjBqJAJoaZTkzT29AzdPYthyGt4XV/HR/ApvMke7Uki6LJa0PDjq6+7q74wFVwbz3twSkvLK58+l7+4q2tfv61vVDYvdJIphl2WiERdhaBRcIldw43Aq1QhxKHAy/D6qOhf/kaleSLPzSTFfgwjySPOwFgpCGIw4zDKx9MBH2xUvVrL81uHDTqH5sEC6i3q17xZVcmiOoOKsxMME5bFKA0ToHXP91LTz0EZzgRO3SDTmAK7hhH2LEqIUffz2dBTumOVIY0SZY80dKa+vpFDrPUkDq2zGFL/2yvE//V6mYma/ZzLNDMo2fyjKBPUJLRIgA65QmbExAIwxe2slI1BATM2J9cNjtEuo/DEPnyaogKTqL08ADWKuZza5UbBj4I+MsLNi9GSa4N9SY++Dxf7Nb9eq58dVNs/FxGXyXeyTXaJTxqkTX6RDukSRlJyS/6QO+feeXSenL9za8lZ3Nkib6rkPAPodrIn</latexit> hi <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> zi Encoder pred head Stop Grad む む 1 Q.
  42. SimSiam <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x0 <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x0

    <latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> W x Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> x0 " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  43. SimSiam <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x0 <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x0

    <latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> W x Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> x0 <latexit sha1_base64="qG5xXlYS9cXfNOcbLQsnyKaRsUE=">AAACZnicfVDRatswFFXcre3cLk1bxh72IhYKyQjBLqPbY+n20D1sa2FJCnEI18p1IirJRpJHg/E37Gv2un3H/mCfUTlNYU1GLwiOzjlXuvfEmeDGBsGfmrfx5Onm1vYzf2f3eX2vsX/QN2muGfZYKlJ9FYNBwRX2LLcCrzKNIGOBg/j6Q6UPvqM2PFXf7DzDkYSp4glnYB01brQjCXYWJ8VNOQ4iw+XizkAUX8pW0LlXP5XtcaMZdINF0XUQLkGTLOtivF87iiYpyyUqywQYMwyDzI4K0JYzgaUf5QYzYNcwxaGDCiSaUbHYqaRHjpnQJNXuKEsX7L8dBUhj5jJ2zmpGs6pV5P+0YW6T96OCqyy3qNjdR0kuqE1pFRCdcI3MirkDwDR3s1I2Aw3Muhh9P/qIbhmNn93DXzPUYFP9pohATyVXpVtuGnUq9JgRbu6NDvku2HA1xnXQP+6GJ92Ty7fN07NlxNvkFXlNWiQk78gpOScXpEcY+UF+kl/kd+2vV/deeC/vrF5t2XNIHpRHbwF9ALrR</latexit> x0 ⇠ N(0, I) <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x, x0 ⇠ N(x0, 2I) " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  44. SimSiam <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x0 <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x0

    <latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> W x Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> x0 <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x,x 0 ⇠ N(x0 , 2 I) <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x,x 0 ⇠ N (x 0 , 2 I) <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> L( , W) = 1 2 EkW x StopGrad( x0)k2 2 " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  45. <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> L( , W) = 1 2 EkW x

    StopGrad( x0)k2 2 (wrt ) <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit> ˙ W = rW L ⇢W <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit> ˙ = r L ⇢ " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  46. <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> L( , W) = 1 2 EkW x

    StopGrad( x0)k2 2 (wrt ) <latexit sha1_base64="mlZmNVp+y6PQL0YNrIoQpqvHXrE=">AAACfnicfVBdSxtBFL3Z2tZuv6J97MtgsJRS424p6otgrQ8+WKrQGCEbwt3J3WRwdmaZmS2GZX9Pf01fLf03nY2xWC1eGDicc+7ce09aSGFdFP1uBQ+WHj56vPwkfPrs+YuX7ZXVU6tLw6nHtdTmLEVLUijqOeEknRWGME8l9dPzz43e/07GCq2+uVlBwxwnSmSCo/PUqP0pGWtXJTm6aZqxfs122QZLFKYSRzfoOeIoq6O60c1U/9VG7U7UjebF7oJ4ATqwqOPRSmvdD+VlTspxidYO4qhwwwqNE1xSHSalpQL5OU5o4KHCnOywmt9as3XPjFmmjX/KsTl7s6PC3NpZnnpns6G9rTXk/7RB6bKdYSVUUTpS/GpQVkrmNGuCY2NhiDs58wC5EX5XxqdokDsfbxgmB+SPMfTFf/y1IINOm3dVgmaSC1X74ybJ+wbdZ8SLa6NHoQ82vh3jXXD6oRtvdbdOPnb29hcRL8NrWIO3EMM27MEhHEMPOPyAn3AJvwII3gQbweaVNWgtel7BPxXs/AHlo8KJ</latexit> ˙ W = rW L ⇢W <latexit sha1_base64="DXRLzEssQr+y+lNfZvud3wvPtqc=">AAAChHicfVBdaxNBFJ1sta3rV1offRkMhSI17KqtvlSKFdoHxQhNW8iGcHdyNxk6O7PM3C2GZX+Sv8Ynwf4XZ9MI/aIXBg7nnDv33pMWSjqKoj+tYOnBw+WV1Ufh4ydPnz1vr60fO1NagX1hlLGnKThUUmOfJCk8LSxCnio8Sc/2G/3kHK2TRh/RrMBhDhMtMymAPDVqHyRjQ1WSA03TLOlNZc13+RueaEgVjK4JcyxAVV/rxmGn5oo6aneibjQvfhvEC9Bhi+qN1lobfrQoc9QkFDg3iKOChhVYkkJhHSalwwLEGUxw4KGGHN2wml9c8w3PjHlmrH+a+Jy92lFB7twsT72z2dHd1BryLm1QUvZxWEldlIRaXA7KSsXJ8CY+PpYWBamZByCs9LtyMQULgnzIYZh8QX+MxW/+4+8FWiBjX1cJ2Ekude2PmyRbDbrPCD//Gz0KfbDxzRhvg+O33Xinu/PjfWfv8yLiVfaSvWKbLGYf2B47ZD3WZ4L9Yr/ZX3YRLAdbwbtg+9IatBY9L9i1Cj79AyCJxdc=</latexit> ˙ = r L ⇢ <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ˙ s = 2(1 + 2)p2s + 2ps 2⇢s ˙ p = (1 + 2)ps + s ⇢p (wrt ) " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  47. : <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ˙ s = 2(1 + 2)p2s +

    2ps 2⇢s ˙ p = (1 + 2)ps + s ⇢p ( ) <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> ˙ p = p2{1 (1 + 2)p} ⇢p " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  48. : む 2 : ( ) & ( ) む

    Q. Q. <latexit sha1_base64="85SJb0yUccJuZk+1ih/lgL1e73o=">AAACQ3icfZDLSgMxFIYz3h1vrS7dDBZBRMqMSHVZ1IUbUcFqoVPLmfR0GkwyQ5IRy9CXcKuv40P4DO7ErWBaK3jDA4GP//xJzvmjlDNtfP/JGRufmJyanpl15+YXFpcKxeULnWSKYo0mPFH1CDRyJrFmmOFYTxWCiDheRtcHg/7lDSrNEnlueik2BcSSdRgFY6V6qFks4Gq7VSj5ZX9Y3m8IRlAiozptFZ31sJ3QTKA0lIPWjcBPTTMHZRjl2HfDTGMK9BpibFiUIFA38+HAfW/dKm2vkyh7pPGG6tcbOQiteyKyTgGmq3/2BuJfvUZmOnvNnMk0Myjpx0edjHsm8Qbbe22mkBreswBUMTurR7uggBqbkeuGh2iXUXhsHz5JUYFJ1GYegooFk327XBxuDeg/I9x+Gi25NtjgZ4y/4WK7HFTKlbOdUnV/FPEMWSVrZIMEZJdUyRE5JTVCCSd35J48OI/Os/PivH5Yx5zRnRXyrZy3d/cXsKU=</latexit> 2 <latexit sha1_base64="qka/CrKQLTtgrd32QEdkppKPDyw=">AAACP3icfZDNSgMxFIUz/jv+tbp0M1gEESkzItVlURduRAWrhU6RO+ltG5pJhiQjlqGv4FZfx8fwCdyJW3dmagW14oXAx7knyb0nSjjTxvefnYnJqemZ2bl5d2FxaXmlUFy90jJVFGtUcqnqEWjkTGDNMMOxniiEOOJ4HfWO8v71LSrNpLg0/QSbMXQEazMKJpdC1ZU3hZJf9ofljUMwghIZ1flN0dkMW5KmMQpDOWjdCPzENDNQhlGOAzdMNSZAe9DBhkUBMepmNhx24G1apeW1pbJHGG+ofr+RQax1P46sMwbT1b97ufhXr5Ga9kEzYyJJDQr6+VE75Z6RXr6512IKqeF9C0AVs7N6tAsKqLH5uG54jHYZhaf24bMEFRiptrMQVCdmYmCX64Q7Of1nhLsvoyXXBhv8jnEcrnbLQaVcudgrVQ9HEc+RdbJBtkhA9kmVnJBzUiOUdMk9eSCPzpPz4rw6b5/WCWd0Z438KOf9A97AryU=</latexit> ⇢ <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> p = 0 <latexit sha1_base64="F1h6NWW9Z6Rt/PfDMQo5SrSIQsM=">AAACsXicfZHbbhMxEIa9y6ldTilc9sYiKioUot0FFW4qVcAFN4gikbYoTsOsd7KxYnst24uIVnnQ9mlwDiDaIkay9Gvms2fmd2GkcD5Nz6P4xs1bt+9sbCZ3791/8LCz9ejY1Y3l2Oe1rO1pAQ6l0Nj3wks8NRZBFRJPiun7Rf3kB1onav3VzwwOFVRajAUHH1KjjmcFVkK3IEWlsZwnrKx96+b06QF9me9me8yJSsFZ/syc5Y7u0dy4UGB2UlNHGVvxZsVfwgMcSLokTcJQl3+ajDrdtJcug14X2Vp0yTqORlvRTujDG4XacwnODbLU+GEL1gsuMUzdODTAp1DhIEgNCt2wXdozpzshU9JxbcPRni6zf99oQTk3U0UgFfiJu1pbJP9VGzR+/HbYCm0aj5qvGo0bSX1NF17TUljkXs6CAG5FmJXyCVjgPvxIkrAPGJax+Ck8/NmgBV/b5y0DWymh52G5ir1YqP+B8PM3GFQSjM2u2nhdHOe9bL+3/+V19/Dd2uINsk2ekF2SkTfkkHwkR6RPOLmISLQZJfGr+Fv8PS5WaByt7zwmlyKe/gLQJM/H</latexit> ˙ s = 2(1 + 2)p2s + 2ps 2⇢s ˙ p = (1 + 2)ps + s ⇢p ( ) <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> ˙ p = p2{1 (1 + 2)p} ⇢p " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  49. む む ( ) <latexit sha1_base64="3GPTXDUNJzSzuzbIHbnhZyOHpNY=">AAACRnicfZBNaxRBEIZr1q84fiV69NK4BKLIMhMk5iIE9ZCLmICbBHaWUNNbs2nS09101wSXYX+G1+Tv+Bf8E96CV3s3G8iHWNDw8NZb3dVv6bQKnGW/ks6du/fuP1h6mD56/OTps+WV53vBNl5SX1pt/UGJgbQy1GfFmg6cJ6xLTfvl8adZf/+EfFDWfOOJo2GNY6MqJZGjNChGloX7UK2514fL3ayXzUvchnwBXVjUzuFKshrHZVOTYakxhEGeOR626FlJTdO0aAI5lMc4pkFEgzWFYTvfeSpWozISlfXxGBZz9epEi3UIk7qMzhr5KNzszcR/9QYNV5vDVhnXMBl58VDVaMFWzAIQI+VJsp5EQOlV3FXII/QoOcaUpsVnip/x9CVe/NWRR7b+TVugH9fKTOPnxsXbGf3PiN8vjZHSGGx+M8bbsLfeyzd6G7vvulsfFxEvwUt4BWuQw3vYgm3YgT5IsPADTuEs+Zn8Ts6TPxfWTrKYeQHXqgN/AYpPsF4=</latexit> ˙ p = f(p)

    <latexit sha1_base64="eNGawypYh8jUPIrz6KQjBDmydQc=">AAACQ3icfZDLSgMxFIYz3h3vunQTLIKIlBkRdSOIunAjKlgtdIqcSU9rMJMZkjNiGfoSbvV1fAifwZ24FUxrBW94IPDxn/8kOX+cKWkpCJ68gcGh4ZHRsXF/YnJqemZ2bv7cprkRWBGpSk01BotKaqyQJIXVzCAkscKL+Hq/27+4QWNlqs+onWE9gZaWTSmAnFSNGinxbCe4nC0F5aBX/DeEfSixfp1cznnLblbkCWoSCqythUFG9QIMSaGw40e5xQzENbSw5lBDgrZe9D7c4ctOafBmatzRxHvq14kCEmvbSeycCdCV/dnrin/1ajk1t+uF1FlOqMXHQ81ccUp5d3vekAYFqbYDEEa6v3JxBQYEuYx8PzpAt4zBI3fxcYYGKDWrRQSmlUjdccu1orUu/WeE20+jI98FG/6M8Tecr5fDzfLm6UZpd68f8RhbZEtshYVsi+2yQ3bCKkwwxe7YPXvwHr1n78V7/bAOeP2ZBfatvLd3Sq2wSA==</latexit> ˙ p = 0 む : <latexit sha1_base64="RjerUXlalf2i8IQxl5ZkTsd6De4=">AAACQXicfZDLSsNAFIYn9VbjXZdugkWoIiURURcuRF24ESvYVmhKOZme1NHJJMxMxBL6Dm71dXwKH8GduHXj9CKoFQ8MfPznn5lz/iDhTGnXfbFyY+MTk1P5aXtmdm5+YXFpuariVFKs0JjH8ioAhZwJrGimOV4lEiEKONaC2+Nev3aHUrFYXOpOgo0I2oKFjII2UjUsJhsHbnOx4Jbcfjmj4A2hQIZVbi5Z634rpmmEQlMOStU9N9GNDKRmlGPX9lOFCdBbaGPdoIAIVSPrj9t11o3ScsJYmiO001e/38ggUqoTBcYZgb5Wv3s98a9ePdXhfiNjIkk1Cjr4KEy5o2Ont7vTYhKp5h0DQCUzszr0GiRQbRKybf8EzTISz8zD5wlK0LHczHyQ7YiJrlmu7W/16D8j3H8ZDdkmWO93jKNQ3S55u6Xdi53C4dEw4jxZJWukSDyyRw7JKSmTCqHkhjyQR/JkPVuv1pv1PrDmrOGdFfKjrI9PCjmvJw==</latexit> f(p) < 0 :
  50. む む ( ) <latexit sha1_base64="3GPTXDUNJzSzuzbIHbnhZyOHpNY=">AAACRnicfZBNaxRBEIZr1q84fiV69NK4BKLIMhMk5iIE9ZCLmICbBHaWUNNbs2nS09101wSXYX+G1+Tv+Bf8E96CV3s3G8iHWNDw8NZb3dVv6bQKnGW/ks6du/fuP1h6mD56/OTps+WV53vBNl5SX1pt/UGJgbQy1GfFmg6cJ6xLTfvl8adZf/+EfFDWfOOJo2GNY6MqJZGjNChGloX7UK2514fL3ayXzUvchnwBXVjUzuFKshrHZVOTYakxhEGeOR626FlJTdO0aAI5lMc4pkFEgzWFYTvfeSpWozISlfXxGBZz9epEi3UIk7qMzhr5KNzszcR/9QYNV5vDVhnXMBl58VDVaMFWzAIQI+VJsp5EQOlV3FXII/QoOcaUpsVnip/x9CVe/NWRR7b+TVugH9fKTOPnxsXbGf3PiN8vjZHSGGx+M8bbsLfeyzd6G7vvulsfFxEvwUt4BWuQw3vYgm3YgT5IsPADTuEs+Zn8Ts6TPxfWTrKYeQHXqgN/AYpPsF4=</latexit> ˙ p = f(p)

    <latexit sha1_base64="eNGawypYh8jUPIrz6KQjBDmydQc=">AAACQ3icfZDLSgMxFIYz3h3vunQTLIKIlBkRdSOIunAjKlgtdIqcSU9rMJMZkjNiGfoSbvV1fAifwZ24FUxrBW94IPDxn/8kOX+cKWkpCJ68gcGh4ZHRsXF/YnJqemZ2bv7cprkRWBGpSk01BotKaqyQJIXVzCAkscKL+Hq/27+4QWNlqs+onWE9gZaWTSmAnFSNGinxbCe4nC0F5aBX/DeEfSixfp1cznnLblbkCWoSCqythUFG9QIMSaGw40e5xQzENbSw5lBDgrZe9D7c4ctOafBmatzRxHvq14kCEmvbSeycCdCV/dnrin/1ajk1t+uF1FlOqMXHQ81ccUp5d3vekAYFqbYDEEa6v3JxBQYEuYx8PzpAt4zBI3fxcYYGKDWrRQSmlUjdccu1orUu/WeE20+jI98FG/6M8Tecr5fDzfLm6UZpd68f8RhbZEtshYVsi+2yQ3bCKkwwxe7YPXvwHr1n78V7/bAOeP2ZBfatvLd3Sq2wSA==</latexit> ˙ p = 0 む : <latexit sha1_base64="RjerUXlalf2i8IQxl5ZkTsd6De4=">AAACQXicfZDLSsNAFIYn9VbjXZdugkWoIiURURcuRF24ESvYVmhKOZme1NHJJMxMxBL6Dm71dXwKH8GduHXj9CKoFQ8MfPznn5lz/iDhTGnXfbFyY+MTk1P5aXtmdm5+YXFpuariVFKs0JjH8ioAhZwJrGimOV4lEiEKONaC2+Nev3aHUrFYXOpOgo0I2oKFjII2UjUsJhsHbnOx4Jbcfjmj4A2hQIZVbi5Z634rpmmEQlMOStU9N9GNDKRmlGPX9lOFCdBbaGPdoIAIVSPrj9t11o3ScsJYmiO001e/38ggUqoTBcYZgb5Wv3s98a9ePdXhfiNjIkk1Cjr4KEy5o2Ont7vTYhKp5h0DQCUzszr0GiRQbRKybf8EzTISz8zD5wlK0LHczHyQ7YiJrlmu7W/16D8j3H8ZDdkmWO93jKNQ3S55u6Xdi53C4dEw4jxZJWukSDyyRw7JKSmTCqHkhjyQR/JkPVuv1pv1PrDmrOGdFfKjrI9PCjmvJw==</latexit> f(p) < 0 : む : <latexit sha1_base64="4CwN7wBSv4ewWRFHC5eS/mYDyFk=">AAACQXicfZDLSsNAFIYn9VbjXZdugkWoIiURUVci6sKNWMG2QlPKyfSkjk4mYWYiltB3cKuv41P4CO7ErRunF0GteGDg4z//zJzzBwlnSrvui5UbG5+YnMpP2zOzc/MLi0vLVRWnkmKFxjyWVwEo5ExgRTPN8SqRCFHAsRbcHvf6tTuUisXiUncSbETQFixkFLSRqmEx2Thwm4sFt+T2yxkFbwgFMqxyc8la91sxTSMUmnJQqu65iW5kIDWjHLu2nypMgN5CG+sGBUSoGll/3K6zbpSWE8bSHKGdvvr9RgaRUp0oMM4I9LX63euJf/XqqQ73GxkTSapR0MFHYcodHTu93Z0Wk0g17xgAKpmZ1aHXIIFqk5Bt+ydolpF4Zh4+T1CCjuVm5oNsR0x0zXJtf6tH/xnh/stoyDbBer9jHIXqdsnbLe1e7BQOj4YR58kqWSNF4pE9ckhOSZlUCCU35IE8kifr2Xq13qz3gTVnDe+skB9lfXwCDeuvKQ==</latexit> f(p) > 0
  51. : <latexit sha1_base64="VU3mFwqdtDA70QbkT9/LWW0pLkE=">AAACY3icfZDfahNBFMYnW7XtVm1avSvCYKjUPw27oVRvhKJeeCNWMG0hk4azk5PN0PnHzGxpWPIGPk1v9UV8gL6HkzSCtuKBgR/f+ebMnK+wUviQZT8bydKdu/eWV1bTtfsPHq43NzaPvKkcxy430riTAjxKobEbRJB4Yh2CKiQeF2fvZ/3jc3ReGP01TCz2FZRajASHEKVB8xkbmkAtfUvtaYfV+e5O/pJ5USo47Ty3bLrL3NhQO2i2snY2L3ob8gW0yKIOBxuN7TiZVwp14BK87+WZDf0aXBBc4jRllUcL/AxK7EXUoND36/lCU7odlSEdGRePDnSu/nmjBuX9RBXRqSCM/c3eTPxXr1eF0Zt+LbStAmp+/dCokjQYOkuHDoVDHuQkAnAn4l8pH4MDHmKGaco+YFzG4ac4+LNFB8G4FzUDVyqhp3G5kr2a0f+McPHbGCmNweY3Y7wNR512vt/e/7LXOni3iHiFbJGnZIfk5DU5IB/JIekSTr6RS/Kd/GhcJWvJZvL42po0Fncekb8qefILTjG3Qw==</latexit> ˙ p = p2{1 (1 + 2)p}

    ⇢p <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> p = 0 " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
  52. む CIFAR- ( ) ne-tuning む [Chen-He ] : む

    [Bao ] : " Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics.
  53. <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x0 <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x0 <latexit

    sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> W x Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> x0 <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x,x 0 ⇠ N(x0 , 2 I) <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x,x 0 ⇠ N (x 0 , 2 I) <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> L( , W) = 1 2 EkW x StopGrad( x0)k2 2 [Tian+ ] SimSiam impl <latexit sha1_base64="eD8Tr3fo5YqK6nI45PCr3QpXMzM=">AAADI3icjZJLj9MwEMed8FrCqwtHLhbVil1UqgShhQvSiofgsIgi6Haluqoc10mtdZzInqBWbj4On4Yb4sKBK58DJ82ifaAVI1n6aeY/4/GM40IKA2H40/MvXb5y9drG9eDGzVu373Q27x6YvNSMD1kuc30YU8OlUHwIAiQ/LDSnWSz5KD56VcdHX7g2IlefYVnwSUZTJRLBKDjXtPObZBTmjEq7X203HCdkMBe9lvFo58UaY/umIpInMH5MEk2ZJZKqVHL8V3ki/di36GECfAH2E+TFW01npy5p0S6qhzuY6KZcZcnqwoouvPrPmmRVES3SOUymnW7YDxvD5yFqoYtaG0w3vS0yy1mZcQVMUmPGUVjAxFINgrkuA1IaXlB2RFM+dqhoxs3ENgup8JbzzHCSa3cU4MZ7MsPSzJhlFjtl3a45G6ud/4qNS0ieT6xQRQlcsfVFSSkx5LjeLp4JzRnIpQPKtHC9Yjanblvg/kAQkNfcPUbz967wh4JrCrl+ZAnVaSZU5R6Xkl5NFwnp4ljoKHCDjc6O8TwcPOlHu/3dj0+7ey/bEW+g++gB2kYReob20Ds0QEPEvH1Pe9Zb+V/9b/53/8da6nttzj10yvxffwA3vAWu</latexit> L( , W) = E  hW x, StopGrad( x0)i kW xkkStopGrad( x0)k " Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics.
  54. <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="VgvSMU7jTw5uLdIErOiZWBSPaT0=">AAACRnicfZDBShxBEIZr1hh1NImao5cmiyghLDNiVvcmMQcvIQayKuwsUtNbszb2dA/dPeIy7GPkmrxOXsGXyE282rO7gobEgoaPv/7urvrTQgrrougmaMy9mH+5sLgULq+8ev1mdW39xOrScOpyLbU5S9GSFIq6TjhJZ4UhzFNJp+nlYd0/vSJjhVbf3aigfo5DJTLB0Xmpl+ToLtKsuh5vna82o1Ynijsf99gU9ndn0O6wuBVNqgmzOj5fCzaTgeZlTspxidb24qhw/QqNE1zSOExKSwXySxxSz6PCnGy/msw8ZpteGbBMG3+UYxP18Y0Kc2tHeeqd9Yz2714t/qvXK12236+EKkpHik8/ykrJnGZ1AGwgDHEnRx6QG+FnZfwCDXLnYwrD5DP5ZQx98Q9/Lcig0+Z9laAZ5kKN/XLD5ENNzxnx+sHoKfTBPqTH/g8nO6243Wp/220efJpFvAgb8A62IYY9OIAjOIYucNDwA37Cr+B38Ce4De6m1kYwu/MWnlQD7gGzVrGM</latexit> x0 <latexit sha1_base64="NI3gCb+S5Z+HFwNU5r0ymvjbw/Q=">AAACVXicfVBdSxtBFJ3dWmtXbWL7UvBlMIhFStgtNpo3qX3wRYxgVMiGcHdyNxmcnV1mZoth2f4aX+3fEX+M0Nl8gC3qgYEz5547c++JMsG18f0Hx32z9Hb53cp7b3Vt/UOtvvHxQqe5YthlqUjVVQQaBZfYNdwIvMoUQhIJvIyuj6r65S9Umqfy3Ewy7CcwkjzmDIyVBvXPYQJmHMVF2BnzcnG5KXcG9YbfbPtB+/s+nZGDvTlptWnQ9KdokDk6gw1nOxymLE9QGiZA617gZ6ZfgDKcCSy9MNeYAbuGEfYslZCg7hfTFUq6bZUhjVNljzR0qj7tKCDRepJE1lnNqP+vVeJztV5u4oN+wWWWG5Rs9lGcC2pSWuVBh1whM2JiCTDF7ayUjUEBMzY1zwt/ol1G4Yl9+DRDBSZVu0UIapRwWdrlRuHXir1mhJuF0TLPBrtIj75MLr41g1azdbbXOPwxj3iFbJIt8oUEZJ8ckmPSIV3CyG9yS+7IH+feeXSX3OWZ1XXmPZ/IP3BrfwHlWLWE</latexit> x0 <latexit

    sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> W x Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> x0 <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x,x 0 ⇠ N(x0 , 2 I) <latexit sha1_base64="2P8kSfidrPJjfDWLcVdYq/MxEXo=">AAACg3icfVBdS9xAFJ2N9StWXe2jL6GLaHVZkkXUF0GqFPvQaqGrwmZdbmZv4uDMJMxMxCXkH/lr+lbsj3GyrmhVvDBw5pxzZ+49UcaZNr7/t+ZMfJicmp6Zdec+zi8s1peWT3WaK4odmvJUnUegkTOJHcMMx/NMIYiI41l0dVDpZ9eoNEvlbzPMsCcgkSxmFIyl+vVvoQBzGcXFTdl8gmuhZmJ0pcCLn+X6k9T3m1ZMBFy0H8nv5Zd+veG3/FF5r0EwBg0yrpP+Um01HKQ0FygN5aB1N/Az0ytAGUY5lm6Ya8yAXkGCXQslCNS9YrRw6a1aZuDFqbJHGm/EPu8oQGg9FJF1VjPql1pFvqV1cxPv9goms9ygpA8fxTn3TOpV6XkDppAaPrQAqGJ2Vo9eggJqbMauGx6iXUbhD/vwcYYKTKo2ihBUIpgs7XJJ2KzQe0a4eTRa5Npgg5cxvgan7Vaw3dr+tdXY/zqOeIaskM9knQRkh+yTI3JCOoSSW/KH3JF/zqSz6bSdrQerUxv3fCL/lbN3D4I+xpA=</latexit> x,x 0 ⇠ N (x 0 , 2 I) <latexit sha1_base64="fnvw+Uj/IFxh4VrNHQKHz3Vlq7Y=">AAACvXicfZFrb9MwFIadcBthQAeCL3yJqCY2NKqkQoMviImL4AOIIug6qe4qxz1prTl2ZJ9AqywSf5M/wO/AabNpF8SRLD16z3t8eZ3kUliMot+ef+Xqtes31m4Gt9Zv37nb2ri3b3VhOPS5ltocJMyCFAr6KFDCQW6AZYmEQXL0tu4PfoCxQqvvuMhhlLGpEqngDJ00bv2iGcMZZ7L8VG0tOUlpbyZ2Gg4H269oahiPuyslKd9X9Pi0e2bkRJs/owhzLL+hzj8YNjm376npyTY9HncPu+NWO+pEywovQ9xAmzTVG294m3SieZGBQi6ZtcM4ynFUMoOCS6gCWljIGT9iUxg6VCwDOyqXUVXhplMmYaqNWwrDpXp2omSZtYsscc76pvZirxb/1RsWmL4clULlBYLiq4PSQoaowzr3cCIMcJQLB4wb4e4a8hlzwaL7nSCg78A9xsBnt/GXHAxDbZ6WlJlpJlTlHjelOzX9z8jmJ0ZHgQs2vhjjZdjvduLdzu7X5+29N03Ea+QReUy2SExekD3ykfRIn3Dyx1v3HngP/dc++NJXK6vvNTP3ybnyf/4FFEvaEQ==</latexit> L( , W) = 1 2 EkW x StopGrad( x0)k2 2 [Tian+ ] SimSiam impl <latexit sha1_base64="eD8Tr3fo5YqK6nI45PCr3QpXMzM=">AAADI3icjZJLj9MwEMed8FrCqwtHLhbVil1UqgShhQvSiofgsIgi6Haluqoc10mtdZzInqBWbj4On4Yb4sKBK58DJ82ifaAVI1n6aeY/4/GM40IKA2H40/MvXb5y9drG9eDGzVu373Q27x6YvNSMD1kuc30YU8OlUHwIAiQ/LDSnWSz5KD56VcdHX7g2IlefYVnwSUZTJRLBKDjXtPObZBTmjEq7X203HCdkMBe9lvFo58UaY/umIpInMH5MEk2ZJZKqVHL8V3ki/di36GECfAH2E+TFW01npy5p0S6qhzuY6KZcZcnqwoouvPrPmmRVES3SOUymnW7YDxvD5yFqoYtaG0w3vS0yy1mZcQVMUmPGUVjAxFINgrkuA1IaXlB2RFM+dqhoxs3ENgup8JbzzHCSa3cU4MZ7MsPSzJhlFjtl3a45G6ud/4qNS0ieT6xQRQlcsfVFSSkx5LjeLp4JzRnIpQPKtHC9Yjanblvg/kAQkNfcPUbz967wh4JrCrl+ZAnVaSZU5R6Xkl5NFwnp4ljoKHCDjc6O8TwcPOlHu/3dj0+7ey/bEW+g++gB2kYReob20Ds0QEPEvH1Pe9Zb+V/9b/53/8da6nttzj10yvxffwA3vAWu</latexit> L( , W) = E  hW x, StopGrad( x0)i kW xkkStopGrad( x0)k " Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics.
  55. : <latexit sha1_base64="fIc2AH4mp14inm7PC5NN3idswE8=">AAACeXicfZDNbhMxFIWd4a8MP03LshtDVKlqSTSToJQFlSrCgg2iSKStlElGd5w7E6u2x7I9iGiUl+nTdAs7noUNThokaBFXsvTp3GNf35Npwa2Loh+N4M7de/cfbDwMHz1+8nSzubV9asvKMByyUpTmPAOLgiscOu4EnmuDIDOBZ9nFYNk/+4LG8lJ9dnONYwmF4jln4LyUNt8k09JRfdROcgOs7g7SmOpJnx7QQdrVkx5te+jpSXdRxweJ5YUEz+3EzEqq02Yr6kSrorchXkOLrOsk3Wrs+nmskqgcE2DtKI60G9dgHGcCF2FSWdTALqDAkUcFEu24Xq25oLtemdK8NP4oR1fqnzdqkNbOZeadEtzM3uwtxX/1RpXLX49rrnTlULHrQXklqCvpMjM65QaZE3MPwAz3f6VsBj4v55MNw+Qd+mUMfvAPf9RowJVmv07AFJKrhV+uSF4u6X9G+Prb6Cn0wcY3Y7wNp91O3O/0P71qHb9dR7xBdsgLskdickiOyXtyQoaEkUtyRb6R742fwfNgL9i/tgaN9Z1n5K8Ker8ALDO+0A==</latexit> ˙ p = 2C1p6 + C2p3 C3p2

    1 + 2 ⇢p <latexit sha1_base64="5Hr53OTIAowhIyWIpjNH1R8x+lQ=">AAACXnicfZDfShtBFMYnq9V0W2vUm4I30wahlBJ2RdRLUS+8KUZojJAJ4ezkbDI4O7vMzErCmmufxlt9Fu98lM7GFfxTemDgx3e+mTnnizIpjA2Ch5q3sPhhabn+0f/0eeXLamNt/dykuebY4alM9UUEBqVQ2LHCSrzINEISSexGl0dlv3uF2ohU/bHTDPsJjJSIBQfrpEHjG7tmCdhxFNNuBaw9Fs/ahF0PGs2gFcyLvoewgiapqj1Yq22xYcrzBJXlEozphUFm+wVoK7jEmc9ygxnwSxhhz6GCBE2/mO8yo1tOGdI41e4oS+fqyxsFJMZMk8g5yxnN214p/qvXy2283y+EynKLij99FOeS2pSWwdCh0MitnDoAroWblfIxaODWxef77BjdMhp/u4dPM9RgU/2zYKBHiVAzt9yI/Srpf0aYPBsd+S7Y8G2M7+F8uxXutnbPdpoHh1XEdbJJvpMfJCR75ICckDbpEE5uyC25I/e1R2/JW/FWn6xerbqzQV6V9/UvSB23fQ==</latexit> kW xk <latexit sha1_base64="5Hr53OTIAowhIyWIpjNH1R8x+lQ=">AAACXnicfZDfShtBFMYnq9V0W2vUm4I30wahlBJ2RdRLUS+8KUZojJAJ4ezkbDI4O7vMzErCmmufxlt9Fu98lM7GFfxTemDgx3e+mTnnizIpjA2Ch5q3sPhhabn+0f/0eeXLamNt/dykuebY4alM9UUEBqVQ2LHCSrzINEISSexGl0dlv3uF2ohU/bHTDPsJjJSIBQfrpEHjG7tmCdhxFNNuBaw9Fs/ahF0PGs2gFcyLvoewgiapqj1Yq22xYcrzBJXlEozphUFm+wVoK7jEmc9ygxnwSxhhz6GCBE2/mO8yo1tOGdI41e4oS+fqyxsFJMZMk8g5yxnN214p/qvXy2283y+EynKLij99FOeS2pSWwdCh0MitnDoAroWblfIxaODWxef77BjdMhp/u4dPM9RgU/2zYKBHiVAzt9yI/Srpf0aYPBsd+S7Y8G2M7+F8uxXutnbPdpoHh1XEdbJJvpMfJCR75ICckDbpEE5uyC25I/e1R2/JW/FWn6xerbqzQV6V9/UvSB23fQ==</latexit> kW xk <latexit sha1_base64="+SQM3l4yMhRu2crR6b540oMlomI=">AAACPnicfZDLSsNAFIYnXmu8VpdugqUgIiURUTdCURduxIrWFppSTqancXAyCTMTsYQ+glt9HV/DF3Anbl06vQhaxQMDH//5Z+acP0g4U9p1X6yJyanpmdncnD2/sLi0vJJfvVZxKilWacxjWQ9AIWcCq5ppjvVEIkQBx1pwe9zv1+5QKhaLK91NsBlBKFiHUdBGukwO3dZKwS25g3J+gzeCAhlVpZW3in47pmmEQlMOSjU8N9HNDKRmlGPP9lOFCdBbCLFhUECEqpkNZu05RaO0nU4szRHaGajfb2QQKdWNAuOMQN+o8V5f/KvXSHXnoJkxkaQaBR1+1Em5o2Onv7jTZhKp5l0DQCUzszr0BiRQbeKxbf8EzTISz8zD5wlK0LHcynyQYcREzywX+tt9+s8I919GQ7YJ1huP8Tdc75S8vdLexW6hfDSKOEfWyQbZJB7ZJ2VySiqkSigJyQN5JE/Ws/VqvVnvQ+uENbqzRn6U9fEJR3KuUw==</latexit> p = 0 " Bao, H. (2025). Feature normalization prevents collapse of non-contrastive learning dynamics.
  56. Φ-Net: む (entohrinal cortex) む CA 訓 <latexit sha1_base64="OYIuD3tQeI2w0dbax8wVkezfngY=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7Xou3Fspepeb5tZ1dOoC97SFUa9SveP0qk2GdtRadtaCdsCxGaZgArRu+l5pmDspwJrDnBpnGFNgtdLBhUUKMupn3h+7RNau0aZQoe6ShffXrjRxirbtxaJ3FkPpnrxB/6zUyE+01cy7TzKBkg4+iTFCT0CIB2uYKmRFdC8AUt7NSdgMKmLE5uW5whHYZhSf24dMUFZhEbeQBqE7MZc8u1wk2C/rPCPefRkuuDfYzPfo3XG5V/Gqler5d3j8YRjxJVsgqWSc+2SX75JickTphJCUP5JE8Oc/Oq/PmvA+sJWd4Z5l8q5LzAQY1sjc=</latexit> xi

    <latexit sha1_base64="oBHF1h2GFlEaPhB/HhOweL1oh7I=">AAACR3icfZDfShtBFMZnY/3TrdWol70ZGoQiJexKTMxd0F54U4zQxEA2hLOTs8mY2dllZlYMS16jt+3r9BH6FN6VXnY2iVBFPTDw4zvfzJzzhang2njeb6e09mZ9Y3Prrftu+/3Obnlvv6uTTDHssEQkqheCRsEldgw3AnupQohDgdfh9LzoX9+i0jyR38wsxUEMY8kjzsBYKQhiMJMwyu/mw5thueJVm57fPGnQJZzWVlBvUr/qLapCVtUe7jmHwShhWYzSMAFa930vNYMclOFM4NwNMo0psCmMsW9RQox6kC+GntNDq4xolCh7pKEL9f8bOcRaz+LQOosh9dNeIT7X62cmOh3kXKaZQcmWH0WZoCahRQJ0xBUyI2YWgCluZ6VsAgqYsTm5bvAF7TIKv9qHL1NUYBJ1lAegxjGXc7vcOPhc0GtGuHswWnJtsA/p0Zehe1z169X6Va3SOltFvEU+kI/kE/FJg7TIBWmTDmEkJd/JD/LT+eXcO3+cv0tryVndOSCPquT8AwgNsjg=</latexit> xj <latexit sha1_base64="gwzGj/YlW7S86DEYY7wqqSaKG3g=">AAACR3icfZDNSiNBFIWrM/62P2N06aYwCCISusWJyU7GWbgRFYwK6RBuV24npdXVTVW1GJq8hlvndeYR5inciUurkwgq6oWCj3NPVd17wlRwbTzvv1P6MTU9Mzs37y4sLi3/XCmvXugkUwybLBGJugpBo+ASm4YbgVepQohDgZfhzWHRv7xFpXkiz80gxXYMPckjzsBYKQhiMP0wyvvDznVnpeJVG57f+LVPx1Dfm0CtQf2qN6oKmdRpp+xsBt2EZTFKwwRo3fK91LRzUIYzgUM3yDSmwG6ghy2LEmLU7Xw09JBuWqVLo0TZIw0dqW9v5BBrPYhD6yyG1B97hfhZr5WZqN7OuUwzg5KNP4oyQU1CiwRolytkRgwsAFPczkpZHxQwY3Ny3eAP2mUUHtuHT1JUYBK1nQegejGXQ7tcL9gp6Dsj3L0aLbk22Nf06NdwsVv1a9Xa2V7l4Pck4jmyTjbIFvHJPjkgR+SUNAkjKbknD+Sv8895dJ6c57G15EzurJF3VXJeAOpOsig=</latexit> hj <latexit sha1_base64="M7L9l5V9YJ8k17bUYdQox0J08kU=">AAACR3icfZDPSiNBEMZ7ou5mR3c1etxLYxBEljAjmj+3sHrYixjBqJAJoaZTkzT29AzdPYthyGt4XV/HR/ApvMke7Uki6LJa0PDjq6+7q74wFVwbz3twSkvLK58+l7+4q2tfv61vVDYvdJIphl2WiERdhaBRcIldw43Aq1QhxKHAy/D6qOhf/kaleSLPzSTFfgwjySPOwFgpCGIw4zDKx9MBH2xUvVrL81uHDTqH5sEC6i3q17xZVcmiOoOKsxMME5bFKA0ToHXP91LTz0EZzgRO3SDTmAK7hhH2LEqIUffz2dBTumOVIY0SZY80dKa+vpFDrPUkDq2zGFL/2yvE//V6mYma/ZzLNDMo2fyjKBPUJLRIgA65QmbExAIwxe2slI1BATM2J9cNjtEuo/DEPnyaogKTqL08ADWKuZza5UbBj4I+MsLNi9GSa4N9SY++Dxf7Nb9eq58dVNs/FxGXyXeyTXaJTxqkTX6RDukSRlJyS/6QO+feeXSenL9za8lZ3Nkib6rkPAPodrIn</latexit> hi <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> zi Stop Grad <latexit sha1_base64="UXgnZhucLLIyKxxyuDmLz6hD2XQ=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9OTdnAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7Qi6goWMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6C11sGRQQoWpno0kHTtkoHSeMpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVSHR+2MiSTVKOj4ozDljo6d4dpOh0mkmvcNAJXMzOrQHkig2oRj2/4pmmUknpuHLxKUoGO5k/kguxETA7Nc198d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIGytyA==</latexit> f <latexit sha1_base64="UXgnZhucLLIyKxxyuDmLz6hD2XQ=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9OTdnAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7Qi6goWMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6C11sGRQQoWpno0kHTtkoHSeMpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVSHR+2MiSTVKOj4ozDljo6d4dpOh0mkmvcNAJXMzOrQHkig2oRj2/4pmmUknpuHLxKUoGO5k/kguxETA7Nc198d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIGytyA==</latexit> f <latexit sha1_base64="ohipUW93olWICBW31kkLiUn2UJE=">AAACPHicfZDLSgMxFIYz9T7eqi7dDJaCiJQZkepS1IUb0YK9QKeUM+lpG5rJDElGLEOfwK2+ju/h3p24dW16EbQVDwQ+/vMnOecPYs6Udt1XKzM3v7C4tLxir66tb2xmt7YrKkokxTKNeCRrASjkTGBZM82xFkuEMOBYDXoXw371HqVikbjT/RgbIXQEazMK2kilbjObcwvuqJxZ8CaQI5O6bW5Zeb8V0SREoSkHpeqeG+tGClIzynFg+4nCGGgPOlg3KCBE1UhHkw6cvFFaTjuS5gjtjNSfN1IIleqHgXGGoLtqujcU/+rVE90+baRMxIlGQccftRPu6MgZru20mESqed8AUMnMrA7tggSqTTi27V+iWUbitXn4JkYJOpIHqQ+yEzIxMMt1/MMh/WeEh2+jIdsE603HOAuVo4JXLBRLx7mz80nEy2SX7JF94pETckauyC0pE0qQPJIn8my9WG/Wu/UxtmasyZ0d8quszy8kHK3K</latexit> h <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="UXgnZhucLLIyKxxyuDmLz6hD2XQ=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9OTdnAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7Qi6goWMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6C11sGRQQoWpno0kHTtkoHSeMpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVSHR+2MiSTVKOj4ozDljo6d4dpOh0mkmvcNAJXMzOrQHkig2oRj2/4pmmUknpuHLxKUoGO5k/kguxETA7Nc198d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIGytyA==</latexit> f <latexit sha1_base64="0VGLZ6c/DGB+9vEkCqaS7AY0zLg=">AAACRXicfZDLShxBFIar1RjTSYzGZTaFw0CQMHSLmctO1IWboIJzIdPDcLrm9Ewx1dVNVXXI0MxbuNXX8Rl8CHfiNlbPBRLRHCj4+M9fVef8YSq4Np5356ysrr1Zf7vxzn3/4ePmp63tzy2dZIphkyUiUZ0QNAousWm4EdhJFUIcCmyH4+Oi3/6FSvNEXppJir0YhpJHnIGx0s8gBjMKo3w07W+VvErD8xvfa3QO9YMFVBvUr3izKpFFnfe3nXIwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls5GntGyVAY0SZY80dKb+fSOHWOtJHFpnMaJ+3ivEl3rdzET1Xs5lmhmUbP5RlAlqElrsTwdcITNiYgGY4nZWykaggBmbkusGJ2iXUfjDPnyWogKTqL08ADWMuZza5YbBt4L+Z4TfS6Ml1wa7TI++Dq39il+tVC8OSodHi4g3yBeyS74Sn9TIITkl56RJGJHkilyTG+fWuXcenMe5dcVZ3Nkh/5Tz5wklVLJK</latexit> h <latexit sha1_base64="5pCyBpbCEsb/Wsd15Ao+4VD0Ajc=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9PTdHAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7QhCwbqMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6CyG2DAqIULWz0aQDp2yUjtONpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVR3j9oZE0mqUdDxR92UOzp2hms7HSaRat43AFQyM6tDeyCBahOObfunaJaReG4evkhQgo7lTuaDDCMmBma50N8d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIkStyQ==</latexit> g Stop Grad augmented signals original signal CA predictor CA predictor CA loss CA loss " Ishikawa, S., Yamada, M., Bao, H., Takezawa, Y. (2025). PhiNets: Brain-inspired non-contrastive learning based on temporal prediction hypothesis.
  57. <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W) Part : <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit>

    ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0)
  58. む む <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) " Nesterov,

    Y. (2003). Introductory lectures on convex optimization: A basic course. - <latexit sha1_base64="T3+E/1NrjncdQhovxjrAHxzJsuU=">AAACZ3icfVBdSxtBFJ2stmpam2hBCr6shkKREnYlROmTYB/6YPEDo4KbhruTmzg4H8vMXTEseeiv6Wv9Of6E/gtnY4ptLD0wcObcc7n3njSTwlEU3VeCufkXLxcWl6qvXi+/qdVXVs+cyS3HDjfS2IsUHEqhsUOCJF5kFkGlEs/T6/2yfn6D1gmjT2mUYVfBUIuB4EBe6tXXDz4lCugqTYuT8bd+Qubp26s3omY0QficxFPSYFMc9VYqraRveK5QE5fg3GUcZdQtwJLgEsfVJHeYAb+GIV56qkGh6xaTK8bhe6/0w4Gx/mkKJ+qfHQUo50Yq9c5yRTdbK8V/1lI1M5kGu91C6Cwn1Pxx8CCXIZmwjCjsC4uc5MgT4Fb43UN+BRY4+SCr1eQz+uMsfvWDDjO0QMZuFQnYoRJ67I8dJh9L9j8j3P42euZzjmdTfU7Otptxu9k+bjX29qeJL7J1tsk+sJjtsD32hR2xDuPsO/vBfrK7yq+gFqwF7x6tQWXa85b9hWDjAbJbu+E=</latexit> L : Rd ! R <latexit sha1_base64="h7xrZED8DEDDTlIZzQs9fydeZKo=">AAACT3icfZBNSwMxEIaz9autX1WPXhaLICJlV0Q9FvTgRVSwKrhFZtNpDSbZJZkVy9Lf4FV/lkd/iTcxrStoFQcCD++8YWbeOJXCUhC8eqWJyanpmXKlOjs3v7BYW1q+sElmOLZ4IhNzFYNFKTS2SJDEq9QgqFjiZXx3MOxf3qOxItHn1E+xraCnRVdwICe1ohgJbmr1oBGMyv8NYQF1VtTpzZK3E3USninUxCVYex0GKbVzMCS4xEE1yiymwO+gh9cONSi07Xy07cBfd0rH7ybGPU3+SP3+IwdlbV/FzqmAbu14byj+2YvV2GTq7rdzodOMUPPPwd1M+pT4wyj8jjDISfYdADfC7e7zWzDAyQVWrUaH6I4zeOwGnaRogBKzmUdgekrogTu2F20N6T8jPHwZHbmcw/FUf8PFdiPcbeye7dSbB0XiZbbK1tgGC9kea7IjdspajDPBHtkTe/ZevDfvvVRYS14BK+xHlSofJnC0Gw==</latexit> <latexit sha1_base64="xsZleZEN3LF1Q2cgXbCIKLx2uFU=">AAACYHicfVBdSxtBFJ1s1cbUj6T6pC9DQ6EUSXdLUB8FffBFVDCJ4IZwd3ITh8zMLjN3i3HJf+lr/Ue+9pd0EiNolB4YOHPuudx7T5Ip6SgMH0vBh6XllY/l1cqntfWNzWrtc9uluRXYEqlK7XUCDpU02CJJCq8zi6AThZ1kdDytd36hdTI1VzTOsKthaORACiAv9arbMRLECp1zUvPoR5z4f69aDxvhDPwtieakzua46NVKzbifilyjIaHAuZsozKhbgCUpFE4qce4wAzGCId54akCj6xaz9Sf8q1f6fJBa/wzxmfqyowDt3Fgn3qmBbt1ibSq+W0v0wmQaHHYLabKc0IinwYNccUr5NBvelxYFqbEnIKz0u3NxCxYE+QQrlfgE/XEWz/yg8wwtUGq/FzHYoZZm4o8dxntT9j8j3D0bPfM5R4upviXtn41ov7F/2awfHc8TL7Nd9oV9YxE7YEfslF2wFhPsnv1mf9hD6W9QDjaD2pM1KM17ttgrBDv/ADUXuFY=</latexit> ⌘ . 1/ <latexit sha1_base64="01PnSFuqSsJTebtdZgUk3NZYgTk=">AAACdHicfVBNaxsxEJW3H0ndjzjtsTmImoBTitkNIe0xkB56aEgKdhKIjJmVZ9ciknaRZtuaxX+hv6bX9n/0j/RcreNA4oQOSDzevOHNvLTUylMc/2lFDx4+ery2/qT99NnzFxudzZenvqicxKEsdOHOU/ColcUhKdJ4XjoEk2o8Sy8Pm/7ZV3ReFXZAsxJHBnKrMiWBAjXu9D73hAGapln9bT4e7AiN/Dh8GfVE5kAmA+FUPqWdcacb9+NF8bsgWYIuW9bJeLO1JyaFrAxakhq8v0jikkY1OFJS47wtKo8lyEvI8SJACwb9qF6cNOfbgZnwrHDhWeIL9uZEDcb7mUmDslnfr/Ya8t5ealacKfswqpUtK0Irr4yzSnMqeJMXnyiHkvQsAJBOhd25nEJIhkKq7bb4iOE4h0fB6LhEB1S4t7UAlxtl5+HYXLxr0P+E8P1aGFDIOVlN9S443e0n+/39L3vdg8Nl4uvsNXvDeixh79kB+8RO2JBJ9oP9ZL/Y79bfaCvqRttX0qi1nHnFblXU/wdkgcAs</latexit> L(wT )  O ✓ 1 T ◆
  59. む む <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) む :

    Descent lemma <latexit sha1_base64="5+hr1bFxR7zFk+0KxrTyGisgJ9U=">AAADjHichVJdaxNBFJ3t+lFX7ZePvgwGS2LbkA2hFrRYqIgPihVMW+jEcHdydzN0dnaZmahhk7/jr/FVwX/jbBKh3dZ6YeFwzzn3ayfKpTC21frtLfm3bt+5u3wvuP/g4crq2vrGsclGmmOXZzLTpxEYlEJh1wor8TTXCGkk8SQ6Pyz5ky+ojcjUJzvOsZdCokQsOFiX6q97r97VWQp2GMXF12m/sFvhtBFs7tNLaUt3KEMLTEEkocI1GpSxYJNJrBJ0h0lQiSOuN247wT+rUqbn3i3KYg2cRU7aZpMbLGzSb39uz6apLtAoy0iMbf1CsXkpLZKh85Zw8t+6L6tcf63WarZmQa+CcAFqZBFH7uQdNsj4KEVluQRjzsJWbnsFaCu4xGnARgZz4OeQ4JmDClI0vWL2q6f0qcsMaJxp9ylLZ9mLjgJSY8Zp5JTllKbKlclruSitdLbxXq8QKh9ZVHzeOB5JajNaviM6EBq5lWMHgGvhZqd8CO601r22IGCv0S2n8b1r9CFHDTbTzwoGOkmFmrplE7ZdopuE8O2v0CF357B61avguN0Md5u7Hzu1g8PFxZfJY/KE1ElInpMD8pYckS7h3nfvh/fT++Wv+B3/hb8/ly55C88jcin8N38A0Kcgzg==</latexit> L(wt+1) = L(wt ⌘rL(wt))  L(wt) hrL(wt), ⌘rL(wt)i + 2 k ⌘rL(wt)k2 2 = L(wt) + ✓ 2 ⌘ ◆ ⌘krL(wt)k2 2 < L(wt) <latexit sha1_base64="JtSbEOdrK83T9gPDi3ZI0tyc2Go=">AAADd3icfVLLbhMxFPUkPMrwaAtLFlhEREnTRJmoatkgVSoLFiCKRNpKdYjuOHcmVj2eke0Uokk+gq9hC5/Bp7DDkwSpnZZcydbRPffclx1mUhjb7f72KtU7d+/d33jgP3z0+Mnm1vbTE5NONMc+T2Wqz0IwKIXCvhVW4lmmEZJQ4ml4cVTwp5eojUjVZzvNcJBArEQkOFjnGm57O+8bLAE7DqP863yY21Ywb/r1N/Sa29I2ZWiBKQgllLhmkzLm15nEMkFbTIKKHXG7cJe2/5+VMr3UtiiLNHAWutAem62RsNmw96W36KY8QLNIIzGyjavJ3NUOmBbx2ImLtLN1iYdbtW6nuzB6EwQrUCMrO3bb3WOjlE8SVJZLMOY86GZ2kIO2gkuc+2xiMAN+ATGeO6ggQTPIF686p6+cZ0SjVLujLF14rypySIyZJqGLLHo1Za5w3sqFSamyjV4PcqGyiUXFl4WjiaQ2pcWXoSOhkVs5dQC4Fq53ysfgtmjdx/J99hbdcBo/uEIfM9RgU72TM9BxItTcDRuz3QKtC4Rv/wIdcnsOylu9CU56nWC/s/9pr3Z4tNr4BnlOXpIGCcgBOSTvyDHpE+599354P71flT/VF9V6tbEMrXgrzTNyzarBX99wGhk=</latexit> L(wt+1) = L(wt ⌘rL(wt))  L(wt) + hrL(wt), ⌘rL(wt)i + 2 k ⌘rL(wt)k2 2 = L(wt) + ✓ 2 ⌘ 1 ◆ ⌘krL(wt)k2 2 " Nesterov, Y. (2003). Introductory lectures on convex optimization: A basic course. - <latexit sha1_base64="T3+E/1NrjncdQhovxjrAHxzJsuU=">AAACZ3icfVBdSxtBFJ2stmpam2hBCr6shkKREnYlROmTYB/6YPEDo4KbhruTmzg4H8vMXTEseeiv6Wv9Of6E/gtnY4ptLD0wcObcc7n3njSTwlEU3VeCufkXLxcWl6qvXi+/qdVXVs+cyS3HDjfS2IsUHEqhsUOCJF5kFkGlEs/T6/2yfn6D1gmjT2mUYVfBUIuB4EBe6tXXDz4lCugqTYuT8bd+Qubp26s3omY0QficxFPSYFMc9VYqraRveK5QE5fg3GUcZdQtwJLgEsfVJHeYAb+GIV56qkGh6xaTK8bhe6/0w4Gx/mkKJ+qfHQUo50Yq9c5yRTdbK8V/1lI1M5kGu91C6Cwn1Pxx8CCXIZmwjCjsC4uc5MgT4Fb43UN+BRY4+SCr1eQz+uMsfvWDDjO0QMZuFQnYoRJ67I8dJh9L9j8j3P42euZzjmdTfU7Otptxu9k+bjX29qeJL7J1tsk+sJjtsD32hR2xDuPsO/vBfrK7yq+gFqwF7x6tQWXa85b9hWDjAbJbu+E=</latexit> L : Rd ! R <latexit sha1_base64="h7xrZED8DEDDTlIZzQs9fydeZKo=">AAACT3icfZBNSwMxEIaz9autX1WPXhaLICJlV0Q9FvTgRVSwKrhFZtNpDSbZJZkVy9Lf4FV/lkd/iTcxrStoFQcCD++8YWbeOJXCUhC8eqWJyanpmXKlOjs3v7BYW1q+sElmOLZ4IhNzFYNFKTS2SJDEq9QgqFjiZXx3MOxf3qOxItHn1E+xraCnRVdwICe1ohgJbmr1oBGMyv8NYQF1VtTpzZK3E3USninUxCVYex0GKbVzMCS4xEE1yiymwO+gh9cONSi07Xy07cBfd0rH7ybGPU3+SP3+IwdlbV/FzqmAbu14byj+2YvV2GTq7rdzodOMUPPPwd1M+pT4wyj8jjDISfYdADfC7e7zWzDAyQVWrUaH6I4zeOwGnaRogBKzmUdgekrogTu2F20N6T8jPHwZHbmcw/FUf8PFdiPcbeye7dSbB0XiZbbK1tgGC9kea7IjdspajDPBHtkTe/ZevDfvvVRYS14BK+xHlSofJnC0Gw==</latexit> <latexit sha1_base64="xsZleZEN3LF1Q2cgXbCIKLx2uFU=">AAACYHicfVBdSxtBFJ1s1cbUj6T6pC9DQ6EUSXdLUB8FffBFVDCJ4IZwd3ITh8zMLjN3i3HJf+lr/Ue+9pd0EiNolB4YOHPuudx7T5Ip6SgMH0vBh6XllY/l1cqntfWNzWrtc9uluRXYEqlK7XUCDpU02CJJCq8zi6AThZ1kdDytd36hdTI1VzTOsKthaORACiAv9arbMRLECp1zUvPoR5z4f69aDxvhDPwtieakzua46NVKzbifilyjIaHAuZsozKhbgCUpFE4qce4wAzGCId54akCj6xaz9Sf8q1f6fJBa/wzxmfqyowDt3Fgn3qmBbt1ibSq+W0v0wmQaHHYLabKc0IinwYNccUr5NBvelxYFqbEnIKz0u3NxCxYE+QQrlfgE/XEWz/yg8wwtUGq/FzHYoZZm4o8dxntT9j8j3D0bPfM5R4upviXtn41ov7F/2awfHc8TL7Nd9oV9YxE7YEfslF2wFhPsnv1mf9hD6W9QDjaD2pM1KM17ttgrBDv/ADUXuFY=</latexit> ⌘ . 1/ <latexit sha1_base64="01PnSFuqSsJTebtdZgUk3NZYgTk=">AAACdHicfVBNaxsxEJW3H0ndjzjtsTmImoBTitkNIe0xkB56aEgKdhKIjJmVZ9ciknaRZtuaxX+hv6bX9n/0j/RcreNA4oQOSDzevOHNvLTUylMc/2lFDx4+ery2/qT99NnzFxudzZenvqicxKEsdOHOU/ColcUhKdJ4XjoEk2o8Sy8Pm/7ZV3ReFXZAsxJHBnKrMiWBAjXu9D73hAGapln9bT4e7AiN/Dh8GfVE5kAmA+FUPqWdcacb9+NF8bsgWYIuW9bJeLO1JyaFrAxakhq8v0jikkY1OFJS47wtKo8lyEvI8SJACwb9qF6cNOfbgZnwrHDhWeIL9uZEDcb7mUmDslnfr/Ya8t5ealacKfswqpUtK0Irr4yzSnMqeJMXnyiHkvQsAJBOhd25nEJIhkKq7bb4iOE4h0fB6LhEB1S4t7UAlxtl5+HYXLxr0P+E8P1aGFDIOVlN9S443e0n+/39L3vdg8Nl4uvsNXvDeixh79kB+8RO2JBJ9oP9ZL/Y79bfaCvqRttX0qi1nHnFblXU/wdkgcAs</latexit> L(wT )  O ✓ 1 T ◆ <latexit sha1_base64="d4Uyi6fmASIkHcAbdWOXfDlRXAM=">AAACWHicfZDfSxtBEMcn1x9qamtMH305GgoiJb0TiT4K8aEvpQpGA7koc5tJXNzdO3bnxHDk/+ir/a/av6abeIJGcWDhw3e+y8x801xJx1H0txa8efvu/crqWv3D+sdPG43N5pnLCiuoJzKV2X6KjpQ01GPJivq5JdSpovP0ujvvn9+QdTIzpzzNaahxYuRYCmQvXSTEmCja/Z6kni4bragdLSp8DnEFLajq+HKztpeMMlFoMiwUOjeIo5yHJVqWQtGsnhSOchTXOKGBR4Oa3LBcrD0Lv3plFI4z65/hcKE+/lGidm6qU+/UyFduuTcXX+ylemkyjw+GpTR5wWTE/eBxoULOwnkm4UhaEqymHlBY6XcPxRVaFOyTq9eTI/LHWfrpB/3KySJndqdM0E60NDN/7CT5NqfXjHj7YPTkc46XU30OZ7vtuNPunOy1DrtV4quwBV9gG2LYh0P4AcfQAwEWfsMd/Kn9CyBYCdburUGt+vMZnlTQ/A9GUrWb</latexit> ⌘  2/
  60. む " Li, H., Xu, Z., Taylor, G., Studer, C.,

    & Goldstein, T. (2018). Visualizing the loss landscape of neural nets. [Li+ ] <latexit sha1_base64="xSP7Z+mT2idx0E6kPyi9VtfYiRA=">AAACVHicfZBNSwMxEIaz63f9qnr0slgEESm7ItqLIOjBi6hgVXSrzKbTGkyySzIrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxJJoWlMHzz/JHRsfGJyanS9Mzs3Hx5YfHcprnhWOepTM1lAhal0FgnQRIvM4OgEokXyf1+r3/xgMaKVJ9RJ8OGgrYWLcGBnHQVJ0iwG4U3tdtyJayG/Qp+QzSAChvUye2CtxU3U54r1MQlWHsdhRk1CjAkuMRuKc4tZsDvoY3XDjUotI2iv3I3WHVKM2ilxj1NQV/9/qMAZW1HJc6pgO7scK8n/tlL1NBkatUahdBZTqj55+BWLgNKg14eQVMY5CQ7DoAb4XYP+B0Y4ORSK5XiA3THGTxyg44zNECpWS9iMG0ldNcd2443evSfER6/jI5cztFwqr/hfLMabVe3T7cqe/uDxCfZMlthayxiO2yPHbITVmecafbEntmL9+q9+yP+2KfV9wZ/ltiP8uc+ABRptII=</latexit> = 108
  61. Edge of stability <latexit sha1_base64="hQmOQyoEuY3wbEJeohb+M59zbsE=">AAACVnicfZDdSiNBEIV7Rt1ks3/Jerk3g0FYliXMiLheCtkLb0SFjQpOkJpOJTb2H901YhjyGt7qY+nLiJ0YYY1iQcPHqdNU1SmsFJ7S9D6Kl5ZXPtTqHxufPn/5+q3Z+n7kTek49riRxp0U4FEKjT0SJPHEOgRVSDwuLrrT/vElOi+M/kdji30FIy2GggMFKc8LJMjBWmeuzprttJPOKnkN2RzabF4HZ61oMx8YXirUxCV4f5qllvoVOBJc4qSRlx4t8AsY4WlADQp9v5otPUnWgzJIhsaFpymZqf//qEB5P1ZFcCqgc7/Ym4pv9gq1MJmG2/1KaFsSav40eFjKhEwyTSQZCIec5DgAcCfC7gk/BwecQm6NRv4Xw3EO98KgfYsOyLhfVQ5upISehGNH+e8pvWeEq2djoJBztpjqazja6GRbna3DzfZOd554nf1ga+wny9gftsN22QHrMc4su2Y37Da6ix7ilbj2ZI2j+Z9V9qLi5iPBc7ZY</latexit> ⇡ progressive sharpen stay on

    edge <latexit sha1_base64="105MajYxG6otBBhPa31G+ooUoLA=">AAACY3icfVBdaxNBFJ2sXzX1I62CDyIMBkFEwu42JvWtUB98ESuYttAJ4e7kbjp0dmaYuSsNS36Nr/qD/AH+D2fbFLSoBwYO557LmXsKp1WgNP3RSW7cvHX7zsbd7ua9+w8e9ra2D4OtvcSJtNr64wICamVwQoo0HjuPUBUaj4qz/XZ+9AV9UNZ8pqXDaQULo0olgaI06z0RBRIIcM7bc0GlB5mJqMx6/XTwdneUv8l5OkjTcb4zakk+HuY7PItKiz5b42C21RmKuZV1hYakhhBOstTRtAFPSmpcdUUd0IE8gwWeRGqgwjBtLi5Y8RdRmfPS+vgM8Qv1940GqhCWVRGdFdBpuD5rxb/OiupaMpW700YZVxMaeRlc1pqT5W09fK48StLLSEB6Ff/O5SnEViiW2O2KdxiP8/ghBn106IGsf9UI8ItKmVU8diFet+x/Rji/MkYWe74qk/+bHOaDbDQYfRr29/bXjW+wp+w5e8kyNmZ77D07YBMm2Yp9Zd/Y987PZDPZTh5fWpPOeucR+wPJs1+DDrpv</latexit> ⇡ 1 ⌘ [Cohen+ ] " Cohen, J., Kaur, S., Li, Y., Kolter, J., & Talwalkar, A. (2021). Gradient descent on neural networks typically occurs at the edge of stability.
  62. む " Cohen, J., Damian, A., Talwalker, A., Kolter, J.,

    & Lee, J. (2025). Understanding optimization in deep learning with central flows. <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t))
  63. む " Cohen, J., Damian, A., Talwalker, A., Kolter, J.,

    & Lee, J. (2025). Understanding optimization in deep learning with central flows. <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) む EoS <latexit sha1_base64="jY31QyIhqDocjVM8FcyAqM0t5uw=">AAACp3icfVFdaxNBFJ2sXzV+NNVHXwaDkIgNu6HUvgiF+uCD0oomDWRiuDs7ux06M7vM3K2GZX+cP8Nf4Kv+A2fTFdpUvDBwOOdc7r1n4kJJh2H4oxPcun3n7r2t+90HDx893u7tPJm6vLRcTHiucjuLwQkljZigRCVmhRWgYyVO4/OjRj+9ENbJ3HzGVSEWGjIjU8kBPbXszVlqgVcJ04BncUq/DnBYVwnW9A3dZQZiBfT94Ko63F23RGPmZKbhy9hzrfHTdeOy1w9H4broTRC1oE/aOlnudPZYkvNSC4NcgXPzKCxwUYFFyZWou6x0ogB+DpmYe2hAC7eo1inU9IVnEprm1j+DdM1e7ahAO7fSsXc2W7pNrSH/qcV6YzKmB4tKmqJEYfjl4LRUFHPaREwTaQVHtfIAuJV+d8rPwGeG/iO6XfZW+OOs+OAHHRfCAub2ZcXAZlqa2h+bsVcN+p8Rvv01euRzjjZTvQmm41G0P9r/uNc/PGoT3yLPyHMyIBF5TQ7JO3JCJoST7+Qn+UV+B8PgOJgGs0tr0Gl7npJrFcAfVMHRfg==</latexit> dw(t) dt = rL(w(t)) 1 2 2(t)rS(w(t))
  64. sharpness ( ) む " Cohen, J., Damian, A., Talwalker,

    A., Kolter, J., & Lee, J. (2025). Understanding optimization in deep learning with central flows. <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) む EoS <latexit sha1_base64="jY31QyIhqDocjVM8FcyAqM0t5uw=">AAACp3icfVFdaxNBFJ2sXzV+NNVHXwaDkIgNu6HUvgiF+uCD0oomDWRiuDs7ux06M7vM3K2GZX+cP8Nf4Kv+A2fTFdpUvDBwOOdc7r1n4kJJh2H4oxPcun3n7r2t+90HDx893u7tPJm6vLRcTHiucjuLwQkljZigRCVmhRWgYyVO4/OjRj+9ENbJ3HzGVSEWGjIjU8kBPbXszVlqgVcJ04BncUq/DnBYVwnW9A3dZQZiBfT94Ko63F23RGPmZKbhy9hzrfHTdeOy1w9H4broTRC1oE/aOlnudPZYkvNSC4NcgXPzKCxwUYFFyZWou6x0ogB+DpmYe2hAC7eo1inU9IVnEprm1j+DdM1e7ahAO7fSsXc2W7pNrSH/qcV6YzKmB4tKmqJEYfjl4LRUFHPaREwTaQVHtfIAuJV+d8rPwGeG/iO6XfZW+OOs+OAHHRfCAub2ZcXAZlqa2h+bsVcN+p8Rvv01euRzjjZTvQmm41G0P9r/uNc/PGoT3yLPyHMyIBF5TQ7JO3JCJoST7+Qn+UV+B8PgOJgGs0tr0Gl7npJrFcAfVMHRfg==</latexit> dw(t) dt = rL(w(t)) 1 2 2(t)rS(w(t))
  65. む : む む <latexit sha1_base64="Ogz5WUyidBxj6A6C8p6HFpWIfhI=">AAACgXicfZDNSuRAEMd74ve46uge99I4CK7IkIhf4EXQw15kFRwVzBAqPZXY2N0J3R11CHkSn8br+gS+jZ1xhHUUCwp+XfUvqusf54Ib6/svDW9icmp6ZnauOf9jYXGptbxyYbJCM+yyTGT6KgaDgivsWm4FXuUaQcYCL+Pbo7p/eYfa8Eyd20GOPQmp4glnYF0pau2EAlQqMJRgb+KkvK+ijc1BxN/fD5VjPZTQMK0TpISo1fY7/jDoZwhG0CajOI2WG9thP2OFRGWZAGOuAz+3vRK05Uxg1QwLgzmwW0jx2qECiaZXDu+r6Jqr9GmSaZfK0mH1/4kSpDEDGTtl/W0z3quLX/ZiObbZJvu9kqu8sKjY2+KkENRmtDaP9rlGZsXAATDN3d8puwENzDqLm83wGN1xGk/cor85arCZ3ihD0KnkqnLHpuFmTd8J4eFd6Mj5HIy7+hkutjrBbmf3bLt9eDRyfJb8IqtknQRkjxySP+SUdAkjj+SJ/CPP3oT32/O9rTep1xjN/CQfwjt4Be1Gxds=</latexit> hw⇤, yixi i <latexit

    sha1_base64="U94f3z6l97Xyl6EofN8Tlv7UmvY=">AAACVHicfZBNSwMxEIaz61et33r0EiyCiJRdEfUkBT14ERWsLbpFZtPpGppklyQrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxxJrixQfDm+ROTU9Mzpdny3PzC4tLyyuq1SXPNsM5SkepmDAYFV1i33ApsZhpBxgIbcfd40G88oDY8VVe2l2FLQqJ4hzOwTrqJEpAS6BEN7pYrQTUYFv0N4QgqZFQXdyveXtROWS5RWSbAmNswyGyrAG05E9gvR7nBDFgXErx1qECiaRXDlft00ylt2km1e8rSofr9RwHSmJ6MnVOCvTfjvYH4Zy+WY5Nt57BVcJXlFhX7HNzJBbUpHeRB21wjs6LnAJjmbnfK7kEDsy61cjk6QXecxjM36DxDDTbV20UEOpFc9d2xSbQzoP+M8PhldORyDsdT/Q3Xu9Vwv7p/uVepHY8SL5F1skG2SEgOSI2ckgtSJ4wo8kSeyYv36r37E/7Up9X3Rn/WyI/yFz8A2bq0Yw==</latexit> > 0 <latexit sha1_base64="nTN0VlqjikVD87chh4w6hftdmaA=">AAACnXicfZBNSxxBEIZ7x3zomo81Hj3YZAmsQZaZICYXQTCBHNQoZF3B3gw1vTVjY3fP0N2jLsP8rvyWHHJN/oY960qSNVjQ8HTVW1TVmxRSWBeGP1rBwqPHT54uLrWXnz1/8bKz8urE5qXhOOC5zM1pAhal0Dhwwkk8LQyCSiQOk4u9pj68RGNFrr+6SYEjBZkWqeDgfCruHO/3mAJ3nqTVVb2xw1IDPNLMliquxE5Uf9MMpewxCTqTSP9oN+kkFnff69qzmUo24k437IfToPchmkGXzOIoXmltsXHOS4XacQnWnkVh4UYVGCe4xLrNSosF8AvI8MyjBoV2VE1vr+kbnxnTNDf+aUen2b87KlDWTlTilc2ydr7WJP9bS9TcZJd+GFVCF6VDzW8Hp6WkLqeNsXQsDHInJx6AG+F3p/wcvJ3O299us4/ojzN44Ad9KdCAy83bioHJlNC1PzZjmw09JITrO6En73M07+p9OHnXj7b728db3d29meOLZI28Jj0Skfdkl3wmR2RAOPlOfpJf5HewHnwK9oPDW2nQmvWskn8iGN4AGmfRJg==</latexit> L(w) = 1 n n X i=1 `(hw, yixi i) <latexit sha1_base64="HT//l12ddZJZk79nPIE4VHhdBgA=">AAACZHicfZDdSiNBEIU7o66a1TUqgiDIYBASdw0zIuqNIOiFN6KCUcEJoaZTGRv7Z+juEeMQn8ZbfR9fwOewEyNolC1o+Dh1iuo6ccqZsUHwUvBGRsd+jU9MFn9PTf+ZKc3OnRuVaYp1qrjSlzEY5Exi3TLL8TLVCCLmeBHf7Pf6F7eoDVPyzHZSbAhIJGszCtZJzdJihJxX7qu7EVdJJfwb4V1aWb+vVpulclAL+uV/h3AAZTKok+ZsYTNqKZoJlJZyMOYqDFLbyEFbRjl2i1FmMAV6AwleOZQg0DTy/gldf9UpLb+ttHvS+n3180QOwpiOiJ1TgL02w72e+GMvFkObbXunkTOZZhYlfV/czrhvld/Lx28xjdTyjgOgmrm/+/QaNFDrUiwWowN0x2k8couOU9RglV7LI9CJYLLrjk2ifz36nxHuPoyOXM7hcKrf4XyjFm7Vtk43y3v7g8QnyBJZIRUSkm2yRw7JCakTSh7II3kiz4VXb8qb9xberV5hMDNPvpS3/AYhSbih</latexit> `(z) = log(1 + exp( z)) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
  66. 2 む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n

    X i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
  67. 2 む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n

    X i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) Edge of stability <latexit sha1_base64="7ljW2Zg0AIm/INKJh8e/dgAFxk4=">AAAClnicfVBdaxNBFJ2sXzV+NLUvBV8Gg5CKhl0p1RdLoYoiSiOYttCJy+zs3c2Qmdll5q5tWPY/+WsEn/SnOJumoKl4YeDcc8/lzjlJqaTDMPzRCa5dv3Hz1trt7p279+6v9zYeHLmisgLGolCFPUm4AyUNjFGigpPSAteJguNkdtDOj7+CdbIwn3FewkTz3MhMCo6einvvWWa5iJC5Ssf17FXUfKnxWdR8GDDNcZpk9VkTz7aZAsrOZAooVQr00PcZDi53rcynuB33+uEwXBS9CqIl6JNljeKNzg5LC1FpMCgUd+40Ckuc1NyiFAqaLqsclFzMeA6nHhquwU3qhemGPvZMSrPC+meQLtg/N2qunZvrxCtbJ2511pL/nCV65TJmLye1NGWFYMTF4axSFAvaJkpTaUGgmnvAhZX+71RMuU8Gfe7dLnsN3pyFj/7QYQmWY2Gf1IzbXEvTeLM5e9qi/wn5+aXQI59ztJrqVXD0fBjtDnc/7fT3D5aJr5GH5BEZkIi8IPvkHRmRMRHkG/lOfpJfwVawF7wJ3l5Ig85yZ5P8VcHoN/+WzZU=</latexit> 1 t t 1 X k=1 L(wk)  e O ✓ 1 t ◆ Phase " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
  68. 2 む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n

    X i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
  69. む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n X

    i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) 2 GD <latexit sha1_base64="uVKnhPBYagm5kR5Bq1HUTeAdM2o=">AAACg3icfVDRahNBFJ2s1ta02lQffRkahFQl7IbQ9kUo1AcflFZI2kInhtnZu5uhM7PLzF1rWPZT+jV91Q/wb5xNU9BUvDBw7rnncuecuFDSYRj+agWPHq89Wd942t7cevZ8u7Pz4szlpRUwFrnK7UXMHShpYIwSFVwUFriOFZzHV8fN/PwbWCdzM8J5ARPNMyNTKTh6ato5+NRjmuMsTqvrejraYwoou5YJoFQJ0BPfp9hjqeUiqkZfBzWzMpvh3rTTDfvhouhDEC1BlyzrdLrTGrIkF6UGg0Jx5y6jsMBJxS1KoaBus9JBwcUVz+DSQ8M1uEm1cFjT155JaJpb/wzSBfvnRsW1c3Mde2Xjxq3OGvKfs1ivXMb0cFJJU5QIRtwdTktFMadNfDSRFgSquQdcWOn/TsWM+3TQh9xusw/gzVn47A+dFGA55vZNxbjNtDS1N5uxdw36n5B/vxd65HOOVlN9CM4G/Wi/v/9l2D06Xia+QV6RXdIjETkgR+QjOSVjIsgNuSU/yM9gLXgbDILhnTRoLXdekr8qeP8bawPFgg==</latexit> L(wT )  e O ✓ 1 T2 ◆ Phase " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
  70. む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n X

    i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) 2 " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss. EoS stable
  71. む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n X

    i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) 2 <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws)  min ⇢ 1 C⌘ , `(0) n " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss. EoS stable
  72. む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n X

    i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws)  min ⇢ 1 C⌘ , `(0) n <latexit sha1_base64="93f0ruv9t2WMuP6IyGCRk2Keak8=">AAACd3icfVBNS+RAEO2J37Orjnr0YLPDLqIyJCLqSQQ97EVU2FHBDKHSU4mN3Z3Q3VHHkB+xv2av+jP8KXuzM46wOy5b0PD6vVdU1YtzwY31/ZeGNzE5NT0zO9f89Hl+YbG1tHxhskIz7LJMZPoqBoOCK+xabgVe5RpBxgIv49ujWr+8Q214pn7YQY49CaniCWdgHRW1NkMBKhUYSrA3cVLeV5HZev88VhEP9VCnB9SPWm2/4w+LfgTBCLTJqM6ipcZO2M9YIVFZJsCY68DPba8EbTkTWDXDwmAO7BZSvHZQgUTTK4dXVfSrY/o0ybR7ytIh+2dHCdKYgYyds97XjGs1+U8tlmOTbbLfK7nKC4uKvQ1OCkFtRuvIaJ9rZFYMHACmududshvQwKwLttkMj9Edp/HEDTrNUYPN9EYZgk4lV5U7Ng23avQ/Izy8Gx1yOQfjqX4EF9udYLeze77TPjwaJT5LVskXsk4CskcOyXdyRrqEkZ/kF3kiz43f3pr3zVt/s3qNUc8K+au84BUs4cJn</latexit> hws, zi i > 0 EoS stable " Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.
  73. む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n X

    i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws)  min ⇢ 1 C⌘ , `(0) n <latexit sha1_base64="93f0ruv9t2WMuP6IyGCRk2Keak8=">AAACd3icfVBNS+RAEO2J37Orjnr0YLPDLqIyJCLqSQQ97EVU2FHBDKHSU4mN3Z3Q3VHHkB+xv2av+jP8KXuzM46wOy5b0PD6vVdU1YtzwY31/ZeGNzE5NT0zO9f89Hl+YbG1tHxhskIz7LJMZPoqBoOCK+xabgVe5RpBxgIv49ujWr+8Q214pn7YQY49CaniCWdgHRW1NkMBKhUYSrA3cVLeV5HZev88VhEP9VCnB9SPWm2/4w+LfgTBCLTJqM6ipcZO2M9YIVFZJsCY68DPba8EbTkTWDXDwmAO7BZSvHZQgUTTK4dXVfSrY/o0ybR7ytIh+2dHCdKYgYyds97XjGs1+U8tlmOTbbLfK7nKC4uKvQ1OCkFtRuvIaJ9rZFYMHACmududshvQwKwLttkMj9Edp/HEDTrNUYPN9EYZgk4lV5U7Ng23avQ/Izy8Gx1yOQfjqX4EF9udYLeze77TPjwaJT5LVskXsk4CskcOyXdyRrqEkZ/kF3kiz43f3pr3zVt/s3qNUc8K+au84BUs4cJn</latexit> hws, zi i > 0 EoS stable 2 " Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.
  74. む む <latexit sha1_base64="Agi/AHyJKsd43sWC36qH113tREc=">AAACmnicfZBdSxwxFIaz0y/dfrjWS3sRuhTWIstMEdsbQdQLi7a14KpgtsOZ7JkxmGSGJNN2O8y/8s/Y2/pDzKzbr7X0QODhvG8457xJIYV1YXjZCu7cvXf/wdx8++Gjx08WOotPj2xeGo4DnsvcnCRgUQqNAyecxJPCIKhE4nFyvt3ox5/RWJHrQzcucKgg0yIVHJxvxZ33+z2mwJ0lafWlXtlgqQEeaWZLFVdiI6o/aYZS9pgEnUmkv72rv/hbHQtmJvpK3OmG/XBS9DZEU+iSaR3Ei601Nsp5qVA7LsHa0ygs3LAC4wSXWLdZabEAfg4ZnnrUoNAOq8nhNX3hOyOa5sY/7eik++ePCpS1Y5V4Z7OsndWa5j+1RM1MdumbYSV0UTrU/GZwWkrqctqkSkfCIHdy7AG4EX53ys/AZ+l89u0220F/nMF3ftCHAg243LysGJhMCV37YzO22tD/jPD1p9GTzzmaTfU2HL3qR+v99Y9r3c3taeJzZJk8Jz0Skddkk+ySAzIgnFyQ7+QHuQqeBVvB22Dvxhq0pn+WyF8VHF4D7kfPyQ==</latexit> L(w) = 1 n n X

    i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws)  min ⇢ 1 C⌘ , `(0) n <latexit sha1_base64="93f0ruv9t2WMuP6IyGCRk2Keak8=">AAACd3icfVBNS+RAEO2J37Orjnr0YLPDLqIyJCLqSQQ97EVU2FHBDKHSU4mN3Z3Q3VHHkB+xv2av+jP8KXuzM46wOy5b0PD6vVdU1YtzwY31/ZeGNzE5NT0zO9f89Hl+YbG1tHxhskIz7LJMZPoqBoOCK+xabgVe5RpBxgIv49ujWr+8Q214pn7YQY49CaniCWdgHRW1NkMBKhUYSrA3cVLeV5HZev88VhEP9VCnB9SPWm2/4w+LfgTBCLTJqM6ipcZO2M9YIVFZJsCY68DPba8EbTkTWDXDwmAO7BZSvHZQgUTTK4dXVfSrY/o0ybR7ytIh+2dHCdKYgYyds97XjGs1+U8tlmOTbbLfK7nKC4uKvQ1OCkFtRuvIaJ9rZFYMHACmududshvQwKwLttkMj9Edp/HEDTrNUYPN9EYZgk4lV5U7Ng23avQ/Izy8Gx1yOQfjqX4EF9udYLeze77TPjwaJT5LVskXsk4CskcOyXdyRrqEkZ/kF3kiz43f3pr3zVt/s3qNUc8K+au84BUs4cJn</latexit> hws, zi i > 0 EoS stable 1 2 " Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.
  75. faster " Bao, H., Sakaue, S., & Takezawa, Y. (2025).

    Any-stepsize gradient descent for separable data under Fenchel–Young losses.
  76. EoS stable EoS " Bao, H., Sakaue, S., & Takezawa,

    Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.