wijsisj X i ✓isi Input <latexit sha1_base64="EmPRd3f6t2Tl+5EEWNhjanjozwc=">AAACsnicfVFNb9QwEPWGrzZ8bdsjF4stCAFdJWjVcqxUDlwQRWLbinoVOd5J1lvbiewJ7SrKP+wf4G9wLQecbRCwRYxk6em9NzP2c1oq6TCKvvWCW7fv3L23th7ef/Dw0eP+xuaRKyorYCwKVdiTlDtQ0sAYJSo4KS1wnSo4Ts8OWv34K1gnC/MZFyVMNM+NzKTg6Kmkn7lEshzQsRRyaWrhZ7kmfBXT55QhXGAtM7rNXKWT+XlSy3njkrlvYDgD5Incbihj4c5ve+EFey4dNCEDM+0GJv1BNIyWRW+CuAMD0tVhstEbsWkhKg0GheLOncZRiZOaW5RCtcMrByUXZzyHUw8N1+Am9TKQhj7zzJRmhfXHIF2yf3bUXDu30Kl3ao4zt6q15D+1VK9sxuztpJamrBCMuF6cVYpiQdu06VRaEKgWHnBhpb87FTNuuUD/J2HI3oF/nIUPftHHEizHwr6sGbe5lqbxj83Z6xb9z8gvfhk98jnHq6neBEdvhvHucPfTaLB/0CW+Rp6Qp+QFicke2SfvySEZE0EuyXdyRX4Eo+BLwANxbQ16Xc8W+asC9RMR+djl</latexit> si ( +1 if P j wijsj ✓i 1 otherwise
wijsisj X i ✓isi Input <latexit sha1_base64="EmPRd3f6t2Tl+5EEWNhjanjozwc=">AAACsnicfVFNb9QwEPWGrzZ8bdsjF4stCAFdJWjVcqxUDlwQRWLbinoVOd5J1lvbiewJ7SrKP+wf4G9wLQecbRCwRYxk6em9NzP2c1oq6TCKvvWCW7fv3L23th7ef/Dw0eP+xuaRKyorYCwKVdiTlDtQ0sAYJSo4KS1wnSo4Ts8OWv34K1gnC/MZFyVMNM+NzKTg6Kmkn7lEshzQsRRyaWrhZ7kmfBXT55QhXGAtM7rNXKWT+XlSy3njkrlvYDgD5Incbihj4c5ve+EFey4dNCEDM+0GJv1BNIyWRW+CuAMD0tVhstEbsWkhKg0GheLOncZRiZOaW5RCtcMrByUXZzyHUw8N1+Am9TKQhj7zzJRmhfXHIF2yf3bUXDu30Kl3ao4zt6q15D+1VK9sxuztpJamrBCMuF6cVYpiQdu06VRaEKgWHnBhpb87FTNuuUD/J2HI3oF/nIUPftHHEizHwr6sGbe5lqbxj83Z6xb9z8gvfhk98jnHq6neBEdvhvHucPfTaLB/0CW+Rp6Qp+QFicke2SfvySEZE0EuyXdyRX4Eo+BLwANxbQ16Xc8W+asC9RMR+djl</latexit> si ( +1 if P j wijsj ✓i 1 otherwise Output
wijsisj X i ✓isi Space of <latexit sha1_base64="SxYxau1idJ47yfvCqZ9fy5fQN9I=">AAACVHicfZBNSwMxEIazWz/rV6tHL4tFEJGyK6V6LOjBi6hgVXRLmU2nNTbJLklWLcv+C6/6swT/iwfTWkGrOBB4eOcNM/NGCWfa+P6b4xampmdm5+aLC4tLyyul8uqFjlNFsUljHqurCDRyJrFpmOF4lSgEEXG8jPoHw/7lPSrNYnluBgm2BPQk6zIKxkrXYfbQzthdHubtUsWv+qPyfkMwhgoZ12m77NTCTkxTgdJQDlrfBH5iWhkowyjHvBimGhOgfejhjUUJAnUrG62ce5tW6XjdWNknjTdSv//IQGg9EJF1CjC3erI3FP/sRWJisunutzImk9SgpJ+Duyn3TOwN8/A6TCE1fGABqGJ2d4/eggJqbGrFYniI9jiFx3bQSYIKTKy2sxBUTzCZ22N74c6Q/jPC45fRks05mEz1N1zsVoN6tX5WqzQOxonPkXWyQbZIQPZIgxyRU9IklEjyRJ7Ji/PqvLsFd/rT6jrjP2vkR7nLH0Untac=</latexit> {wij } Energy
wijsisj X i ✓isi Space of <latexit sha1_base64="SxYxau1idJ47yfvCqZ9fy5fQN9I=">AAACVHicfZBNSwMxEIazWz/rV6tHL4tFEJGyK6V6LOjBi6hgVXRLmU2nNTbJLklWLcv+C6/6swT/iwfTWkGrOBB4eOcNM/NGCWfa+P6b4xampmdm5+aLC4tLyyul8uqFjlNFsUljHqurCDRyJrFpmOF4lSgEEXG8jPoHw/7lPSrNYnluBgm2BPQk6zIKxkrXYfbQzthdHubtUsWv+qPyfkMwhgoZ12m77NTCTkxTgdJQDlrfBH5iWhkowyjHvBimGhOgfejhjUUJAnUrG62ce5tW6XjdWNknjTdSv//IQGg9EJF1CjC3erI3FP/sRWJisunutzImk9SgpJ+Duyn3TOwN8/A6TCE1fGABqGJ2d4/eggJqbGrFYniI9jiFx3bQSYIKTKy2sxBUTzCZ22N74c6Q/jPC45fRks05mEz1N1zsVoN6tX5WqzQOxonPkXWyQbZIQPZIgxyRU9IklEjyRJ7Ji/PqvLsFd/rT6jrjP2vkR7nLH0Untac=</latexit> {wij } Energy <latexit sha1_base64="jbxw4VlEsrOjJcyoYcGCaVKKfZg=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hobaFbymw6XUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkZmbn5vMLi4Wl5ZXVtfXixo2OU0WxTmMeq2YIGjmTWDfMcGwmCkGEHBthvzbqN+5QaRbLazNIsC0gkqzHKBgrXTU6Xme95JW9cbm/wZ9AiUzqolN0KkE3pqlAaSgHrVu+l5h2BsowynFYCFKNCdA+RNiyKEGgbmfjXYfujlW6bi9W9knjjtXvPzIQWg9EaJ0CzK2e7o3EP3uhmJpsesftjMkkNSjp5+Beyl0Tu6Mg3C5TSA0fWACqmN3dpbeggBobV6EQnKA9TuGZHXSeoAITq70sABUJJof22CjYH9F/Rrj/MlqyOfvTqf6Gm4Oyf1g+vKyUqrVJ4gtki2yTXeKTI1Ilp+SC1AklEXkgj+TJeXZenTfn/dOacyZ/NsmPyuU/APCEs/Q=</latexit> W0 <latexit sha1_base64="7QfmSbRU9jUZ7w5eHyWAh0pTY30=">AAACTXicfZBNSwMxEIaz9aNaP6tHL4tFEJGyK0U9FurBi6hobaFbymw6XUOT7JJkxbL0J3jVn+XZH+JNxLRW0CoOBB7eecPMvGHCmTae9+LkZmbn5vMLi4Wl5ZXVtfXixo2OU0WxTmMeq2YIGjmTWDfMcGwmCkGEHBthvzbqN+5QaRbLazNIsC0gkqzHKBgrXTU6fme95JW9cbm/wZ9AiUzqolN0KkE3pqlAaSgHrVu+l5h2BsowynFYCFKNCdA+RNiyKEGgbmfjXYfujlW6bi9W9knjjtXvPzIQWg9EaJ0CzK2e7o3EP3uhmJpsesftjMkkNSjp5+Beyl0Tu6Mg3C5TSA0fWACqmN3dpbeggBobV6EQnKA9TuGZHXSeoAITq70sABUJJof22CjYH9F/Rrj/MlqyOfvTqf6Gm4Oyf1g+vKyUqrVJ4gtki2yTXeKTI1Ilp+SC1AklEXkgj+TJeXZenTfn/dOacyZ/NsmPyuU/APJrs/U=</latexit> W1 <latexit sha1_base64="TWCE2cASg/NetlRic2wlKMwAIRI=">AAACTXicfZBNSwMxEIaz9aNav1o9elksgoiU3SLqsaAHL6KitUK3lNl0uoYm2SXJimXpT/CqP8uzP8SbiGmtoK04EHh45w0z84YJZ9p43quTm5mdm88vLBaWlldW14ql9Rsdp4pincY8VrchaORMYt0ww/E2UQgi5NgIe8fDfuMelWaxvDb9BFsCIsm6jIKx0lWjXW0Xy17FG5U7Df4YymRcF+2Ssx90YpoKlIZy0Lrpe4lpZaAMoxwHhSDVmADtQYRNixIE6lY22nXgblul43ZjZZ807kj9+SMDoXVfhNYpwNzpyd5Q/LMXionJpnvUyphMUoOSfg3uptw1sTsMwu0whdTwvgWgitndXXoHCqixcRUKwQna4xSe2UHnCSowsdrNAlCRYHJgj42CvSH9Z4SHb6Mlm7M/meo03FQr/kHl4HK/XDseJ75ANskW2SE+OSQ1ckouSJ1QEpFH8kSenRfnzXl3Pr6sOWf8Z4P8qlz+E/RSs/Y=</latexit> W2 attractor <latexit sha1_base64="P+BWQ+cxOm67fTzjf3D/m22OLUM=">AAACTXicfZBNSwMxEIaz9avWr6pHL4tFkCJlV6R6FPTgRaxoW8EtZTadrsEkuyRZsSz9CV71Z3n2h3gTMf0QtBUHAg/vvGFm3jDhTBvPe3NyM7Nz8wv5xcLS8srqWnF9o6HjVFGs05jH6iYEjZxJrBtmON4kCkGEHJvh/cmg33xApVksr00vwZaASLIuo2CsdNVsl9vFklfxhuVOgz+GEhlXrb3uHASdmKYCpaEctL71vcS0MlCGUY79QpBqTIDeQ4S3FiUI1K1suGvf3bFKx+3Gyj5p3KH680cGQuueCK1TgLnTk72B+GcvFBOTTfeolTGZpAYlHQ3uptw1sTsIwu0whdTwngWgitndXXoHCqixcRUKwSna4xSe20EXCSowsSpnAahIMNm3x0bB3oD+M8Ljt9GSzdmfTHUaGvsVv1qpXh6Ujk/GiefJFtkmu8Qnh+SYnJEaqRNKIvJEnsmL8+q8Ox/O58iac8Z/Nsmvyi18AeUas+4=</latexit> W⇤
W3 (W2 (W1x)) <latexit sha1_base64="6TapuefMkgmpaPP+inRJtlMT8zc=">AAACinicfZDfaxQxEMdza9X2Wu1VH/sSehRE9NitpSqCFFqhL8UKvV6hex6zudltbJJdklnxWPa/8a/pa33xvzF7vfrjKh0IfDLznUzmmxRKOgrDn63g3sL9Bw8Xl9rLK48er3bWnpy4vLQC+yJXuT1NwKGSBvskSeFpYRF0onCQXOw19cFXtE7m5pgmBQ41ZEamUgD51KjzfvD51aiSX2oeZ0iO/76+5HFqQVRxAZYkKP6h/sM3qnrU6Ya9cBr8NkQz6LJZHI3WWtvxOBelRkNCgXNnUVjQsGoeFgrrdlw6LEBcQIZnHg1odMNqumjNN31mzNPc+mOIT7N/d1SgnZvoxCs10LmbrzXJ/9YSPTeZ0jfDSpqiJDTienBaKk45b1zkY2lRkJp4AGGl/zsX5+D9Iu91ux3vo1/O4qEf9LFAC5Tb51UMNtPS1H7ZLH7R0F1C+HYj9OR9juZdvQ0nW71op7fzabu7uzdzfJGtsw32jEXsNdtlB+yI9Zlg39klu2I/gpVgK3gbvLuWBq1Zz1P2TwT7vwAsPciC</latexit> W3 ij W3 ij @E @W3 ij <latexit sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> W1 ij W1 ij @E @W1 ij
sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> W1 ij W1 ij @E @W1 ij <latexit sha1_base64="16tW0XfXb2RwDG2mlIPN3uCIsSE=">AAACS3icfZDbSgMxEIaz1Xqop6qX3iwWQUTKrkj1UlDBG7EFe4BuKbPptAaT7JJkxbL0CbzVx/IBfA7vxAvTg6CtOBD4+OcPM/OHMWfaeN6bk5mbzy4sLi3nVlbX1jfym1s1HSWKYpVGPFKNEDRyJrFqmOHYiBWCCDnWw/vzYb/+gEqzSN6afowtAT3JuoyCsVLlsp0veEVvVO4s+BMokEmV25vOcdCJaCJQGspB66bvxaaVgjKMchzkgkRjDPQeeti0KEGgbqWjTQfunlU6bjdS9knjjtSfP1IQWvdFaJ0CzJ2e7g3FP3uhmJpsuqetlMk4MSjpeHA34a6J3GEMbocppIb3LQBVzO7u0jtQQI0NK5cLLtAep/DaDrqJUYGJ1EEagOoJJgf22F5wOKT/jPD4bbRkc/anU52F2lHRLxVLlePC2fkk8SWyQ3bJPvHJCTkjV6RMqoQSJE/kmbw4r8678+F8jq0ZZ/Jnm/yqTPYLcpGzPw==</latexit> E
sha1_base64="kTU9AoqSWteK4Y2ZuxL6LF4weE8=">AAACfXicfVBdSxtBFJ1sbWvTD2P72JfBIMSSht0g2peCYAt9KSo0KrjpcndyNw7OzC4zd4vLsr+jv8ZX+xv6a9pJjKCx9MLA4ZxzOXNPWijpKAx/t4JHK4+fPF191n7+4uWrtc7662OXl1bgSOQqt6cpOFTS4IgkKTwtLIJOFZ6kF/sz/eQHWidz842qAscapkZmUgB5KulEn3tVv0rqmPCSarIlNs0W/8jjzIKIhr3qfZXc0ba+D5NONxyE8+EPQbQAXbaYw2S9tR1PclFqNCQUOHcWhQWNa7AkhcKmHZcOCxAXMMUzDw1odON6flvDNz0z4Vlu/TPE5+zdjRq0c5VOvVMDnbtlbUb+U0v1UjJlH8a1NEVJaMRNcFYqTjmfFccn0qIgVXkAwkr/dy7OwbdEvt52O/6E/jiLX33QQYEWKLfv6hjsVEvT+GOncX+G/meEy1ujR77naLnVh+B4OIh2BjtH2929/UXjq+wt22A9FrFdtse+sEM2YoL9ZFfsmv1q/Qk2g34wuLEGrcXOG3Zvgt2/7RTEAg==</latexit> E(y, ytrue) = 1 2 (y ytrue)2 <latexit sha1_base64="Sa08x10wlIIJd3XUxZR/R5ysQ1k=">AAACinicfZBRaxNBEMc3Z7U1rZrqoy+LQRDRcBdKqwhSaIW+FFtomkIvhrnN3HXt7t6xOyeG476Nn8ZX+9Jv416aVk2lAwu/nfnPzs4/KZR0FIaXreDe0v0HyysP26trjx4/6aw/PXZ5aQUORK5ye5KAQyUNDkiSwpPCIuhE4TA532nqw29onczNEU0LHGnIjEylAPKpcefj8Et/XMmvNY8zJMdvrm95nFoQVVyAJQmKf6r/8LWqHne6YS+cBb8N0Ry6bB4H4/XWRjzJRanRkFDg3GkUFjSqmoeFwrodlw4LEOeQ4alHAxrdqJotWvOXPjPhaW79McRn2b87KtDOTXXilRrozC3WmuR/a4lemEzpu1ElTVESGnE1OC0Vp5w3LvKJtChITT2AsNL/nYsz8H6R97rdjnfRL2dx3w/6XKAFyu3rKgabaWlqv2wWv2noLiF8vxZ68j5Hi67ehuN+L9rsbR5udLd35o6vsOfsBXvFIrbFttkeO2ADJtgP9pP9YhfBWtAP3gcfrqRBa97zjP0Twe5vJhvIfw==</latexit> W2 ij W2 ij @E @W2 ij <latexit sha1_base64="aRbCJv6ipmy6bOFjrDoBynyD32o=">AAACinicfZDfaxQxEMdz22rrWfWqj76EHoKIHrul9AdCKbSCL9IKXq/QPY/Z3Ow2NskuyWzpsex/41/jq33pf2P2eq16FQcCn8x8J5P5JoWSjsLwuhUsLD54uLT8qP145cnTZ53V58cuL63AvshVbk8ScKikwT5JUnhSWASdKBwk5/tNfXCB1sncfKFJgUMNmZGpFEA+NersDr5Go0p+q3mcITl+d33H49SCqOICLElQ/EP9m29V9ajTDXvhNPh9iGbQZbM4Gq22NuJxLkqNhoQC506jsKBh1TwsFNbtuHRYgDiHDE89GtDohtV00Zq/8pkxT3PrjyE+zf7ZUYF2bqITr9RAZ26+1iT/WUv03GRKt4eVNEVJaMTN4LRUnHLeuMjH0qIgNfEAwkr/dy7OwPtF3ut2Oz5Av5zFT37QYYEWKLdvqhhspqWp/bJZ/Lah/wnh8lboyfsczbt6H47Xe9Fmb/PzRndvf+b4MnvJ1thrFrEttsc+siPWZ4J9Zz/YT3YVrATrwU7w/kYatGY9L9hfERz8Ah/5yHw=</latexit> W1 ij W1 ij @E @W1 ij <latexit sha1_base64="16tW0XfXb2RwDG2mlIPN3uCIsSE=">AAACS3icfZDbSgMxEIaz1Xqop6qX3iwWQUTKrkj1UlDBG7EFe4BuKbPptAaT7JJkxbL0CbzVx/IBfA7vxAvTg6CtOBD4+OcPM/OHMWfaeN6bk5mbzy4sLi3nVlbX1jfym1s1HSWKYpVGPFKNEDRyJrFqmOHYiBWCCDnWw/vzYb/+gEqzSN6afowtAT3JuoyCsVLlsp0veEVvVO4s+BMokEmV25vOcdCJaCJQGspB66bvxaaVgjKMchzkgkRjDPQeeti0KEGgbqWjTQfunlU6bjdS9knjjtSfP1IQWvdFaJ0CzJ2e7g3FP3uhmJpsuqetlMk4MSjpeHA34a6J3GEMbocppIb3LQBVzO7u0jtQQI0NK5cLLtAep/DaDrqJUYGJ1EEagOoJJgf22F5wOKT/jPD4bbRkc/anU52F2lHRLxVLlePC2fkk8SWyQ3bJPvHJCTkjV6RMqoQSJE/kmbw4r8678+F8jq0ZZ/Jnm/yqTPYLcpGzPw==</latexit> E <latexit sha1_base64="Flm7yEYkdgFO72bE/MGYwVd2u/g=">AAACZnicfZDRahNBFIYna9UataaWUtCbwSC0UsOuhOhNoVAL3hQrmKbQDeHs7Nlk6MzsMnO2NCwLfZre1tfxDXwMJ2kETYsHBj7+8x/OnD8plHQUhj8bwYOVh48erz5pPn32fO1Fa/3lictLK7AvcpXb0wQcKmmwT5IUnhYWQScKB8n5waw/uEDrZG6+07TAoYaxkZkUQF4atV7FmQVRpYO6Sqneex8bSBTww+3BzqjVDjvhvPhdiBbQZos6Hq03unGai1KjIaHAubMoLGhYgSUpFNbNuHRYgDiHMZ55NKDRDav5ETV/65WUZ7n1zxCfq39PVKCdm+rEOzXQxC33ZuK9vUQvbabs07CSpigJjbhdnJWKU85nCfFUWhSkph5AWOn/zsUEfErkc2w248/oj7N45Bd9LdAC5fZdFYMda2lqf+w43p3R/4xw+cfoyeccLad6F04+dKJep/et294/WCS+yl6zN2ybRewj22df2DHrM8Gu2DW7YT8av4K1YDPYurUGjcXMBvunAv4b4iC6dA==</latexit> dW dt = rE(W)
solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 <latexit sha1_base64="kHwHKp8xeEs1XIvKfO8vZURGGRQ=">AAACgnicfVBdaxpBFB03trH2I6Z57MtSKaSSyq4G04cEBFvoS6mFGgVXl7vj1QzOzC4zs0HZ7D/pr+lr8gf6bzp+FFotvTDM4ZxzufeeKOFMG8/7WXAOio8eH5aelJ8+e/7iqHL88lrHqaLYozGP1SACjZxJ7BlmOA4ShSAijv1o3lnp/VtUmsXym1kmOBIwk2zKKBhLhZXWx9P+OGs28jP7Nfz87VWgUxFm7MrPxzK4W4bs3caw0RchC+7GjbBS9ereutx94G9BlWyrGx4XzoNJTFOB0lAOWg99LzGjDJRhlGNeDlKNCdA5zHBooQSBepStD8zdN5aZuNNY2SeNu2b/7MhAaL0UkXUKMDd6V1uR/9QisTPZTN+PMiaT1KCkm8HTlLsmdlfpuROmkBq+tACoYnZ3l96AAmpsxuVy8AHtcQo/20FfElRgYlXLAlAzwWRuj50FZyv0PyMsfhstsjn7u6nug+tG3W/VW1/Pq+3ONvESeUVek1PikwvSJp9Il/QIJd/JD3JPHpyiU3N8p7mxOoVtzwn5q5zLX3IRxB0=</latexit> E(W32, W21) = n X i=1 kyi W32W21xi k2 <latexit sha1_base64="Wujxj6EHytPJuuuiEeDDmEToTTk=">AAADRHicjVJNi9RAEO3ErzV+zerRS+MgzIg7JLKsXoSF9eBFXMHZXdgeQ6VTk2k23QndHXEIuXnVn+V/8D94E8GT2JNNZJ0VtSDw+tWrfl1VScpcGBuGnz3/wsVLl69sXA2uXb9x89Zg8/aBKSrNccqLvNBHCRjMhcKpFTbHo1IjyCTHw+Rkb5U/fIvaiEK9tssSZxIyJeaCg3VUPPjOEsyEqrm7wzQBm2vgdQoxg7xcQFOntnk6Mt1xq+dp0oFxD+gWZaaScc0ykBKYwu4GJ2iZ0dnalhlTxnrD5L8N4V+G0Bv+elrPjAOGKu1ajQfDcBK2Qc+DqAND0sV+vOlts7TglURleQ7GHEdhaWc1aCt4jm52lcES+AlkeOygAolmVrcrauh9x6R0Xmj3KUtb9mxFDdKYpUycUoJdmPXcivxjLpFrznb+ZFYLVVYWFT81nlc5tQVd7Z+mQiO3+dIB4Fq4t1O+ALcE6/6SIGDP0DWn8YUzelmiBlvoBzUDnUmhGtdsxh6u0N+E8K4XOuTmHK1P9Tw4eDSJdiY7r7aHu3vdxDfIXXKPjEhEHpNd8pzskynh3hvvvffB++h/8r/4X/1vp1Lf62rukN/C//ET+zoRyQ==</latexit> ( da↵ dt = (s↵ a↵b↵)b↵ P 6=↵ b (a↵b ) db↵ dt = (s↵ a↵b↵)a↵ P 6=↵ a (b↵a )
Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 #
Saxe, A., McClelland, J., & Ganguli, S. (2014). Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. <latexit sha1_base64="2oa+EUQF+8zDjmnZ6Nf+98VygGw=">AAACS3icfZDPSgMxEMaz9X/91+rRy2IRRKTsSqkeC/XgRVSwtdAtMptOazDJLklWWpY+gVd9LB/A5/AmHkzrCtqKA4Ef33xhZr4w5kwbz3t1cnPzC4tLyyv51bX1jc1Ccaupo0RRbNCIR6oVgkbOJDYMMxxbsUIQIceb8L4+7t88oNIsktdmGGNHQF+yHqNgrHQ1uC2UvLI3KXcW/AxKJKvL26JTCboRTQRKQzlo3fa92HRSUIZRjqN8kGiMgd5DH9sWJQjUnXSy6cjds0rX7UXKPmncifrzRwpC66EIrVOAudPTvbH4Zy8UU5NN76STMhknBiX9GtxLuGsidxyD22UKqeFDC0AVs7u79A4UUGPDyueDU7THKTy3gy5iVGAidZAGoPqCyZE9th8cjuk/Iwy+jZZszv50qrPQPCr71XL1qlKq1bPEl8kO2SX7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc052Z9t8qtyC5/TlrNy</latexit> x <latexit sha1_base64="KgDhIumMmgJR5Go00Yuc+VxfdEU=">AAACS3icfZDPSgMxEMaz1Wqtf6tHL4tFEJGyK0U9FurBi6hgreCWMptO29AkuyRZsSx9Aq/6WD6Az+FNPJitFbQVBwI/vvnCzHxhzJk2nvfq5Obm8wuLhaXi8srq2vpGafNGR4mi2KARj9RtCBo5k9gwzHC8jRWCCDk2w0E96zfvUWkWyWszjLEloCdZl1EwVroatjfKXsUblzsL/gTKZFKX7ZJTDToRTQRKQzlofed7sWmloAyjHEfFINEYAx1AD+8sShCoW+l405G7a5WO242UfdK4Y/XnjxSE1kMRWqcA09fTvUz8sxeKqcmme9JKmYwTg5J+De4m3DWRm8XgdphCavjQAlDF7O4u7YMCamxYxWJwivY4hed20EWMCkyk9tMAVE8wObLH9oKDjP4zwsO30ZLN2Z9OdRZuDiv+UeXoqlqu1SeJF8g22SF7xCfHpEbOyCVpEEqQPJIn8uy8OG/Ou/PxZc05kz9b5Ffl8p/VfbNz</latexit> y <latexit sha1_base64="ZV+Uu1RpUfkG1TV/vY1UBfCpr9A=">AAACW3icfVBNSwMxEE3X7/WrKp68LBZBRMpuFfUiCHrwIlawH+DWMptOazDJLklWLMv+Eq/6ozz4X0xrBW3FgYTHmze8mRclnGnj++8FZ2p6ZnZufsFdXFpeWS2urdd1nCqKNRrzWDUj0MiZxJphhmMzUQgi4tiIHs8H/cYTKs1ieWv6CbYE9CTrMgrGUu3iav+0cZ8dVHL7V4L8uV0s+WV/WN4kCEagREZVba8VDsNOTFOB0lAOWt8FfmJaGSjDKMfcDVONCdBH6OGdhRIE6lY23Dz3dizT8bqxsk8ab8j+nMhAaN0XkVUKMA96vDcg/+xFYszZdE9aGZNJalDSL+Nuyj0Te4NYvA5TSA3vWwBUMbu7Rx9AATU2PNcNL9Aep/DKGl0nqMDEai8LQfUEk7k9thfuD9B/Qnj+Flpkcw7GU50E9Uo5OCof3RyWzs5Hic+TLbJNdklAjskZuSRVUiOUpOSFvJK3wocz5bjO0pfUKYxmNsivcjY/AVWStgo=</latexit> y = W32W21x <latexit sha1_base64="E9AJox7CytGn6g24b3PvgSNCYm0=">AAACUHicfZBNSwMxEIZn6/f6rUcvi0UQkbIroh4FPXgRFWwruFVm02mNJtklyYpl6X/wqj/Lm//Em6a1glZxIOThnTfM5E0ywY0Nw1evNDI6Nj4xOeVPz8zOzS8sLtVMmmtGVZaKVF8kaEhwRVXLraCLTBPKRFA9uTvo9ev3pA1P1bntZNSQ2Fa8xRlaJ9XqV8VW1L1eKIeVsF/Bb4gGUIZBnV4vettxM2W5JGWZQGMuozCzjQK15UxQ149zQxmyO2zTpUOFkkyj6K/bDdac0gxaqXZH2aCvfn9RoDSmIxPnlGhvzHCvJ/7ZS+TQZNvaaxRcZbklxT4Ht3IR2DToZRE0uSZmRccBMs3d7gG7QY3MusR8Pz4k9zlNx27QSUYabao3ihh1W3LVdZ9tx5s9+s+ID19GRy7naDjV31DbqkQ7lZ2z7fL+wSDxSViBVViHCHZhH47gFKrA4BYe4QmevRfvzXsveZ/WrxuW4UeV/A96xLQ9</latexit> W21 <latexit sha1_base64="P2oHYPL1zEwh4CR/Fop55holW90=">AAACUHicfZBNSyNBEIZr4sfG8WONHr0MBkFEwkwU9Si4By+iwiYRnCg1nUps7e4ZunvEMOQ/7FV/1t72n3jTToywRrGg6Ye33qaq3yQT3Ngw/OeVpqZnZn+U5/z5hcWln8uVlaZJc82owVKR6osEDQmuqGG5FXSRaUKZCGold0fDfuuetOGp+m37GbUl9hTvcobWSc3WVbFTH1wvV8NaOKrgM0RjqMK4zq4r3m7cSVkuSVkm0JjLKMxsu0BtORM08OPcUIbsDnt06VChJNMuRusOgg2ndIJuqt1RNhip/78oUBrTl4lzSrQ3ZrI3FL/sJXJisu0etAuustySYm+Du7kIbBoMswg6XBOzou8AmeZu94DdoEZmXWK+H/8i9zlNJ27QaUYabaq3ihh1T3I1cJ/txdtD+s6ID+9GRy7naDLVz9Cs16K92t75bvXwaJx4GdZgHTYhgn04hGM4gwYwuIU/8AhP3l/v2XspeW/W9xtW4UOV/Fd+lbQ/</latexit> W32 # <latexit sha1_base64="9lo7Ig4L9iW1kjp3PrYcMIEiVqI=">AAACaXicfZDdSgMxEIXT9b/+tXojihAsgoiUXREVQRD0whtRwargljKbTttgkl2SrFiWXvk03urT+Ay+hGldQas4EPg4c4bJnCgR3Fjffyt4I6Nj4xOTU8Xpmdm5+VJ54drEqWZYY7GI9W0EBgVXWLPcCrxNNIKMBN5E98f9/s0DasNjdWW7CdYltBVvcQbWSY3SatoIQSQdoAeHFL44yqFRqvhVf1D0NwQ5VEheF41yYSdsxiyVqCwTYMxd4Ce2noG2nAnsFcPUYALsHtp451CBRFPPBnf06LpTmrQVa/eUpQP1+0QG0piujJxTgu2Y4V5f/LMXyaHNtrVfz7hKUouKfS5upYLamPZDok2ukVnRdQBMc/d3yjqggVkXZbEYnqA7TuOZW3SeoAYb680sBN2WXPXcse1wq0//GeHxy+jI5RwMp/obrrerwW5193KncnScJz5JVsga2SAB2SNH5JRckBph5Ik8kxfyWnj3yt6St/xp9Qr5zCL5UV7lA1Teu5g=</latexit> u↵ := a↵b↵
1 n n X i=1 (yif(xi)) + 2LRn(F) + r ln(2/ ) 2n " Bartlett, P., & Mendelson, S. (2002). Rademacher and Gaussian complexities: Risk bounds and structural results.
1 n n X i=1 (yif(xi)) + 2LRn(F) + r ln(2/ ) 2n " Bartlett, P., & Mendelson, S. (2002). Rademacher and Gaussian complexities: Risk bounds and structural results.
1 n n X i=1 (yif(xi)) + 2LRn(F) + r ln(2/ ) 2n " Bartlett, P., & Mendelson, S. (2002). Rademacher and Gaussian complexities: Risk bounds and structural results.
1 n n X i=1 (yif(xi)) + 2LRn(F) + r ln(2/ ) 2n " Bartlett, P., & Mendelson, S. (2002). Rademacher and Gaussian complexities: Risk bounds and structural results.
1 n n X i=1 (yif(xi)) + 2LRn(F) + r ln(2/ ) 2n " Bartlett, P., & Mendelson, S. (2002). Rademacher and Gaussian complexities: Risk bounds and structural results. Rademacher ෳࡶ ϊΠζͷద߹ͷ͢͠͞
w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
<latexit sha1_base64="RG5XaoGbDSP8VCyzHoRXbyj69Lw=">AAACsHicfZFdaxQxFIaz41ddv7Z66U1wEarIMlNK7Y1QqBfeiBXcdqFZhjPZM7uxSWZIzlSX6fxB/4H/wlu9MrMdsd2KBwJv3vOEk7zJSq08xfH3XnTj5q3bdzbu9u/df/Dw0WDz8ZEvKidxLAtduEkGHrWyOCZFGielQzCZxuPs9KDtH5+h86qwn2hZ4tTA3KpcSaBgpYOZ0MqkNQkqhLI5LRuRO5C1MECLLOdftuhFU4vzy3tx3vA3vOMWQH/hZoVetQKdDobxKF4Vvy6STgxZV4fpZm9HzApZGbQkNXh/ksQlTWtwpKTGpi8qjyXIU5jjSZAWDPppvYqj4c+DM+N54cKyxFfu5RM1GO+XJgtke0u/3mvNf/YyszaZ8r1prWxZEVp5MTivNKeCt1nzmXIoSS+DAOlUuDuXCwixUfiRfl+8xfA4h+/DoA8lOqDCvawFuLlRtgmPnYtXrfofCF//gEGFnJP1VK+Lo+1Rsjva/bgz3D/oEt9gT9kztsUS9prts3fskI2ZZN/YD/aT/Yq2o0mURnCBRr3uzBN2paLPvwF3GNoo</latexit> lim t!1 w(t) kw(t)k = ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) <latexit sha1_base64="2beUvCkyvZB/RdhsLSZwXSLFBHA=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVg/37ip1oNGMCj6G8Ih1MmwTm9q3k7USVkuUVkmwJjrMMhsuwBtORPYr0S5wQzYHXTx2qECiaZdDJbu0zWndGiSaveUpQP1+48CpDE9GTtnuaQZ7ZXin71Yjky2yX674CrLLSr2OTjJBbUpLROhHa6RWdFzAExztztlt6CBWZdbpRIdojtO47EbdJKhBpvqzSIC3ZVc9d2x3WirpP+M8PhldORyDkdT/Q0X241wt7F7tlNvHgwTnyYrZJWsk5DskSY5IqekRRjJyBN5Ji/eq/fuT/hTn1bfG/5ZJj/Kr34ADoa1dA==</latexit> w(T) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
ˆ w k ˆ wk <latexit sha1_base64="Zw/0V8Yw0bx42w0Ht/geiS1KiM0=">AAACW3icfZDBSiNBEIY7o646665R8eSlMSzIsoQZEdejoAcvosJGBSeEmk5N0tjdM3TXrIYhT7LX9aE8+C72xAhuXCxo+Pjrb6rqTwslHUXRYyOYm1/4tLi0HH5e+fJ1tbm2funy0grsiFzl9joFh0oa7JAkhdeFRdCpwqv09qjuX/1G62RuftGowK6GgZGZFEBe6jVXkyFQlWigYZrxu3Gv2Yra0aT4e4in0GLTOu+tNfaSfi5KjYaEAudu4qigbgWWpFA4DpPSYQHiFgZ449GARtetJpuP+Tev9HmWW/8M8Yn69kcF2rmRTr2zXtHN9mrxv71Uz0ym7KBbSVOUhEa8DM5KxSnndSy8Ly0KUiMPIKz0u3MxBAuCfHhhmByjP87iqR90VqAFyu33KgE70NKM/bGD5EdNHxnh/tXoyeccz6b6Hi532/F+e/9ir3V4NE18iW2xbbbDYvaTHbITds46TLCS/WF/2UPjKZgLwmDlxRo0pn822D8VbD4DluO2sA==</latexit> ˆ w <latexit sha1_base64="BlYTNRSSw5Mn+3FsC7sLJk53Ec4=">AAACzXicfVFda9RAFJ3Er7pau1XQB18GF0GkhE0ptfhUqA++SKu4baGzDZPZu+nQySTM3NQN2QR88jf6C/wbTnZTP7bihYFzzzmXe+feOFfS4nD43fNv3b5z997a/d6Dh+uPNvqbj49tVhgBI5GpzJzG3IKSGkYoUcFpboCnsYKT+PKg1U+uwFiZ6c9Y5jBOeaLlVAqOjor6XxnCDC2WClgqdVSxlONFPKVfmNR0mcTVp/p8UrP5b21+vk0XlVXT2ACDpqnZNDNcKSrfUqa4ThTQX/6tMpLXycxB0+kJ0DDqD4bBcBH0Jgg7MCBdHEWb3g6bZKJIQaNQ3NqzcJjjuOIGpVBQ91hhIefikidw5qDmKdhxtdhVTV86ZkLdrO5ppAv2z4qKp9aWaeyc7cB2VWvJf2pxutIZp3vjSuq8QNBi2XhaKIoZbQ9BJ9KAQFU6wIWRbnYqLrjhAt25ej32DtznDHxwjQ5zMBwz87pi3CTuTLX7bMK2WvQ/I59dGx1yew5Xt3oTHG8H4W6w+3FnsH/QbXyNPCcvyCsSkjdkn7wnR2REBPnhrXtPvWf+oV/4c79ZWn2vq3lC/gr/209H+uI+</latexit> minw2Rd kwk2 s.t. 8i : hw, yixi i 1 <latexit sha1_base64="b9lHQB0WiDlL52SNoShtYtOjOvY=">AAACW3icfVBRSxtBEN6crdqr1VjpU18WQ6GUEu6CxORNsA++SC00KnghzG3mkiW7e8funDYc+SV9bX+UD/4X92KEVloHBj6+7xtm5ksLJR1F0W0jWHvxcn1j81X4euvN9k5z9+25y0srcCByldvLFBwqaXBAkhReFhZBpwov0tlxrV9co3UyN99pXuBQw8TITAogT42aO8kUqEo00DTN+M1i1GxF7chXt8trEPei2IN+v9fp9Hm8lKKoxVZ1NtptHCTjXJQaDQkFzl3FUUHDCixJoXARJqXDAsQMJnjloQGNblgtL1/wD54Z8yy3vg3xJfvnRAXaublOvbM+0T3VavKfWqqfbKasN6ykKUpCIx4WZ6XilPM6Fj6WFgWpuQcgrPS3czEFC4J8eGGYfEH/nMVTv+hrgRYot5+qBOxES7Pwz06SzzV6zgg/Ho0e+Zwfw+T/B+eddtxtd78dtI6OV4lvsvdsn31kMTtkR+yEnbEBE6xkP9kv9rtxF6wFYbD1YA0aq5k99lcF7+4BBc627Q==</latexit> ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0) <latexit sha1_base64="2beUvCkyvZB/RdhsLSZwXSLFBHA=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVg/37ip1oNGMCj6G8Ih1MmwTm9q3k7USVkuUVkmwJjrMMhsuwBtORPYr0S5wQzYHXTx2qECiaZdDJbu0zWndGiSaveUpQP1+48CpDE9GTtnuaQZ7ZXin71Yjky2yX674CrLLSr2OTjJBbUpLROhHa6RWdFzAExztztlt6CBWZdbpRIdojtO47EbdJKhBpvqzSIC3ZVc9d2x3WirpP+M8PhldORyDkdT/Q0X241wt7F7tlNvHgwTnyYrZJWsk5DskSY5IqekRRjJyBN5Ji/eq/fuT/hTn1bfG/5ZJj/Kr34ADoa1dA==</latexit> w(T) " Soudry, D., Hoffer, E., Nacson, M., Gunasekar, S., & Srebro, N. (2018). The implicit bias of gradient descent on separable data.
<latexit sha1_base64="/5xVUTgu9hqLfvIQJ/px178bQcU=">AAACVHicfVBdS+NAFJ3Ej3XjqlUf9mFfhi2CLFIScat9E/XBl8UKWxWaUm6mN+3gZBJmJmIJ+TX7uv4dwf/ig5N+gC6rBwbOnHvuzL0nygTXxvefHHdhcWn508pnb/XL2vpGbXPrSqe5YthhqUjVTQQaBZfYMdwIvMkUQhIJvI5uT6v69R0qzVP524wz7CUwlDzmDIyV+rWvYQJmFMVF2B7xcn65L/u1ut9o+UHr5yGdkqODGWm2aNDwJ6iTGdr9TWcnHKQsT1AaJkDrbuBnpleAMpwJLL0w15gBu4Uhdi2VkKDuFZMNSrpjlQGNU2WPNHSivu4oINF6nETWWY2o/61V4v9q3dzER72Cyyw3KNn0ozgX1KS0ioMOuEJmxNgSYIrbWSkbgQJmbGieF56hXUbhL/vwRYYKTKp+FCGoYcJlaZcbhnsV+8gI93OjZZ4Ndp4efZ9c7TeCZqN5eVA/PplFvEK+ke9klwTkkByTc9ImHcJISf6Qv+TBeXSe3QV3aWp1nVnPNnkDd/0Fbmy1Uw==</latexit> x <latexit sha1_base64="8osqE/1MTXNe11T2/tw9Ei9cA68=">AAACXnicfZDPTttAEMY3hlIwpQS4IHHZEiFVVRXZFQRyQ8CBCyKVGoIUR9F4M05WrNfW7hoRWT7zNFzhWbjxKKzzB9Gq7Ugr/fabmd2ZL0wF18bznivOwuKHpY/LK+7qp7XP69WNzSudZIphmyUiUdchaBRcYttwI/A6VQhxKLAT3pyW+c4tKs0T+cuMU+zFMJQ84gyMlfrVL0EMZhRGeaeYU9Aa8bfLXdGv1rx60/ObB4d0Ckf7M2g0qV/3JlEjs2j1Nyp7wSBhWYzSMAFad30vNb0clOFMYOEGmcYU2A0MsWtRQoy6l092KeieVQY0SpQ90tCJ+r4jh1jrcRzaynJE/WeuFP+W62YmOurlXKaZQcmmH0WZoCahpTF0wBUyI8YWgCluZ6VsBAqYsfa5bnCGdhmFF/bhyxQVmER9ywNQw5jLwi43DL6X9L9CuJsXWnKtsXP36L/h6kfdb9QbP/drxyczi5fJDtklX4lPDskxOSct0iaM3JMH8kieKi/OkrPmrE9LncqsZ4v8Fs72K8e2uNU=</latexit> W x Stop Grad <latexit sha1_base64="kBPSW11GbXNZi8RGtRiuapnCgeA=">AAACRXicfZDNSsNAFIUn/tb41+rSTbAURKQkItVlURduRAVrxabIzfSmDp1MwsxELKFv4VZfx2fwIdyJW53UClrFCwMf556ZufcECWdKu+6zNTE5NT0zW5iz5xcWl5aLpZULFaeSYoPGPJaXASjkTGBDM83xMpEIUcCxGfQO8n7zFqVisTjX/QTbEXQFCxkFbaQrPwJ9E4RZc3BdLLtVd1jOb/BGUCajOr0uWRW/E9M0QqEpB6VanpvodgZSM8pxYPupwgRoD7rYMiggQtXOhiMPnIpROk4YS3OEdobq9xsZREr1o8A48xHVeC8X/+q1Uh3utTMmklSjoJ8fhSl3dOzk+zsdJpFq3jcAVDIzq0NvQALVJiXb9g/RLCPx2Dx8kqAEHcvNzAfZjZgYmOW6/lZO/xnh7stoyDbBeuMx/oaL7apXq9bOdsr1/VHEBbJG1skG8cguqZMjckoahBJB7skDebSerBfr1Xr7tE5Yozur5EdZ7x9bnrHZ</latexit> W <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="3hJD/RTeoGI1v4fhSpNhj7iQlus=">AAACSHicfZDNSsNAFIUn9T/+69JNsBREpCQi6lLUhRuxgq1CE8rN9KYdOpnEmYlYQp/Drb6Ob+BbuBN3TmoFteKFgY9zz8zce8KUM6Vd98UqTUxOTc/MztnzC4tLyyuraw2VZJJinSY8kTchKORMYF0zzfEmlQhxyPE67J0U/es7lIol4kr3Uwxi6AgWMQraSIEfg+6GUe7XumzQWim7VXdYzjh4IyiTUdVaq1bFbyc0i1FoykGppuemOshBakY5Dmw/U5gC7UEHmwYFxKiCfDj1wKkYpe1EiTRHaGeofr+RQ6xUPw6Ns5hS/e4V4l+9ZqajwyBnIs00Cvr5UZRxRydOEYHTZhKp5n0DQCUzszq0CxKoNkHZtn+KZhmJ5+bhixQl6ERu5z7ITszEwCzX8XcK+s8I919GQ7YJ1vsd4zg0dqvefnX/cq98dDyKeJZskE2yRTxyQI7IGamROqHkljyQR/JkPVuv1pv1/mktWaM76+RHlUof9c+yHg==</latexit> <latexit sha1_base64="BhlfswrLabLnLZv8qiWt2tZD++A=">AAACR3icfZDNSgMxFIUz9X/816WbYBFEpMyIVrsTdeFGVLAqdEq5k96pwUxmSDJiGfoabvV1fASfwp24NNNWUFEvBD7OPUnuPWEquDae9+KURkbHxicmp9zpmdm5+YXFpUudZIphnSUiUdchaBRcYt1wI/A6VQhxKPAqvD0s+ld3qDRP5IXpptiMoSN5xBkYKwVBDOYmjPL7XstrLZS9Ss3zazu7dAB720Oo1qhf8fpVJsM6ay06a0E7YVmM0jABWjd8LzXNHJThTGDPDTKNKbBb6GDDooQYdTPvD92ja1Zp0yhR9khD++rXGznEWnfj0DqLIfXPXiH+1mtkJtpr5lymmUHJBh9FmaAmoUUCtM0VMiO6FoApbmel7AYUMGNzct3gCO0yCk/sw6cpKjCJ2sgDUJ2Yy55drhNsFvSfEe4/jZZcG+xnevRvuNyq+NVK9Xy7vH8wjHiSrJBVsk58skv2yTE5I3XCSEoeyCN5cp6dV+fNeR9YS87wzjL5ViXnA50Osf4=</latexit> x0 " Tian, Y., Chen, X., & Ganguli, S. (2021). Understanding self-supervised learning dynamics without contrastive pairs.
<latexit sha1_base64="oBHF1h2GFlEaPhB/HhOweL1oh7I=">AAACR3icfZDfShtBFMZnY/3TrdWol70ZGoQiJexKTMxd0F54U4zQxEA2hLOTs8mY2dllZlYMS16jt+3r9BH6FN6VXnY2iVBFPTDw4zvfzJzzhang2njeb6e09mZ9Y3Prrftu+/3Obnlvv6uTTDHssEQkqheCRsEldgw3AnupQohDgdfh9LzoX9+i0jyR38wsxUEMY8kjzsBYKQhiMJMwyu/mw5thueJVm57fPGnQJZzWVlBvUr/qLapCVtUe7jmHwShhWYzSMAFa930vNYMclOFM4NwNMo0psCmMsW9RQox6kC+GntNDq4xolCh7pKEL9f8bOcRaz+LQOosh9dNeIT7X62cmOh3kXKaZQcmWH0WZoCahRQJ0xBUyI2YWgCluZ6VsAgqYsTm5bvAF7TIKv9qHL1NUYBJ1lAegxjGXc7vcOPhc0GtGuHswWnJtsA/p0Zehe1z169X6Va3SOltFvEU+kI/kE/FJg7TIBWmTDmEkJd/JD/LT+eXcO3+cv0tryVndOSCPquT8AwgNsjg=</latexit> xj <latexit sha1_base64="gwzGj/YlW7S86DEYY7wqqSaKG3g=">AAACR3icfZDNSiNBFIWrM/62P2N06aYwCCISusWJyU7GWbgRFYwK6RBuV24npdXVTVW1GJq8hlvndeYR5inciUurkwgq6oWCj3NPVd17wlRwbTzvv1P6MTU9Mzs37y4sLi3/XCmvXugkUwybLBGJugpBo+ASm4YbgVepQohDgZfhzWHRv7xFpXkiz80gxXYMPckjzsBYKQhiMP0wyvvDznVnpeJVG57f+LVPx1Dfm0CtQf2qN6oKmdRpp+xsBt2EZTFKwwRo3fK91LRzUIYzgUM3yDSmwG6ghy2LEmLU7Xw09JBuWqVLo0TZIw0dqW9v5BBrPYhD6yyG1B97hfhZr5WZqN7OuUwzg5KNP4oyQU1CiwRolytkRgwsAFPczkpZHxQwY3Ny3eAP2mUUHtuHT1JUYBK1nQegejGXQ7tcL9gp6Dsj3L0aLbk22Nf06NdwsVv1a9Xa2V7l4Pck4jmyTjbIFvHJPjkgR+SUNAkjKbknD+Sv8895dJ6c57G15EzurJF3VXJeAOpOsig=</latexit> hj <latexit sha1_base64="M7L9l5V9YJ8k17bUYdQox0J08kU=">AAACR3icfZDPSiNBEMZ7ou5mR3c1etxLYxBEljAjmj+3sHrYixjBqJAJoaZTkzT29AzdPYthyGt4XV/HR/ApvMke7Uki6LJa0PDjq6+7q74wFVwbz3twSkvLK58+l7+4q2tfv61vVDYvdJIphl2WiERdhaBRcIldw43Aq1QhxKHAy/D6qOhf/kaleSLPzSTFfgwjySPOwFgpCGIw4zDKx9MBH2xUvVrL81uHDTqH5sEC6i3q17xZVcmiOoOKsxMME5bFKA0ToHXP91LTz0EZzgRO3SDTmAK7hhH2LEqIUffz2dBTumOVIY0SZY80dKa+vpFDrPUkDq2zGFL/2yvE//V6mYma/ZzLNDMo2fyjKBPUJLRIgA65QmbExAIwxe2slI1BATM2J9cNjtEuo/DEPnyaogKTqL08ADWKuZza5UbBj4I+MsLNi9GSa4N9SY++Dxf7Nb9eq58dVNs/FxGXyXeyTXaJTxqkTX6RDukSRlJyS/6QO+feeXSenL9za8lZ3Nkib6rkPAPodrIn</latexit> hi <latexit sha1_base64="ZOTBYZ4IDYySEXx7pqSQY6rin/I=">AAACR3icfZDRShtBFIZnY6vpaq3Ry94MDYJICbslxuQuqBfeiAqNCtkQzk7OJkNmZ5eZWTEueQ1v9XV8BJ/Cu9JLZ5MIbakeGPj4zz8z5/xhKrg2nvfklJY+fFxeKX9yV9c+r3/ZqGxe6CRTDDssEYm6CkGj4BI7hhuBV6lCiEOBl+H4sOhfXqPSPJE/zSTFXgxDySPOwFgpCGIwozDKb6d93t+oerWW57f29ukcmvUFNFrUr3mzqpJFnfUrznYwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls6GndNsqAxolyh5p6Ez980YOsdaTOLTOYkj9b68Q/9frZiZq9nIu08ygZPOPokxQk9AiATrgCpkREwvAFLezUjYCBczYnFw3OEK7jMIT+/BpigpMonbzANQw5nJqlxsG3wt6zwg3r0ZLrg32NT36Nlz8qPmNWuO8Xm0fLCIuk6/kG9khPtknbXJMzkiHMJKSO3JPHpxH59n55fyeW0vO4s4W+atKzgsJ67I5</latexit> zi Stop Grad <latexit sha1_base64="UXgnZhucLLIyKxxyuDmLz6hD2XQ=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9OTdnAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7Qi6goWMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6C11sGRQQoWpno0kHTtkoHSeMpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVSHR+2MiSTVKOj4ozDljo6d4dpOh0mkmvcNAJXMzOrQHkig2oRj2/4pmmUknpuHLxKUoGO5k/kguxETA7Nc198d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIGytyA==</latexit> f <latexit sha1_base64="UXgnZhucLLIyKxxyuDmLz6hD2XQ=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9OTdnAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7Qi6goWMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6C11sGRQQoWpno0kHTtkoHSeMpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVSHR+2MiSTVKOj4ozDljo6d4dpOh0mkmvcNAJXMzOrQHkig2oRj2/4pmmUknpuHLxKUoGO5k/kguxETA7Nc198d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIGytyA==</latexit> f <latexit sha1_base64="ohipUW93olWICBW31kkLiUn2UJE=">AAACPHicfZDLSgMxFIYz9T7eqi7dDJaCiJQZkepS1IUb0YK9QKeUM+lpG5rJDElGLEOfwK2+ju/h3p24dW16EbQVDwQ+/vMnOecPYs6Udt1XKzM3v7C4tLxir66tb2xmt7YrKkokxTKNeCRrASjkTGBZM82xFkuEMOBYDXoXw371HqVikbjT/RgbIXQEazMK2kilbjObcwvuqJxZ8CaQI5O6bW5Zeb8V0SREoSkHpeqeG+tGClIzynFg+4nCGGgPOlg3KCBE1UhHkw6cvFFaTjuS5gjtjNSfN1IIleqHgXGGoLtqujcU/+rVE90+baRMxIlGQccftRPu6MgZru20mESqed8AUMnMrA7tggSqTTi27V+iWUbitXn4JkYJOpIHqQ+yEzIxMMt1/MMh/WeEh2+jIdsE603HOAuVo4JXLBRLx7mz80nEy2SX7JF94pETckauyC0pE0qQPJIn8my9WG/Wu/UxtmasyZ0d8quszy8kHK3K</latexit> h <latexit sha1_base64="WynUUi/JkqYOBDgkoTqFVFEGUzg=">AAACRXicfZDNSgMxFIUz/jv+69JNsAgiUmZEq92JunAjKlgVO0XupHdqMJMZkoxYhr6FW30dn8GHcCduNdNWUFEvBD7OPUnuPWEquDae9+wMDA4Nj4yOjbsTk1PTM7Nz82c6yRTDGktEoi5C0Ci4xJrhRuBFqhDiUOB5eLNX9M9vUWmeyFPTTrERQ0vyiDMwVroMYjDXYZTfda5mS1656vnVzS3ag+2NPlSq1C973SqRfh1fzTnLQTNhWYzSMAFa130vNY0clOFMYMcNMo0psBtoYd2ihBh1I++O3KHLVmnSKFH2SEO76tcbOcRat+PQOosR9c9eIf7Wq2cm2m7kXKaZQcl6H0WZoCahxf60yRUyI9oWgCluZ6XsGhQwY1Ny3WAf7TIKD+3DRykqMIlazQNQrZjLjl2uFawV9J8R7j6Nllwb7Gd69G84Wy/7lXLlZKO0s9uPeIwskiWyQnyyRXbIATkmNcKIJPfkgTw6T86L8+q89awDTv/OAvlWzvsHQuSyWg==</latexit> x <latexit sha1_base64="UXgnZhucLLIyKxxyuDmLz6hD2XQ=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9OTdnAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7Qi6goWMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6C11sGRQQoWpno0kHTtkoHSeMpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVSHR+2MiSTVKOj4ozDljo6d4dpOh0mkmvcNAJXMzOrQHkig2oRj2/4pmmUknpuHLxKUoGO5k/kguxETA7Nc198d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIGytyA==</latexit> f <latexit sha1_base64="0VGLZ6c/DGB+9vEkCqaS7AY0zLg=">AAACRXicfZDLShxBFIar1RjTSYzGZTaFw0CQMHSLmctO1IWboIJzIdPDcLrm9Ewx1dVNVXXI0MxbuNXX8Rl8CHfiNlbPBRLRHCj4+M9fVef8YSq4Np5356ysrr1Zf7vxzn3/4ePmp63tzy2dZIphkyUiUZ0QNAousWm4EdhJFUIcCmyH4+Oi3/6FSvNEXppJir0YhpJHnIGx0s8gBjMKo3w07W+VvErD8xvfa3QO9YMFVBvUr3izKpFFnfe3nXIwSFgWozRMgNZd30tNLwdlOBM4dYNMYwpsDEPsWpQQo+7ls5GntGyVAY0SZY80dKb+fSOHWOtJHFpnMaJ+3ivEl3rdzET1Xs5lmhmUbP5RlAlqElrsTwdcITNiYgGY4nZWykaggBmbkusGJ2iXUfjDPnyWogKTqL08ADWMuZza5YbBt4L+Z4TfS6Ml1wa7TI++Dq39il+tVC8OSodHi4g3yBeyS74Sn9TIITkl56RJGJHkilyTG+fWuXcenMe5dcVZ3Nkh/5Tz5wklVLJK</latexit> h <latexit sha1_base64="5pCyBpbCEsb/Wsd15Ao+4VD0Ajc=">AAACPHicfZDLSsNAFIYn9VbjrdWlm2ApiEhJRNRlURduxBasFZoiJ9PTdHAyCTMTsYQ+gVt9Hd/DvTtx69rpRdAqHhj4+M8/M+f8QcKZ0q77YuVmZufmF/KL9tLyyupaobh+peJUUmzQmMfyOgCFnAlsaKY5XicSIQo4NoPbk2G/eYdSsVhc6n6C7QhCwbqMgjZSPbwplNyKOyrnN3gTKJFJ1W6KVtnvxDSNUGjKQamW5ya6nYHUjHIc2H6qMAF6CyG2DAqIULWz0aQDp2yUjtONpTlCOyP1+40MIqX6UWCcEeiemu4Nxb96rVR3j9oZE0mqUdDxR92UOzp2hms7HSaRat43AFQyM6tDeyCBahOObfunaJaReG4evkhQgo7lTuaDDCMmBma50N8d0n9GuP8yGrJNsN50jL/haq/iHVQO6vul6vEk4jzZJFtkm3jkkFTJGamRBqEEyQN5JE/Ws/VqvVnvY2vOmtzZID/K+vgEIkStyQ==</latexit> g Stop Grad augmented signals original signal CA predictor CA predictor CA loss CA loss " Ishikawa, S., Yamada, M., Bao, H., Takezawa, Y. (2025). PhiNets: Brain-inspired non-contrastive learning based on temporal prediction hypothesis.
ˆ w <latexit sha1_base64="nd5RJtKI1vr5qCfT2VrS2rB0b5o=">AAACVnicfZDdSgMxEIWz63/9a/XSm2ARVKTsiqiXBb3wRlSwKrhFZtPZGkyyS5JVy9LX8FYfS19GzNYKWsWBwMeZE2bmxJngxgbBm+ePjU9MTk3PVGbn5hcWq7WlC5PmmmGLpSLVVzEYFFxhy3Ir8CrTCDIWeBnfHZT9y3vUhqfq3PYybEvoKp5wBtZJUSTB3sYJfVgPNm6q9aARDIr+hnAIdTKs05uatxN1UpZLVJYJMOY6DDLbLkBbzgT2K1FuMAN2B128dqhAomkXg6X7dM0pHZqk2j1l6UD9/qMAaUxPxs5ZLmlGe6X4Zy+WI5Ntst8uuMpyi4p9Dk5yQW1Ky0Roh2tkVvQcANPc7U7ZLWhg1uVWqUSH6I7TeOwGnWSowaZ6s4hAdyVXfXdsN9oq6T8jPH4ZHbmcw9FUf8PFdiPcbeye7dSbB8PEp8kKWSXrJCR7pEmOyClpEUYy8kSeyYv36r37E/7Up9X3hn+WyY/yqx/J17VQ</latexit> w(0)
Y. (2003). Introductory lectures on convex optimization: A basic course. - <latexit sha1_base64="T3+E/1NrjncdQhovxjrAHxzJsuU=">AAACZ3icfVBdSxtBFJ2stmpam2hBCr6shkKREnYlROmTYB/6YPEDo4KbhruTmzg4H8vMXTEseeiv6Wv9Of6E/gtnY4ptLD0wcObcc7n3njSTwlEU3VeCufkXLxcWl6qvXi+/qdVXVs+cyS3HDjfS2IsUHEqhsUOCJF5kFkGlEs/T6/2yfn6D1gmjT2mUYVfBUIuB4EBe6tXXDz4lCugqTYuT8bd+Qubp26s3omY0QficxFPSYFMc9VYqraRveK5QE5fg3GUcZdQtwJLgEsfVJHeYAb+GIV56qkGh6xaTK8bhe6/0w4Gx/mkKJ+qfHQUo50Yq9c5yRTdbK8V/1lI1M5kGu91C6Cwn1Pxx8CCXIZmwjCjsC4uc5MgT4Fb43UN+BRY4+SCr1eQz+uMsfvWDDjO0QMZuFQnYoRJ67I8dJh9L9j8j3P42euZzjmdTfU7Otptxu9k+bjX29qeJL7J1tsk+sJjtsD32hR2xDuPsO/vBfrK7yq+gFqwF7x6tQWXa85b9hWDjAbJbu+E=</latexit> L : Rd ! R <latexit sha1_base64="h7xrZED8DEDDTlIZzQs9fydeZKo=">AAACT3icfZBNSwMxEIaz9autX1WPXhaLICJlV0Q9FvTgRVSwKrhFZtNpDSbZJZkVy9Lf4FV/lkd/iTcxrStoFQcCD++8YWbeOJXCUhC8eqWJyanpmXKlOjs3v7BYW1q+sElmOLZ4IhNzFYNFKTS2SJDEq9QgqFjiZXx3MOxf3qOxItHn1E+xraCnRVdwICe1ohgJbmr1oBGMyv8NYQF1VtTpzZK3E3USninUxCVYex0GKbVzMCS4xEE1yiymwO+gh9cONSi07Xy07cBfd0rH7ybGPU3+SP3+IwdlbV/FzqmAbu14byj+2YvV2GTq7rdzodOMUPPPwd1M+pT4wyj8jjDISfYdADfC7e7zWzDAyQVWrUaH6I4zeOwGnaRogBKzmUdgekrogTu2F20N6T8jPHwZHbmcw/FUf8PFdiPcbeye7dSbB0XiZbbK1tgGC9kea7IjdspajDPBHtkTe/ZevDfvvVRYS14BK+xHlSofJnC0Gw==</latexit> <latexit sha1_base64="xsZleZEN3LF1Q2cgXbCIKLx2uFU=">AAACYHicfVBdSxtBFJ1s1cbUj6T6pC9DQ6EUSXdLUB8FffBFVDCJ4IZwd3ITh8zMLjN3i3HJf+lr/Ue+9pd0EiNolB4YOHPuudx7T5Ip6SgMH0vBh6XllY/l1cqntfWNzWrtc9uluRXYEqlK7XUCDpU02CJJCq8zi6AThZ1kdDytd36hdTI1VzTOsKthaORACiAv9arbMRLECp1zUvPoR5z4f69aDxvhDPwtieakzua46NVKzbifilyjIaHAuZsozKhbgCUpFE4qce4wAzGCId54akCj6xaz9Sf8q1f6fJBa/wzxmfqyowDt3Fgn3qmBbt1ibSq+W0v0wmQaHHYLabKc0IinwYNccUr5NBvelxYFqbEnIKz0u3NxCxYE+QQrlfgE/XEWz/yg8wwtUGq/FzHYoZZm4o8dxntT9j8j3D0bPfM5R4upviXtn41ov7F/2awfHc8TL7Nd9oV9YxE7YEfslF2wFhPsnv1mf9hD6W9QDjaD2pM1KM17ttgrBDv/ADUXuFY=</latexit> ⌘ . 1/ <latexit sha1_base64="01PnSFuqSsJTebtdZgUk3NZYgTk=">AAACdHicfVBNaxsxEJW3H0ndjzjtsTmImoBTitkNIe0xkB56aEgKdhKIjJmVZ9ciknaRZtuaxX+hv6bX9n/0j/RcreNA4oQOSDzevOHNvLTUylMc/2lFDx4+ery2/qT99NnzFxudzZenvqicxKEsdOHOU/ColcUhKdJ4XjoEk2o8Sy8Pm/7ZV3ReFXZAsxJHBnKrMiWBAjXu9D73hAGapln9bT4e7AiN/Dh8GfVE5kAmA+FUPqWdcacb9+NF8bsgWYIuW9bJeLO1JyaFrAxakhq8v0jikkY1OFJS47wtKo8lyEvI8SJACwb9qF6cNOfbgZnwrHDhWeIL9uZEDcb7mUmDslnfr/Ya8t5ealacKfswqpUtK0Irr4yzSnMqeJMXnyiHkvQsAJBOhd25nEJIhkKq7bb4iOE4h0fB6LhEB1S4t7UAlxtl5+HYXLxr0P+E8P1aGFDIOVlN9S443e0n+/39L3vdg8Nl4uvsNXvDeixh79kB+8RO2JBJ9oP9ZL/Y79bfaCvqRttX0qi1nHnFblXU/wdkgcAs</latexit> L(wT ) O ✓ 1 T ◆
& Goldstein, T. (2018). Visualizing the loss landscape of neural nets. [Li+ ] <latexit sha1_base64="xSP7Z+mT2idx0E6kPyi9VtfYiRA=">AAACVHicfZBNSwMxEIaz63f9qnr0slgEESm7ItqLIOjBi6hgVXSrzKbTGkyySzIrlqX/wqv+LMH/4sG0VtAqDgQe3nnDzLxJJoWlMHzz/JHRsfGJyanS9Mzs3Hx5YfHcprnhWOepTM1lAhal0FgnQRIvM4OgEokXyf1+r3/xgMaKVJ9RJ8OGgrYWLcGBnHQVJ0iwG4U3tdtyJayG/Qp+QzSAChvUye2CtxU3U54r1MQlWHsdhRk1CjAkuMRuKc4tZsDvoY3XDjUotI2iv3I3WHVKM2ilxj1NQV/9/qMAZW1HJc6pgO7scK8n/tlL1NBkatUahdBZTqj55+BWLgNKg14eQVMY5CQ7DoAb4XYP+B0Y4ORSK5XiA3THGTxyg44zNECpWS9iMG0ldNcd2443evSfER6/jI5cztFwqr/hfLMabVe3T7cqe/uDxCfZMlthayxiO2yPHbITVmecafbEntmL9+q9+yP+2KfV9wZ/ltiP8uc+ABRptII=</latexit> = 108
& Lee, J. (2025). Understanding optimization in deep learning with central flows. <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t))
& Lee, J. (2025). Understanding optimization in deep learning with central flows. <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) む EoS <latexit sha1_base64="jY31QyIhqDocjVM8FcyAqM0t5uw=">AAACp3icfVFdaxNBFJ2sXzV+NNVHXwaDkIgNu6HUvgiF+uCD0oomDWRiuDs7ux06M7vM3K2GZX+cP8Nf4Kv+A2fTFdpUvDBwOOdc7r1n4kJJh2H4oxPcun3n7r2t+90HDx893u7tPJm6vLRcTHiucjuLwQkljZigRCVmhRWgYyVO4/OjRj+9ENbJ3HzGVSEWGjIjU8kBPbXszVlqgVcJ04BncUq/DnBYVwnW9A3dZQZiBfT94Ko63F23RGPmZKbhy9hzrfHTdeOy1w9H4broTRC1oE/aOlnudPZYkvNSC4NcgXPzKCxwUYFFyZWou6x0ogB+DpmYe2hAC7eo1inU9IVnEprm1j+DdM1e7ahAO7fSsXc2W7pNrSH/qcV6YzKmB4tKmqJEYfjl4LRUFHPaREwTaQVHtfIAuJV+d8rPwGeG/iO6XfZW+OOs+OAHHRfCAub2ZcXAZlqa2h+bsVcN+p8Rvv01euRzjjZTvQmm41G0P9r/uNc/PGoT3yLPyHMyIBF5TQ7JO3JCJoST7+Qn+UV+B8PgOJgGs0tr0Gl7npJrFcAfVMHRfg==</latexit> dw(t) dt = rL(w(t)) 1 2 2(t)rS(w(t))
A., Kolter, J., & Lee, J. (2025). Understanding optimization in deep learning with central flows. <latexit sha1_base64="CARv2nJ+nhBeOVEQJmaph73E5fs=">AAACfnicfVBNSxxBEO2dmKibr1WPXposCWuImxkR9RIQzMGDIQayKjjLUtNTszZ29wzdNSbLMP/DX+PV/IX8m/SsK5g1pKDg8d4rquolhZKOwvB3K3iy8PTZ4tJy+/mLl69ed1ZWT1xeWoEDkavcniXgUEmDA5Kk8KywCDpReJpcHjT66RVaJ3PznSYFDjWMjcykAPLUqLMVZxZElcYa6CLJ+I8ebdRVSjX/xDdjA4kCftR7qG6MOt2wH06LPwbRDHTZrI5HK63tOM1FqdGQUODceRQWNKzAkhQK63ZcOixAXMIYzz00oNENq+lzNX/rmZRnufVtiE/ZhxMVaOcmOvHO5ko3rzXkP7VEz22mbG9YSVOUhEbcLc5KxSnnTXI8lRYFqYkHIKz0t3NxAT498vm22/Fn9M9Z/OIXfS3QAuX2fRWDHWtpav/sOP7QoP8Z4ee90SOfczSf6mNwstWPdvo737a7+wezxJfYOnvDeixiu2yfHbJjNmCCXbMbdst+BSx4F2wGH++sQWs2s8b+qmDvD5bBwlI=</latexit> dw(t) dt = rL(w(t)) む EoS <latexit sha1_base64="jY31QyIhqDocjVM8FcyAqM0t5uw=">AAACp3icfVFdaxNBFJ2sXzV+NNVHXwaDkIgNu6HUvgiF+uCD0oomDWRiuDs7ux06M7vM3K2GZX+cP8Nf4Kv+A2fTFdpUvDBwOOdc7r1n4kJJh2H4oxPcun3n7r2t+90HDx893u7tPJm6vLRcTHiucjuLwQkljZigRCVmhRWgYyVO4/OjRj+9ENbJ3HzGVSEWGjIjU8kBPbXszVlqgVcJ04BncUq/DnBYVwnW9A3dZQZiBfT94Ko63F23RGPmZKbhy9hzrfHTdeOy1w9H4broTRC1oE/aOlnudPZYkvNSC4NcgXPzKCxwUYFFyZWou6x0ogB+DpmYe2hAC7eo1inU9IVnEprm1j+DdM1e7ahAO7fSsXc2W7pNrSH/qcV6YzKmB4tKmqJEYfjl4LRUFHPaREwTaQVHtfIAuJV+d8rPwGeG/iO6XfZW+OOs+OAHHRfCAub2ZcXAZlqa2h+bsVcN+p8Rvv01euRzjjZTvQmm41G0P9r/uNc/PGoT3yLPyHMyIBF5TQ7JO3JCJoST7+Qn+UV+B8PgOJgGs0tr0Gl7npJrFcAfVMHRfg==</latexit> dw(t) dt = rL(w(t)) 1 2 2(t)rS(w(t))
X i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
X i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) Edge of stability <latexit sha1_base64="7ljW2Zg0AIm/INKJh8e/dgAFxk4=">AAAClnicfVBdaxNBFJ2sXzV+NLUvBV8Gg5CKhl0p1RdLoYoiSiOYttCJy+zs3c2Qmdll5q5tWPY/+WsEn/SnOJumoKl4YeDcc8/lzjlJqaTDMPzRCa5dv3Hz1trt7p279+6v9zYeHLmisgLGolCFPUm4AyUNjFGigpPSAteJguNkdtDOj7+CdbIwn3FewkTz3MhMCo6einvvWWa5iJC5Ssf17FXUfKnxWdR8GDDNcZpk9VkTz7aZAsrOZAooVQr00PcZDi53rcynuB33+uEwXBS9CqIl6JNljeKNzg5LC1FpMCgUd+40Ckuc1NyiFAqaLqsclFzMeA6nHhquwU3qhemGPvZMSrPC+meQLtg/N2qunZvrxCtbJ2511pL/nCV65TJmLye1NGWFYMTF4axSFAvaJkpTaUGgmnvAhZX+71RMuU8Gfe7dLnsN3pyFj/7QYQmWY2Gf1IzbXEvTeLM5e9qi/wn5+aXQI59ztJrqVXD0fBjtDnc/7fT3D5aJr5GH5BEZkIi8IPvkHRmRMRHkG/lOfpJfwVawF7wJ3l5Ig85yZ5P8VcHoN/+WzZU=</latexit> 1 t t 1 X k=1 L(wk) e O ✓ 1 t ◆ Phase " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
X i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss.
i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) 2 " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss. EoS stable
i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) 2 <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws) min ⇢ 1 C⌘ , `(0) n " Wu, J., Bartlett, P. L., Telgarsky, M., & Yu, B. (2024). Large stepsize gradient descent for logistic loss. EoS stable
i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws) min ⇢ 1 C⌘ , `(0) n <latexit sha1_base64="93f0ruv9t2WMuP6IyGCRk2Keak8=">AAACd3icfVBNS+RAEO2J37Orjnr0YLPDLqIyJCLqSQQ97EVU2FHBDKHSU4mN3Z3Q3VHHkB+xv2av+jP8KXuzM46wOy5b0PD6vVdU1YtzwY31/ZeGNzE5NT0zO9f89Hl+YbG1tHxhskIz7LJMZPoqBoOCK+xabgVe5RpBxgIv49ujWr+8Q214pn7YQY49CaniCWdgHRW1NkMBKhUYSrA3cVLeV5HZev88VhEP9VCnB9SPWm2/4w+LfgTBCLTJqM6ipcZO2M9YIVFZJsCY68DPba8EbTkTWDXDwmAO7BZSvHZQgUTTK4dXVfSrY/o0ybR7ytIh+2dHCdKYgYyds97XjGs1+U8tlmOTbbLfK7nKC4uKvQ1OCkFtRuvIaJ9rZFYMHACmududshvQwKwLttkMj9Edp/HEDTrNUYPN9EYZgk4lV5U7Ng23avQ/Izy8Gx1yOQfjqX4EF9udYLeze77TPjwaJT5LVskXsk4CskcOyXdyRrqEkZ/kF3kiz43f3pr3zVt/s3qNUc8K+au84BUs4cJn</latexit> hws, zi i > 0 EoS stable " Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.
i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws) min ⇢ 1 C⌘ , `(0) n <latexit sha1_base64="93f0ruv9t2WMuP6IyGCRk2Keak8=">AAACd3icfVBNS+RAEO2J37Orjnr0YLPDLqIyJCLqSQQ97EVU2FHBDKHSU4mN3Z3Q3VHHkB+xv2av+jP8KXuzM46wOy5b0PD6vVdU1YtzwY31/ZeGNzE5NT0zO9f89Hl+YbG1tHxhskIz7LJMZPoqBoOCK+xabgVe5RpBxgIv49ujWr+8Q214pn7YQY49CaniCWdgHRW1NkMBKhUYSrA3cVLeV5HZev88VhEP9VCnB9SPWm2/4w+LfgTBCLTJqM6ipcZO2M9YIVFZJsCY68DPba8EbTkTWDXDwmAO7BZSvHZQgUTTK4dXVfSrY/o0ybR7ytIh+2dHCdKYgYyds97XjGs1+U8tlmOTbbLfK7nKC4uKvQ1OCkFtRuvIaJ9rZFYMHACmududshvQwKwLttkMj9Edp/HEDTrNUYPN9EYZgk4lV5U7Ng23avQ/Izy8Gx1yOQfjqX4EF9udYLeze77TPjwaJT5LVskXsk4CskcOyXdyRrqEkZ/kF3kiz43f3pr3zVt/s3qNUc8K+au84BUs4cJn</latexit> hws, zi i > 0 EoS stable 2 " Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.
i=1 `(hw, zi i) <latexit sha1_base64="UqUu31KDfKZfPPqHtylbqfTZtOY=">AAACiHicfVBNaxRBEO0djSajSTbJ0UvjIsR8LDMSkggGgvHgQTGCmwQyy1LTW7Np0t0zdNeoyzB/xl+Tq978N/ZsVthsxIKCx3uvqKqXFko6iqLfreDBw4VHjxeXwidPl1dW22vrZy4vrcCeyFVuL1JwqKTBHklSeFFYBJ0qPE+vTxr9/CtaJ3PzhcYF9jWMjMykAPLUoP0m0UBXaVZ9qwcVbcc1P+IzFPFdniBBYiBVwD9szmovB+1O1I0mxe+DeAo6bFqng7XWXjLMRanRkFDg3GUcFdSvwJIUCuswKR0WIK5hhJceGtDo+tXkzZq/8MyQZ7n1bYhP2NmJCrRzY516Z3Olm9ca8p9aquc2U3bYr6QpSkIjbhdnpeKU8yZDPpQWBamxByCs9LdzcQUWBPmkwzB5h/45ix/9ok8FWqDcblUJ2JGWpvbPjpKdBv3PCN//Gj3yOcfzqd4HZ6+68X53//Ne5/hkmvgie8aes00WswN2zN6zU9Zjgv1gN+wn+xWEQRQcBK9vrUFrOrPB7lTw9g8vaMeY</latexit> wt+1 = wt ⌘rL(wt) <latexit sha1_base64="JW+yb2ql4nYT5YCxGzjH/2b8ytA=">AAACk3icfVBdixMxFE3Hr7V+bFfxyZdgEbqylBlZVsGXxS7og4sr2N2FTSmZ9M40bJIZkjtqCflJ/hqfBP0vZtoK2hUvhJycey435+S1kg7T9HsnuXb9xs1bW7e7d+7eu7/d23lw6qrGChiLSlX2POcOlDQwRokKzmsLXOcKzvLLUds/+wTWycp8xEUNE81LIwspOEZq2nvzbsA0x3le+M9h6naZAsq0NPEukHlWWC58FvyIAfKwt3ozUGqQ7gZvArOynCML014/HabLoldBtgZ9sq6T6U5nn80q0WgwKBR37iJLa5x4blEKBaHLGgc1F5e8hIsIDdfgJn7pONCnkZnRorLxGKRL9s8Jz7VzC51HZevObfZa8p+9XG9sxuLlxEtTNwhGrBYXjaJY0TZOOpMWBKpFBFxYGf9OxZzHkDCG3u2yI4jmLBzHRe9rsBwr+8wzbsuYcYhmS7bXov8J+ZffwohiztlmqlfB6fNhdjA8+LDfPxytE98ij8kTMiAZeUEOyVtyQsZEkK/kG/lBfiaPklfJ6+RoJU0665mH5K9Kjn8B6UDNQw==</latexit> L(ws) min ⇢ 1 C⌘ , `(0) n <latexit sha1_base64="93f0ruv9t2WMuP6IyGCRk2Keak8=">AAACd3icfVBNS+RAEO2J37Orjnr0YLPDLqIyJCLqSQQ97EVU2FHBDKHSU4mN3Z3Q3VHHkB+xv2av+jP8KXuzM46wOy5b0PD6vVdU1YtzwY31/ZeGNzE5NT0zO9f89Hl+YbG1tHxhskIz7LJMZPoqBoOCK+xabgVe5RpBxgIv49ujWr+8Q214pn7YQY49CaniCWdgHRW1NkMBKhUYSrA3cVLeV5HZev88VhEP9VCnB9SPWm2/4w+LfgTBCLTJqM6ipcZO2M9YIVFZJsCY68DPba8EbTkTWDXDwmAO7BZSvHZQgUTTK4dXVfSrY/o0ybR7ytIh+2dHCdKYgYyds97XjGs1+U8tlmOTbbLfK7nKC4uKvQ1OCkFtRuvIaJ9rZFYMHACmududshvQwKwLttkMj9Edp/HEDTrNUYPN9EYZgk4lV5U7Ng23avQ/Izy8Gx1yOQfjqX4EF9udYLeze77TPjwaJT5LVskXsk4CskcOyXdyRrqEkZ/kF3kiz43f3pr3zVt/s3qNUc8K+au84BUs4cJn</latexit> hws, zi i > 0 EoS stable 1 2 " Bao, H., Sakaue, S., & Takezawa, Y. (2025). Any-stepsize gradient descent for separable data under Fenchel–Young losses.